From 31d223160e4ded6e1fba3a1ee90a4184eeb1e436 Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Sun, 12 May 2024 21:34:06 +0200 Subject: [PATCH 1/2] fix/improvement: postgis postgres init take account of double init for healtcheck --- docker-compose.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index f8daf200..be1dd8e1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -48,11 +48,11 @@ services: test: [ 'CMD-SHELL', - "pg_isready --quiet --dbname=$${POSTGRES_DB:-bloom_db} --username=$${POSTGRES_USER:-bloom_user}" + "for i in 1 2 3; do pg_isready --quiet --dbname=$${POSTGRES_DB:-bloom_db} --username=$${POSTGRES_USER:-bloom_user} && break || sleep 5; done" ] - interval: 100ms - timeout: 14s - retries: 140 + interval: 15s + timeout: 45s + retries: 3 start_period: 0s bloom-frontend: @@ -84,7 +84,7 @@ services: # it happens that init is launch before second and definitve postgres healthy state # and fails # so giving init 3 chances and 15 seconds to init before failing - command: /bin/bash -c "cd backend; for i in 1 2 3; do alembic upgrade head && break || sleep 5; done" + command: /bin/bash -c "cd backend;alembic upgrade head" # Debug start: #command: bash #tty: true From 91b99b566d8371400c42ad6370fe122b06d7e76e Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Sun, 12 May 2024 22:50:42 +0200 Subject: [PATCH 2/2] fix: fix load data to use bloom/tasks (loading positions missing) --- .../init_script/load_positions_data.py | 2 +- .../bloom/tasks/load_spire_data_from_csv.py | 84 +++++++++++++++++++ docker-compose-load-data.yaml | 7 +- 3 files changed, 88 insertions(+), 5 deletions(-) create mode 100644 backend/bloom/tasks/load_spire_data_from_csv.py diff --git a/backend/alembic/init_script/load_positions_data.py b/backend/alembic/init_script/load_positions_data.py index a3d176d8..140e1809 100644 --- a/backend/alembic/init_script/load_positions_data.py +++ b/backend/alembic/init_script/load_positions_data.py @@ -13,7 +13,7 @@ df = pd.read_csv( Path(settings.data_folder).joinpath("./spire_positions_subset.csv"), - sep="," + sep=";" ) df.to_sql("spire_vessel_positions", engine, if_exists="append", index=False) diff --git a/backend/bloom/tasks/load_spire_data_from_csv.py b/backend/bloom/tasks/load_spire_data_from_csv.py new file mode 100644 index 00000000..3968b1f1 --- /dev/null +++ b/backend/bloom/tasks/load_spire_data_from_csv.py @@ -0,0 +1,84 @@ +from pathlib import Path +from time import perf_counter + +import pandas as pd +from bloom.config import settings +from bloom.container import UseCases +from bloom.domain.spire_ais_data import SpireAisData +from bloom.infra.database.errors import DBException +from bloom.logger import logger +from pydantic import ValidationError +from shapely import wkb +import numpy as np + + +def map_to_domain(row: pd.Series) -> SpireAisData: + isna = row.isna() + + return SpireAisData( + spire_update_statement=row['spire_update_statement'], + vessel_ais_class=row['vessel_ais_class'], + vessel_flag=row['vessel_flag'], + vessel_name=row['vessel_name'], + vessel_callsign=row['vessel_callsign'], + vessel_timestamp=row['vessel_timestamp'], + vessel_update_timestamp=row['vessel_update_timestamp'], + vessel_ship_type=row['vessel_ship_type'], + vessel_sub_ship_type=row['vessel_sub_ship_type'], + vessel_mmsi=row['vessel_mmsi'], + vessel_imo=row['vessel_imo'], + vessel_width=row['vessel_width'], + vessel_length=row['vessel_length'], + position_accuracy=row['position_accuracy'], + position_collection_type=row['position_collection_type'], + position_course=row['position_course'], + position_heading=row['position_heading'], + position_latitude=row['position_latitude'], + position_longitude=row['position_longitude'], + position_maneuver=row['position_maneuver'], + position_navigational_status=row['position_navigational_status'], + position_rot=row['position_rot'], + position_speed=row['position_speed'], + position_timestamp=row['position_timestamp'], + position_update_timestamp=row['position_update_timestamp'], + voyage_destination=row['voyage_destination'], + voyage_draught=row['voyage_draught'], + voyage_eta=row['voyage_eta'], + voyage_timestamp=row['voyage_timestamp'], + voyage_update_timestamp=row['voyage_update_timestamp'], + created_at=row['created_at'], + ) + + +def run(csv_file_name: str): + use_cases = UseCases() + db = use_cases.db() + spire_ais_data_repository = use_cases.spire_ais_data_repository() + + total = 0 + try: + df = pd.read_csv(csv_file_name, sep=";") + df = df.rename(columns={}) + df=df.replace(np.NaN,None) + spire_ais_data = df.apply(map_to_domain, axis=1) + with db.session() as session: + spire_ais_data = spire_ais_data_repository.batch_create_ais_data(session=session, ais_list=list(spire_ais_data)) + session.commit() + total = len(spire_ais_data) + #print(spire_ais_data) + except ValidationError as e: + logger.error("Erreur de validation des données de bateau") + logger.error(e.errors()) + except DBException: + logger.error("Erreur d'insertion en base") + logger.info(f"{total} ais data créés") + + +if __name__ == "__main__": + time_start = perf_counter() + file_name = Path(settings.data_folder).joinpath("./spire_positions_subset.csv") + logger.info(f"DEBUT - Chargement des données AIS Spire depuis le fichier {file_name}") + run(file_name) + time_end = perf_counter() + duration = time_end - time_start + logger.info(f"FIN - Chargement des données AIS Spire en {duration:.2f}s") diff --git a/docker-compose-load-data.yaml b/docker-compose-load-data.yaml index 32d9b601..c887e215 100644 --- a/docker-compose-load-data.yaml +++ b/docker-compose-load-data.yaml @@ -16,10 +16,9 @@ services: command: - -c - "/venv/bin/python3 backend/bloom/tasks/load_dim_vessel_from_csv.py && - /venv/bin/python3 backend/alembic/init_script/load_positions_data.py && - /venv/bin/python3 backend/alembic/init_script/load_amp_data.py && + /venv/bin/python3 backend/bloom/tasks/load_dim_zone_amp_from_csv.py && /venv/bin/python3 backend/bloom/tasks/load_dim_port_from_csv.py && - /venv/bin/python3 backend/bloom/tasks/compute_port_geometry_buffer.py" + /venv/bin/python3 backend/bloom/tasks/load_spire_data_from_csv.py" volumes: - ./:/project/ - ./data:/project/data @@ -29,7 +28,7 @@ services: networks: - bloom_net depends_on: - init: + bloom-init: condition: service_completed_successfully networks: