From e20984222432ba32c2b30d19fe7534ff2dd90033 Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Tue, 20 Apr 2021 15:28:28 +0300 Subject: [PATCH 01/23] 14329_fix_CI_for_clearcut --- clearcut_detection_backend/cron_tab_prod | 2 +- clearcut_detection_backend/docker-compose-prod.yml | 10 +++++----- clearcut_detection_backend/tasks.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/clearcut_detection_backend/cron_tab_prod b/clearcut_detection_backend/cron_tab_prod index 74da6f6..86b6272 100644 --- a/clearcut_detection_backend/cron_tab_prod +++ b/clearcut_detection_backend/cron_tab_prod @@ -1,3 +1,3 @@ # Cron tab prod -0 0,12 * * * python update.py +0 0,12 * * * python update_new.py diff --git a/clearcut_detection_backend/docker-compose-prod.yml b/clearcut_detection_backend/docker-compose-prod.yml index 1e42e11..ce6cdd6 100644 --- a/clearcut_detection_backend/docker-compose-prod.yml +++ b/clearcut_detection_backend/docker-compose-prod.yml @@ -7,7 +7,7 @@ db_prod: env_file: - db.env volumes: - - ./db-data_prod/data:/var/lib/postgresql/data + - /home/clearcut/prod/db-data_prod/data:/var/lib/postgresql/data - ./:/code model_prod_2: @@ -35,10 +35,10 @@ react_prod: volumes: - ./frontend:/frontend working_dir: /frontend - expose: - - "3001" - links: - - "django_prod" +# expose: +# - "3001" +# links: +# - "django_prod" command: /bin/bash -c "npm install && npm run-script build" django_prod: diff --git a/clearcut_detection_backend/tasks.py b/clearcut_detection_backend/tasks.py index f8cdd4b..d9abb61 100755 --- a/clearcut_detection_backend/tasks.py +++ b/clearcut_detection_backend/tasks.py @@ -60,6 +60,6 @@ def rundev(ctx, createdb=False): @task def runbackend(ctx): collect_static_element(ctx) - # thread_cron = threading.Thread(target=devcron, args=(ctx,)) - # thread_cron.start() + thread_cron = threading.Thread(target=devcron, args=(ctx,)) + thread_cron.start() ctx.run('uwsgi --ini uwsgi.ini') From 96e3af9fbca77a7126559c27c032b61d9ed30a91 Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Tue, 20 Apr 2021 16:02:05 +0300 Subject: [PATCH 02/23] 14329_fix_CI_for_clearcut --- clearcut_detection_backend/docker-compose-prod.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clearcut_detection_backend/docker-compose-prod.yml b/clearcut_detection_backend/docker-compose-prod.yml index ce6cdd6..0aa114e 100644 --- a/clearcut_detection_backend/docker-compose-prod.yml +++ b/clearcut_detection_backend/docker-compose-prod.yml @@ -76,8 +76,8 @@ nginx_prod: - "django_prod" expose: - "8080" -# ports: -# - "8082:8082" + ports: + - "8080:8080" environment: - VIRTUAL_HOST=clearcut.quantumobile.com - VIRTUAL_PORT=8080 From fe3eb32f27e092dae5abea22823356ec26a180b8 Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Mon, 17 May 2021 19:02:29 +0300 Subject: [PATCH 03/23] 15374 update docker-compose-prod.yml --- .../clearcut_detection_backend/prod_settings.py | 4 ++++ clearcut_detection_backend/cron_tab_prod | 2 +- clearcut_detection_backend/docker-compose-prod.yml | 14 +++----------- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/clearcut_detection_backend/clearcut_detection_backend/prod_settings.py b/clearcut_detection_backend/clearcut_detection_backend/prod_settings.py index d3d8592..66bb4f5 100644 --- a/clearcut_detection_backend/clearcut_detection_backend/prod_settings.py +++ b/clearcut_detection_backend/clearcut_detection_backend/prod_settings.py @@ -9,3 +9,7 @@ EMAIL_HOST_USER = 'SET_ME_PLEASE_SMTP_USERNAME' EMAIL_HOST_PASSWORD = 'SET_ME_PLEASE_SMTP_PASSWORD' EMAIL_ADMIN_MAIL = ['SET_ME_PLEASE_SMTP_ADMIN_MAIL'] + +# DOWNLOADED_IMAGES_DIR = Path('/data/source_images/') +# MODEL_TIFFS_DIR = Path('/data/model_tiffs') +# MAPBOX_TIFFS_DIR = Path('/data/mapbox_tiffs') diff --git a/clearcut_detection_backend/cron_tab_prod b/clearcut_detection_backend/cron_tab_prod index 86b6272..7305744 100644 --- a/clearcut_detection_backend/cron_tab_prod +++ b/clearcut_detection_backend/cron_tab_prod @@ -1,3 +1,3 @@ # Cron tab prod -0 0,12 * * * python update_new.py +# 0 0,12 * * * python update_new.py diff --git a/clearcut_detection_backend/docker-compose-prod.yml b/clearcut_detection_backend/docker-compose-prod.yml index 0aa114e..88c7d33 100644 --- a/clearcut_detection_backend/docker-compose-prod.yml +++ b/clearcut_detection_backend/docker-compose-prod.yml @@ -2,8 +2,8 @@ db_prod: image: clearcut_detection/postgis_prod expose: - 5432 -# ports: -# - "5433:5432" + ports: + - "5433:5432" env_file: - db.env volumes: @@ -14,8 +14,6 @@ model_prod_2: image: clearcut_detection/model2_prod env_file: - ./model2/model.env -# - ./model2/db.env -# - ./model2/rabbitmq.env volumes: - ./model2/:/model - ./data/:/model/data @@ -35,10 +33,6 @@ react_prod: volumes: - ./frontend:/frontend working_dir: /frontend -# expose: -# - "3001" -# links: -# - "django_prod" command: /bin/bash -c "npm install && npm run-script build" django_prod: @@ -59,15 +53,13 @@ django_prod: # - UPLOAD_TO_MAPBOX=1 # skip uploading to mapbox server volumes: - ./:/code + - /home/clearcut/prod/data_prod:/data working_dir: /code links: - "db_prod" -# - "model_prod_2" - "rabbitmq_prod" expose: - 9000 -# ports: -# - "9001:9000" command: /bin/bash -c "exec invoke runbackend" nginx_prod: From 12005c02a183e880b7d8fb408e0ca850f3cc9822 Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Mon, 17 May 2021 19:20:51 +0300 Subject: [PATCH 04/23] 15374 update prod_settings.py --- .../clearcut_detection_backend/prod_settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clearcut_detection_backend/clearcut_detection_backend/prod_settings.py b/clearcut_detection_backend/clearcut_detection_backend/prod_settings.py index 66bb4f5..9b20d44 100644 --- a/clearcut_detection_backend/clearcut_detection_backend/prod_settings.py +++ b/clearcut_detection_backend/clearcut_detection_backend/prod_settings.py @@ -10,6 +10,6 @@ EMAIL_HOST_PASSWORD = 'SET_ME_PLEASE_SMTP_PASSWORD' EMAIL_ADMIN_MAIL = ['SET_ME_PLEASE_SMTP_ADMIN_MAIL'] -# DOWNLOADED_IMAGES_DIR = Path('/data/source_images/') -# MODEL_TIFFS_DIR = Path('/data/model_tiffs') -# MAPBOX_TIFFS_DIR = Path('/data/mapbox_tiffs') +DOWNLOADED_IMAGES_DIR = Path('/data/source_images/') +MODEL_TIFFS_DIR = Path('/data/model_tiffs') +MAPBOX_TIFFS_DIR = Path('/data/mapbox_tiffs') From c6f22de6d863f1d4cca30887f1df1358c8000dfc Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Mon, 17 May 2021 19:51:21 +0300 Subject: [PATCH 05/23] 15374 increase celery task time limit --- clearcut_detection_backend/model2/celeryconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clearcut_detection_backend/model2/celeryconfig.py b/clearcut_detection_backend/model2/celeryconfig.py index c38290c..d657685 100644 --- a/clearcut_detection_backend/model2/celeryconfig.py +++ b/clearcut_detection_backend/model2/celeryconfig.py @@ -11,7 +11,7 @@ worker_prefetch_multiplier = 1 worker_max_tasks_per_child = 1 task_acks_late = True -task_time_limit = 60 * 15 +task_time_limit = 60 * 20 # database_table_names = { # 'task': 'django_celery_results_taskresult', # 'group': 'myapp_groupmeta', From c833de126b2f83729c9a2a3c938d330a7d94e9a7 Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Mon, 17 May 2021 19:52:49 +0300 Subject: [PATCH 06/23] 15374 update clearcut_detection_backend/cron_tab_prod --- clearcut_detection_backend/cron_tab_prod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clearcut_detection_backend/cron_tab_prod b/clearcut_detection_backend/cron_tab_prod index 7305744..86b6272 100644 --- a/clearcut_detection_backend/cron_tab_prod +++ b/clearcut_detection_backend/cron_tab_prod @@ -1,3 +1,3 @@ # Cron tab prod -# 0 0,12 * * * python update_new.py +0 0,12 * * * python update_new.py From 56d102f7d0e33bdc63daa34507502d118c243cd4 Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Mon, 17 May 2021 20:50:11 +0300 Subject: [PATCH 07/23] 15374 update docker-compose-prod.yml for model --- clearcut_detection_backend/docker-compose-prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clearcut_detection_backend/docker-compose-prod.yml b/clearcut_detection_backend/docker-compose-prod.yml index 88c7d33..986775a 100644 --- a/clearcut_detection_backend/docker-compose-prod.yml +++ b/clearcut_detection_backend/docker-compose-prod.yml @@ -16,7 +16,7 @@ model_prod_2: - ./model2/model.env volumes: - ./model2/:/model - - ./data/:/model/data + - /home/clearcut/prod/data_prod:/data working_dir: /model environment: - DB_HOST=db_prod From 22a1fbcda4ab3998f21f6f7587837ec03b662e7b Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Tue, 18 May 2021 09:38:53 +0300 Subject: [PATCH 08/23] 15374 update docker-compose-prod.yml - add deploy for gpu --- clearcut_detection_backend/docker-compose-prod.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/clearcut_detection_backend/docker-compose-prod.yml b/clearcut_detection_backend/docker-compose-prod.yml index 986775a..6aa64a2 100644 --- a/clearcut_detection_backend/docker-compose-prod.yml +++ b/clearcut_detection_backend/docker-compose-prod.yml @@ -26,6 +26,14 @@ model_prod_2: links: - "rabbitmq_prod" - "db_prod" + deploy: + resources: + reservations: + devices: + - capabilities: ["gpu"] + driver: nvidia + device_ids: ["GPU-b51efca2-01e2-2215-9434-5cc94ec7716e"] + command: /bin/bash -c "./run_predict.sh" # todo react_prod: From 16f0833cfeee03cec3b84396d89d3c2bcbd66ece Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Tue, 18 May 2021 09:59:34 +0300 Subject: [PATCH 09/23] 15374 fix docker-compose-prod.yml --- .../docker-compose-prod.yml | 189 +++++++++--------- 1 file changed, 95 insertions(+), 94 deletions(-) diff --git a/clearcut_detection_backend/docker-compose-prod.yml b/clearcut_detection_backend/docker-compose-prod.yml index 6aa64a2..9c3f42f 100644 --- a/clearcut_detection_backend/docker-compose-prod.yml +++ b/clearcut_detection_backend/docker-compose-prod.yml @@ -1,101 +1,102 @@ -db_prod: - image: clearcut_detection/postgis_prod - expose: - - 5432 - ports: - - "5433:5432" - env_file: - - db.env - volumes: - - /home/clearcut/prod/db-data_prod/data:/var/lib/postgresql/data - - ./:/code +services: + db_prod: + image: clearcut_detection/postgis_prod + expose: + - 5432 + ports: + - "5433:5432" + env_file: + - db.env + volumes: + - /home/clearcut/prod/db-data_prod/data:/var/lib/postgresql/data + - ./:/code -model_prod_2: - image: clearcut_detection/model2_prod - env_file: - - ./model2/model.env - volumes: - - ./model2/:/model - - /home/clearcut/prod/data_prod:/data - working_dir: /model - environment: - - DB_HOST=db_prod - - RABBITMQ_HOST=rabbitmq_prod - - CUDA_VISIBLE_DEVICES=0 - - PREDICT=1 - links: - - "rabbitmq_prod" - - "db_prod" - deploy: - resources: - reservations: - devices: - - capabilities: ["gpu"] - driver: nvidia - device_ids: ["GPU-b51efca2-01e2-2215-9434-5cc94ec7716e"] + model_prod_2: + image: clearcut_detection/model2_prod + env_file: + - ./model2/model.env + volumes: + - ./model2/:/model + - /home/clearcut/prod/data_prod:/data + working_dir: /model + environment: + - DB_HOST=db_prod + - RABBITMQ_HOST=rabbitmq_prod + - CUDA_VISIBLE_DEVICES=0 + - PREDICT=1 + links: + - "rabbitmq_prod" + - "db_prod" + deploy: + resources: + reservations: + devices: + - capabilities: ["gpu"] + driver: "nvidia" + device_ids: ["GPU-b51efca2-01e2-2215-9434-5cc94ec7716e"] - command: /bin/bash -c "./run_predict.sh" # todo + command: /bin/bash -c "./run_predict.sh" # todo -react_prod: - image: node:10.15.0 - volumes: - - ./frontend:/frontend - working_dir: /frontend - command: /bin/bash -c "npm install && npm run-script build" + react_prod: + image: node:10.15.0 + volumes: + - ./frontend:/frontend + working_dir: /frontend + command: /bin/bash -c "npm install && npm run-script build" -django_prod: - image: clearcut_detection/backend_prod - env_file: - - db.env - - django.env - - rabbitmq.env - environment: - - DJANGO_SETTINGS_MODULE=clearcut_detection_backend.prod_settings - - DB_HOST=db_prod - - RABBITMQ_HOST=rabbitmq_prod -# - SENTINEL_DOWNLOAD=1 # prevent requesting to sentinel -# - DOWNLOAD_IMG=0 # skip downloading images if SENTINEL_DOWNLOAD=true -# - PREPARE_TIF=1 # skip preprocessing images -# - MAKE_PREDICT=0 # skip predicting of clearcuts -# - CONVERT_TO_TIFF=0 # skip converting to tiff for uploading -# - UPLOAD_TO_MAPBOX=1 # skip uploading to mapbox server - volumes: - - ./:/code - - /home/clearcut/prod/data_prod:/data - working_dir: /code - links: - - "db_prod" - - "rabbitmq_prod" - expose: - - 9000 - command: /bin/bash -c "exec invoke runbackend" + django_prod: + image: clearcut_detection/backend_prod + env_file: + - db.env + - django.env + - rabbitmq.env + environment: + - DJANGO_SETTINGS_MODULE=clearcut_detection_backend.prod_settings + - DB_HOST=db_prod + - RABBITMQ_HOST=rabbitmq_prod + # - SENTINEL_DOWNLOAD=1 # prevent requesting to sentinel + # - DOWNLOAD_IMG=0 # skip downloading images if SENTINEL_DOWNLOAD=true + # - PREPARE_TIF=1 # skip preprocessing images + # - MAKE_PREDICT=0 # skip predicting of clearcuts + # - CONVERT_TO_TIFF=0 # skip converting to tiff for uploading + # - UPLOAD_TO_MAPBOX=1 # skip uploading to mapbox server + volumes: + - ./:/code + - /home/clearcut/prod/data_prod:/data + working_dir: /code + links: + - "db_prod" + - "rabbitmq_prod" + expose: + - 9000 + command: /bin/bash -c "exec invoke runbackend" -nginx_prod: - image: nginx:1.15.8 - links: - - "django_prod" - expose: - - "8080" - ports: - - "8080:8080" - environment: - - VIRTUAL_HOST=clearcut.quantumobile.com - - VIRTUAL_PORT=8080 - - LETSENCRYPT_HOST=clearcut.quantumobile.com - - LETSENCRYPT_EMAIL=o.tsiapa@quantumobile.com - volumes: - - ./nginx/nginx_prod.conf:/etc/nginx/conf.d/default.conf - - ./frontend/build:/frontend/build + nginx_prod: + image: nginx:1.15.8 + links: + - "django_prod" + expose: + - "8080" + ports: + - "8080:8080" + environment: + - VIRTUAL_HOST=clearcut.quantumobile.com + - VIRTUAL_PORT=8080 + - LETSENCRYPT_HOST=clearcut.quantumobile.com + - LETSENCRYPT_EMAIL=o.tsiapa@quantumobile.com + volumes: + - ./nginx/nginx_prod.conf:/etc/nginx/conf.d/default.conf + - ./frontend/build:/frontend/build -rabbitmq_prod: - hostname: "rabbitmq_prod" - labels: - NAME: "rabbitmq_prod" - image: "rabbitmq" - env_file: - - rabbitmq.env - expose: - - 15672 - - 5672 - restart: always + rabbitmq_prod: + hostname: "rabbitmq_prod" + labels: + NAME: "rabbitmq_prod" + image: "rabbitmq" + env_file: + - rabbitmq.env + expose: + - 15672 + - 5672 + restart: always From e2eaefd9e8d05531f29309a55d294f44d395c299 Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Tue, 18 May 2021 16:41:06 +0300 Subject: [PATCH 10/23] 15374 fix path to lancover --- clearcut_detection_backend/model2/celeryconfig.py | 2 +- clearcut_detection_backend/model2/run_predict.sh | 2 +- clearcut_detection_backend/model2/utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clearcut_detection_backend/model2/celeryconfig.py b/clearcut_detection_backend/model2/celeryconfig.py index d657685..2d5a5c6 100644 --- a/clearcut_detection_backend/model2/celeryconfig.py +++ b/clearcut_detection_backend/model2/celeryconfig.py @@ -11,7 +11,7 @@ worker_prefetch_multiplier = 1 worker_max_tasks_per_child = 1 task_acks_late = True -task_time_limit = 60 * 20 +task_time_limit = 60 * 60 # database_table_names = { # 'task': 'django_celery_results_taskresult', # 'group': 'myapp_groupmeta', diff --git a/clearcut_detection_backend/model2/run_predict.sh b/clearcut_detection_backend/model2/run_predict.sh index 4e9de35..721f4f5 100755 --- a/clearcut_detection_backend/model2/run_predict.sh +++ b/clearcut_detection_backend/model2/run_predict.sh @@ -1,3 +1,3 @@ #!/bin/bash -celery -A tasks worker -Q model_predict_queue --concurrency=2 --loglevel=DEBUG +celery -A tasks worker -Q model_predict_queue --concurrency=8 --loglevel=DEBUG diff --git a/clearcut_detection_backend/model2/utils.py b/clearcut_detection_backend/model2/utils.py index 0c83dcf..404dc22 100644 --- a/clearcut_detection_backend/model2/utils.py +++ b/clearcut_detection_backend/model2/utils.py @@ -15,7 +15,7 @@ SCOPES = ['https://www.googleapis.com/auth/drive.file'] -LANDCOVER_POLYGONS_PATH = 'data/landcover' +LANDCOVER_POLYGONS_PATH = '/data/landcover' SENTINEL_TILES = f"{LANDCOVER_POLYGONS_PATH}/S2A_OPER_GIP_TILPAR_MPC__20151209T095117_V20150622T000000_21000101T000000_B00.kml" LANDCOVER_GEOJSON = f'{LANDCOVER_POLYGONS_PATH}/landcover_polygons.geojson' From af4859caf5c96c1e4923536265f9294f7c69069d Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Tue, 18 May 2021 17:59:47 +0300 Subject: [PATCH 11/23] 15374 fix clearcut_detection_backend/model2/utils.py path to landcover file --- clearcut_detection_backend/model2/utils.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/clearcut_detection_backend/model2/utils.py b/clearcut_detection_backend/model2/utils.py index 404dc22..35b4c6e 100644 --- a/clearcut_detection_backend/model2/utils.py +++ b/clearcut_detection_backend/model2/utils.py @@ -4,7 +4,7 @@ import io import os.path import logging - +from pathlib import Path from googleapiclient.discovery import build from googleapiclient.http import MediaIoBaseDownload from google.oauth2 import service_account @@ -15,9 +15,10 @@ SCOPES = ['https://www.googleapis.com/auth/drive.file'] -LANDCOVER_POLYGONS_PATH = '/data/landcover' -SENTINEL_TILES = f"{LANDCOVER_POLYGONS_PATH}/S2A_OPER_GIP_TILPAR_MPC__20151209T095117_V20150622T000000_21000101T000000_B00.kml" -LANDCOVER_GEOJSON = f'{LANDCOVER_POLYGONS_PATH}/landcover_polygons.geojson' +LANDCOVER_POLYGONS_PATH = Path('/data/landcover') +SENTINEL_TILES = LANDCOVER_POLYGONS_PATH / \ + 'S2A_OPER_GIP_TILPAR_MPC__20151209T095117_V20150622T000000_21000101T000000_B00.kml' +LANDCOVER_GEOJSON = LANDCOVER_POLYGONS_PATH / 'landcover_polygons.geojson' class LandcoverPolygons: """ @@ -37,8 +38,8 @@ def __init__(self, tile, crs): gpd.io.file.fiona.drvsupport.supported_drivers['KML'] = 'rw' def get_polygon(self): - polygon_path = os.path.join(LANDCOVER_POLYGONS_PATH, f"{self.tile}.geojson") - if os.path.exists(polygon_path): + polygon_path = LANDCOVER_POLYGONS_PATH / f'{self.tile}.geojson' + if polygon_path.is_file(): logging.info(f'{self.tile} forest polygons file exists.') polygons = gpd.read_file(polygon_path) else: @@ -54,20 +55,19 @@ def get_polygon(self): def create_polygon(self): polygons = [] - if os.path.exists(SENTINEL_TILES): + if SENTINEL_TILES.is_file(): sentinel_tiles = gpd.read_file(SENTINEL_TILES, driver='KML') sentinel_tiles = sentinel_tiles[sentinel_tiles['Name'] == self.tile] bounding_polygon = sentinel_tiles['geometry'].values[0] polygons = gpd.read_file(LANDCOVER_GEOJSON) polygons = polygons[polygons['geometry'].intersects(bounding_polygon)] - - polygon_path = os.path.join(LANDCOVER_POLYGONS_PATH, f"{self.tile}.geojson") + polygon_path = LANDCOVER_POLYGONS_PATH / f'{self.tile}.geojson' polygons.to_file(polygon_path, driver='GeoJSON') return polygons def weights_exists_or_download(path, file_id): - if not os.path.exists(path): + if not Path(path).exists(): creds_file = os.environ.get('CREDENTIAL_FILE') creds = service_account.Credentials.from_service_account_file(creds_file, scopes=SCOPES) From db3ca7452e7f8e30a0a25ab825c7b36fb60a2a68 Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Tue, 18 May 2021 18:57:13 +0300 Subject: [PATCH 12/23] 15374 add logs --- clearcut_detection_backend/model2/predict_raster.py | 2 -- clearcut_detection_backend/model2/utils.py | 5 +++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/clearcut_detection_backend/model2/predict_raster.py b/clearcut_detection_backend/model2/predict_raster.py index 0ac3172..246a344 100644 --- a/clearcut_detection_backend/model2/predict_raster.py +++ b/clearcut_detection_backend/model2/predict_raster.py @@ -264,8 +264,6 @@ def get_intersected_polygons(polygons, masks, mask_column_name): polygons[mask_column_name] = masked_values return polygons - # tile = os.path.basename(img_path) - landcover = LandcoverPolygons(tile, src_crs) forest_polygons = landcover.get_polygon() diff --git a/clearcut_detection_backend/model2/utils.py b/clearcut_detection_backend/model2/utils.py index 35b4c6e..2b9f73e 100644 --- a/clearcut_detection_backend/model2/utils.py +++ b/clearcut_detection_backend/model2/utils.py @@ -39,6 +39,7 @@ def __init__(self, tile, crs): def get_polygon(self): polygon_path = LANDCOVER_POLYGONS_PATH / f'{self.tile}.geojson' + logging.info(f'LANDCOVER_POLYGONS_PATH: {polygon_path}') if polygon_path.is_file(): logging.info(f'{self.tile} forest polygons file exists.') polygons = gpd.read_file(polygon_path) @@ -62,7 +63,11 @@ def create_polygon(self): polygons = gpd.read_file(LANDCOVER_GEOJSON) polygons = polygons[polygons['geometry'].intersects(bounding_polygon)] polygon_path = LANDCOVER_POLYGONS_PATH / f'{self.tile}.geojson' + logging.info(f'forests_polygons_file_path: {polygon_path}') polygons.to_file(polygon_path, driver='GeoJSON') + else: + logging.error(f'{SENTINEL_TILES} doth not exists') + raise FileNotFoundError return polygons From 08cc20a761edc1fe619cf6327f1cedebd49aca5b Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Wed, 19 May 2021 08:08:21 +0300 Subject: [PATCH 13/23] 15374 fix path to results in model --- .../model2/predict_raster.py | 15 +++++++-------- .../model2/run_predict_tasks/run_predict.py | 6 ++---- clearcut_detection_backend/model2/utils.py | 2 ++ 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/clearcut_detection_backend/model2/predict_raster.py b/clearcut_detection_backend/model2/predict_raster.py index 246a344..ac035a3 100644 --- a/clearcut_detection_backend/model2/predict_raster.py +++ b/clearcut_detection_backend/model2/predict_raster.py @@ -3,7 +3,6 @@ import cv2 import torch import logging -import imageio import rasterio import argparse import geopandas @@ -89,13 +88,13 @@ def predict_raster(img_current, img_previous, channels, network, model_weights_p left_column = i * input_size right_column = (i + 1) * input_size - corners=[ - source_current.xy(bottom_row, left_column), - source_current.xy(bottom_row, right_column), - source_current.xy(upper_row, right_column), - source_current.xy(upper_row, left_column), - source_current.xy(bottom_row, left_column) - ] + corners = [ + source_current.xy(bottom_row, left_column), + source_current.xy(bottom_row, right_column), + source_current.xy(upper_row, right_column), + source_current.xy(upper_row, left_column), + source_current.xy(bottom_row, left_column) + ] window = Window(bottom_row, left_column, input_size, input_size) image_current = reshape_as_image(source_current.read(window=window)) diff --git a/clearcut_detection_backend/model2/run_predict_tasks/run_predict.py b/clearcut_detection_backend/model2/run_predict_tasks/run_predict.py index e393189..b570838 100644 --- a/clearcut_detection_backend/model2/run_predict_tasks/run_predict.py +++ b/clearcut_detection_backend/model2/run_predict_tasks/run_predict.py @@ -21,13 +21,12 @@ def run_predict(session, task_id): RpT.update_task_by_id(session, task_id, params) image_path = Path(params['path_img_0']) list_tif_path = list(image_path.parts) - filename = list_tif_path[2] + filename = list_tif_path[3] predicted_filename = f'predicted_{filename}_{params["image_date_0"]}_{params["image_date_1"]}.geojson' - list_tif_path = list_tif_path[:1] + list_tif_path = list_tif_path[:2] list_tif_path.append('predicted') list_tif_path.append(filename) result_directory_path = Path(*list_tif_path) - # print(result_directory_path) result_directory_path.mkdir(parents=True, exist_ok=True) if not (result_directory_path / predicted_filename).is_file(): @@ -63,7 +62,6 @@ def run_predict(session, task_id): params['date_finished'] = str(datetime.now()) RpT.update_task_by_id(session, task_id, params) - # RpT.update_tileinformation(session, params['tile_id']) return params['result'] else: params['result'] = str(result_directory_path / predicted_filename) diff --git a/clearcut_detection_backend/model2/utils.py b/clearcut_detection_backend/model2/utils.py index 2b9f73e..9b51923 100644 --- a/clearcut_detection_backend/model2/utils.py +++ b/clearcut_detection_backend/model2/utils.py @@ -57,8 +57,10 @@ def get_polygon(self): def create_polygon(self): polygons = [] if SENTINEL_TILES.is_file(): + logging.info(f'read forests_polygons_file: {SENTINEL_TILES}, for tile {self.tile}') sentinel_tiles = gpd.read_file(SENTINEL_TILES, driver='KML') sentinel_tiles = sentinel_tiles[sentinel_tiles['Name'] == self.tile] + logging.info(f'sentinel_tiles for {self.tile}: {sentinel_tiles}') bounding_polygon = sentinel_tiles['geometry'].values[0] polygons = gpd.read_file(LANDCOVER_GEOJSON) polygons = polygons[polygons['geometry'].intersects(bounding_polygon)] From b2ce131ea21a22c873115e13c598057a21b2cb47 Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Wed, 19 May 2021 08:16:08 +0300 Subject: [PATCH 14/23] 15374 disable cron --- clearcut_detection_backend/cron_tab_prod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clearcut_detection_backend/cron_tab_prod b/clearcut_detection_backend/cron_tab_prod index 86b6272..9aed533 100644 --- a/clearcut_detection_backend/cron_tab_prod +++ b/clearcut_detection_backend/cron_tab_prod @@ -1,3 +1,3 @@ # Cron tab prod -0 0,12 * * * python update_new.py +#0 0,12 * * * python update_new.py From c86337ab5f9ed76b5e627311c1f15f6a7c31f99e Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Wed, 19 May 2021 09:21:19 +0300 Subject: [PATCH 15/23] 15374 add path vars to config file --- .../model2/config/__init__.py | 14 ++++++++++++++ .../model2/predict_raster.py | 12 +++--------- .../model2/run_predict_tasks/run_predict.py | 7 ++----- clearcut_detection_backend/model2/utils.py | 8 +------- 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/clearcut_detection_backend/model2/config/__init__.py b/clearcut_detection_backend/model2/config/__init__.py index 477ca69..e2e5fa3 100644 --- a/clearcut_detection_backend/model2/config/__init__.py +++ b/clearcut_detection_backend/model2/config/__init__.py @@ -1,5 +1,6 @@ import os import yaml +from pathlib import Path with open('/model/predict_config.yml', 'r') as config: cfg = yaml.load(config, Loader=yaml.SafeLoader) @@ -18,3 +19,16 @@ RABBITMQ_PASS = os.environ.get('RABBITMQ_DEFAULT_PASS', 'guest') RABBITMQ_HOST = os.environ.get('RABBITMQ_HOST', 'rabbitmq_prod') RABBITMQ_PORT_NUMBER = os.environ.get('RABBITMQ_NODE_PORT_NUMBER', 5672) +CUDA_VISIBLE_DEVICES = os.environ.get('CUDA_VISIBLE_DEVICES', '0') + +SCOPES = ['https://www.googleapis.com/auth/drive.file'] + +LANDCOVER_POLYGONS_PATH = Path('/data/landcover') +PREDICTED_PATH = Path('/data/predicted') +LANDCOVER_FILENAME = 'S2A_OPER_GIP_TILPAR_MPC__20151209T095117_V20150622T000000_21000101T000000_B00.kml' +SENTINEL_TILES = LANDCOVER_POLYGONS_PATH / LANDCOVER_FILENAME +LANDCOVER_GEOJSON = LANDCOVER_POLYGONS_PATH / 'landcover_polygons.geojson' + +CLOUDS_PROBABILITY_THRESHOLD = 15 +NEAREST_POLYGONS_NUMBER = 10 +DATES_FOR_TILE = 2 diff --git a/clearcut_detection_backend/model2/predict_raster.py b/clearcut_detection_backend/model2/predict_raster.py index ac035a3..fad5dcc 100644 --- a/clearcut_detection_backend/model2/predict_raster.py +++ b/clearcut_detection_backend/model2/predict_raster.py @@ -8,29 +8,23 @@ import geopandas import numpy as np import segmentation_models_pytorch as smp - from catalyst.dl.utils import UtilsFactory -from geopandas import GeoSeries from scipy import spatial from shapely.geometry import Polygon -from shapely.ops import unary_union from torchvision import transforms from torch import nn from tqdm import tqdm from rasterio.windows import Window from rasterio.plot import reshape_as_image from skimage.transform import match_histograms - from utils import LandcoverPolygons +import warnings +from config import CLOUDS_PROBABILITY_THRESHOLD, NEAREST_POLYGONS_NUMBER, DATES_FOR_TILE -CLOUDS_PROBABILITY_THRESHOLD = 15 -NEAREST_POLYGONS_NUMBER = 10 -DATES_FOR_TILE = 2 -import warnings warnings.filterwarnings('ignore') -os.environ.get('CUDA_VISIBLE_DEVICES', '0') + logging.basicConfig(format='%(asctime)s %(message)s') diff --git a/clearcut_detection_backend/model2/run_predict_tasks/run_predict.py b/clearcut_detection_backend/model2/run_predict_tasks/run_predict.py index b570838..2ad71fa 100644 --- a/clearcut_detection_backend/model2/run_predict_tasks/run_predict.py +++ b/clearcut_detection_backend/model2/run_predict_tasks/run_predict.py @@ -1,7 +1,7 @@ import os from distutils.util import strtobool from pathlib import Path -from config import models, threshold, input_size +from config import models, threshold, input_size, PREDICTED_PATH from utils import weights_exists_or_download from predict_raster import predict_raster, polygonize, postprocessing, save_polygons from services.run_predict_tasks_service import RunPredictTasks as RpT @@ -23,10 +23,7 @@ def run_predict(session, task_id): list_tif_path = list(image_path.parts) filename = list_tif_path[3] predicted_filename = f'predicted_{filename}_{params["image_date_0"]}_{params["image_date_1"]}.geojson' - list_tif_path = list_tif_path[:2] - list_tif_path.append('predicted') - list_tif_path.append(filename) - result_directory_path = Path(*list_tif_path) + result_directory_path = PREDICTED_PATH / filename result_directory_path.mkdir(parents=True, exist_ok=True) if not (result_directory_path / predicted_filename).is_file(): diff --git a/clearcut_detection_backend/model2/utils.py b/clearcut_detection_backend/model2/utils.py index 9b51923..cb174b1 100644 --- a/clearcut_detection_backend/model2/utils.py +++ b/clearcut_detection_backend/model2/utils.py @@ -8,17 +8,11 @@ from googleapiclient.discovery import build from googleapiclient.http import MediaIoBaseDownload from google.oauth2 import service_account - import geopandas as gpd +from config import SCOPES, LANDCOVER_POLYGONS_PATH, SENTINEL_TILES, LANDCOVER_GEOJSON logging.basicConfig(format='%(asctime)s %(message)s') -SCOPES = ['https://www.googleapis.com/auth/drive.file'] - -LANDCOVER_POLYGONS_PATH = Path('/data/landcover') -SENTINEL_TILES = LANDCOVER_POLYGONS_PATH / \ - 'S2A_OPER_GIP_TILPAR_MPC__20151209T095117_V20150622T000000_21000101T000000_B00.kml' -LANDCOVER_GEOJSON = LANDCOVER_POLYGONS_PATH / 'landcover_polygons.geojson' class LandcoverPolygons: """ From 1bccc356eae2701154d56f5062ba2423595fe7c2 Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Wed, 19 May 2021 12:22:01 +0300 Subject: [PATCH 16/23] 15374 fix logging in model2 --- clearcut_detection_backend/model2/predict_raster.py | 8 ++++---- clearcut_detection_backend/model2/run_predict.sh | 2 +- .../model2/run_predict_tasks/run_predict.py | 4 +++- clearcut_detection_backend/model2/utils.py | 6 +++--- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/clearcut_detection_backend/model2/predict_raster.py b/clearcut_detection_backend/model2/predict_raster.py index fad5dcc..1357f66 100644 --- a/clearcut_detection_backend/model2/predict_raster.py +++ b/clearcut_detection_backend/model2/predict_raster.py @@ -63,10 +63,10 @@ def mask_postprocess(mask): def predict_raster(img_current, img_previous, channels, network, model_weights_path, input_size=56, neighbours=3): - print(f'network:{network}') - print(f'model_weights_path: {model_weights_path}') - print(f'channels: {channels}') - print(f'neighbours: {neighbours}') + logging.info(f'network:{network}') + logging.info(f'model_weights_path: {model_weights_path}') + logging.info(f'channels: {channels}') + logging.info(f'neighbours: {neighbours}') model, device = load_model(network, model_weights_path, channels, neighbours) with rasterio.open(img_current) as source_current, \ diff --git a/clearcut_detection_backend/model2/run_predict.sh b/clearcut_detection_backend/model2/run_predict.sh index 721f4f5..ee62097 100755 --- a/clearcut_detection_backend/model2/run_predict.sh +++ b/clearcut_detection_backend/model2/run_predict.sh @@ -1,3 +1,3 @@ #!/bin/bash -celery -A tasks worker -Q model_predict_queue --concurrency=8 --loglevel=DEBUG +celery -A tasks worker -Q model_predict_queue --concurrency=8 --loglevel=WARNING --task-events --purge diff --git a/clearcut_detection_backend/model2/run_predict_tasks/run_predict.py b/clearcut_detection_backend/model2/run_predict_tasks/run_predict.py index 2ad71fa..80b9469 100644 --- a/clearcut_detection_backend/model2/run_predict_tasks/run_predict.py +++ b/clearcut_detection_backend/model2/run_predict_tasks/run_predict.py @@ -1,4 +1,5 @@ import os +import logging from distutils.util import strtobool from pathlib import Path from config import models, threshold, input_size, PREDICTED_PATH @@ -9,6 +10,7 @@ predict = strtobool(os.environ.get('PREDICT', 'true')) +logging.basicConfig(format='%(asctime)s %(message)s') def run_predict(session, task_id): """ @@ -64,5 +66,5 @@ def run_predict(session, task_id): params['result'] = str(result_directory_path / predicted_filename) params['date_finished'] = str(datetime.now()) RpT.update_task_by_id(session, task_id, params) - print(f'file {str(result_directory_path / predicted_filename)} exist. Skip') + logging.info(f'file {str(result_directory_path / predicted_filename)} exist. Skip') return str(result_directory_path / predicted_filename) diff --git a/clearcut_detection_backend/model2/utils.py b/clearcut_detection_backend/model2/utils.py index cb174b1..3f3aa6b 100644 --- a/clearcut_detection_backend/model2/utils.py +++ b/clearcut_detection_backend/model2/utils.py @@ -35,17 +35,17 @@ def get_polygon(self): polygon_path = LANDCOVER_POLYGONS_PATH / f'{self.tile}.geojson' logging.info(f'LANDCOVER_POLYGONS_PATH: {polygon_path}') if polygon_path.is_file(): - logging.info(f'{self.tile} forest polygons file exists.') + logging.info(f'{self.tile} forests polygons file exists.') polygons = gpd.read_file(polygon_path) else: - logging.info(f'{self.tile} forest polygons file does not exist. Creating polygons...') + logging.info(f'{self.tile} forests polygons file does not exist. Creating polygons...') polygons = self.create_polygon() if len(polygons) > 0: polygons = polygons.to_crs(self.crs) polygons = list(polygons['geometry']) else: - logging.info('No forest polygons.') + logging.info('No forests polygons.') return polygons def create_polygon(self): From b5041b868793a6e4af39c7d5b0aa4a264090f95c Mon Sep 17 00:00:00 2001 From: Ivan Lukinov Date: Mon, 11 Oct 2021 12:36:40 +0300 Subject: [PATCH 17/23] Update django.Dockerfile --- clearcut_detection_backend/django.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clearcut_detection_backend/django.Dockerfile b/clearcut_detection_backend/django.Dockerfile index f34a3b3..b390734 100644 --- a/clearcut_detection_backend/django.Dockerfile +++ b/clearcut_detection_backend/django.Dockerfile @@ -19,7 +19,7 @@ RUN apt-get update -y && apt-get install -y \ build-essential \ python-dev \ python3-dev \ - python-numpy \ +# python-numpy \ python3-numpy \ libspatialite-dev \ sqlite3 \ From 2f23945048d22f350d1c6000dfb66208b318f37b Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Fri, 22 Oct 2021 13:40:01 +0300 Subject: [PATCH 18/23] fix clearcut_detection_backend/django.Dockerfile --- clearcut_detection_backend/django.Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/clearcut_detection_backend/django.Dockerfile b/clearcut_detection_backend/django.Dockerfile index f34a3b3..482a8f9 100644 --- a/clearcut_detection_backend/django.Dockerfile +++ b/clearcut_detection_backend/django.Dockerfile @@ -19,8 +19,6 @@ RUN apt-get update -y && apt-get install -y \ build-essential \ python-dev \ python3-dev \ - python-numpy \ - python3-numpy \ libspatialite-dev \ sqlite3 \ libpq-dev \ @@ -36,7 +34,7 @@ RUN apt-get update -y && apt-get install -y \ bash-completion \ cmake -RUN /bin/bash -c "pip install numpy==1.16.4" +RUN /bin/bash -c "pip3 install numpy==1.16.4" # Compile and install OpenJPEG RUN cd src && tar -xvf openjpeg-${OPENJPEG_VERSION}.tar.gz && cd openjpeg-${OPENJPEG_VERSION}/ \ From d2f32c444685a940b0eeb2cb194c64e7a2e9d164 Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Fri, 22 Oct 2021 13:41:19 +0300 Subject: [PATCH 19/23] fix clearcut_detection_backend/django.Dockerfile --- clearcut_detection_backend/django.Dockerfile | 4 ++-- clearcut_detection_backend/requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clearcut_detection_backend/django.Dockerfile b/clearcut_detection_backend/django.Dockerfile index 482a8f9..9fcb1b3 100644 --- a/clearcut_detection_backend/django.Dockerfile +++ b/clearcut_detection_backend/django.Dockerfile @@ -45,7 +45,7 @@ RUN cd src && tar -xvf openjpeg-${OPENJPEG_VERSION}.tar.gz && cd openjpeg-${OPEN # Compile and install GDAL RUN cd src && tar -xvf gdal-${GDAL_VERSION}.tar.gz && cd gdal-${GDAL_VERSION} \ - && ./configure --with-python --with-spatialite --with-pg --with-curl --with-openjpeg=$ROOTDIR \ + && ./configure --with-python=python3 --with-proj=/usr/local --with-spatialite --with-pg --with-curl --with-openjpeg=$ROOTDIR \ && make && make install && ldconfig \ && apt-get update -y \ && apt-get remove -y --purge build-essential wget \ @@ -62,7 +62,7 @@ RUN apt-get install -y libgdal-dev RUN apt-get update -y \ && apt-get upgrade -y \ && apt-get install -y postgresql-client locales \ - && apt-get install -y gdal-bin python-gdal python3-gdal \ + && apt-get install -y gdal-bin python3-gdal \ && apt-get update && apt-get install -y gettext libgettextpo-dev \ # Cleanup && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ diff --git a/clearcut_detection_backend/requirements.txt b/clearcut_detection_backend/requirements.txt index 91df2ee..28a0a2c 100644 --- a/clearcut_detection_backend/requirements.txt +++ b/clearcut_detection_backend/requirements.txt @@ -31,4 +31,4 @@ sentinelhub==2.5.3 Shapely==1.6.4.post2 six==1.12.0 tqdm==4.29.1 -uWSGI==2.0.15 +uWSGI==2.0.18 From 151238e7bb17fd48f1798af2ab1cf43603b42c91 Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Tue, 26 Oct 2021 18:13:34 +0300 Subject: [PATCH 20/23] fix_numpy --- clearcut_detection_backend/base.Dockerfile | 71 +++++++++++++++++++++ clearcut_detection_backend/requirements.txt | 22 +++---- 2 files changed, 82 insertions(+), 11 deletions(-) create mode 100644 clearcut_detection_backend/base.Dockerfile diff --git a/clearcut_detection_backend/base.Dockerfile b/clearcut_detection_backend/base.Dockerfile new file mode 100644 index 0000000..39b69d4 --- /dev/null +++ b/clearcut_detection_backend/base.Dockerfile @@ -0,0 +1,71 @@ +FROM python:3.7.7 + +ENV PYTHONUNBUFFERED 1 +ENV ROOTDIR /usr/local/ +ARG GDAL_VERSION=3.1.4 + +WORKDIR ${ROOTDIR}/ + +RUN apt-get update && apt-get upgrade -y\ + && apt-get install software-properties-common -y\ + && add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -sc)-pgdg main"\ + && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -\ + && apt-get update\ + && apt-get install postgresql-client-13 -y\ + && apt-get install sqlite3 -y + +# See https://docs.djangoproject.com/en/3.1/ref/contrib/gis/install/geolibs/ + +RUN wget https://download.osgeo.org/geos/geos-3.8.1.tar.bz2 \ + && tar -xjf geos-3.8.1.tar.bz2 \ + && cd geos-3.8.1 \ + && ./configure \ + && make \ + && make install \ + && cd .. \ + && rm -rf geos-3.8.1 geos-3.8.1.tar.bz2 + +RUN wget https://download.osgeo.org/proj/proj-6.3.2.tar.gz \ + && wget https://download.osgeo.org/proj/proj-datumgrid-1.8.tar.gz \ + && tar -xzf proj-6.3.2.tar.gz \ + && mkdir proj-6.3.2/nad && cd proj-6.3.2/nad \ + && tar -xzf ../../proj-datumgrid-1.8.tar.gz \ + && cd .. \ + && ./configure \ + && make \ + && make install \ + && cd .. \ + && rm -rf proj-6.3.2 proj-6.3.2.tar.gz proj-datumgrid-1.8.tar.gz + +RUN apt-get -y install python3-pip +RUN pip3 install numpy==1.17.3 + +RUN wget https://download.osgeo.org/gdal/${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz \ + && tar -xzf gdal-${GDAL_VERSION}.tar.gz \ + && cd gdal-${GDAL_VERSION} \ + && ./configure --with-python=python3.7 --with-proj=${ROOTDIR} \ + && make \ + && make install \ + && cd swig/python \ + && python3 setup.py build \ + && python3 setup.py install \ + && cd ${ROOTDIR} \ + && rm -rf gdal-${GDAL_VERSION} gdal-${GDAL_VERSION}.tar.gz + +RUN git clone https://github.com/mapbox/tippecanoe.git \ + && cd tippecanoe \ + && git checkout tags/1.36.0 \ + && make -j \ + && make install \ + && cd .. \ + && rm -rf tippecanoe + +RUN ldconfig + +RUN mkdir /code + +WORKDIR /code + +COPY requirements.txt /code/ + +RUN pip3 install -r requirements.txt \ No newline at end of file diff --git a/clearcut_detection_backend/requirements.txt b/clearcut_detection_backend/requirements.txt index 28a0a2c..e4cdd7a 100644 --- a/clearcut_detection_backend/requirements.txt +++ b/clearcut_detection_backend/requirements.txt @@ -15,20 +15,20 @@ google-cloud-core==1.0.3 google-cloud-storage==1.23.0 google-resumable-media==0.5.0 googleapis-common-protos==1.6.0 -gsutil==4.47 -imageio==2.5.0 +#gsutil==4.47 +#imageio==2.5.0 invoke==1.2.0 -numpy==1.16.4 +#numpy==1.16.4 oauth2client==4.1.3 -pandas==0.24.2 -Pillow==6.1.0 +#pandas==0.24.2 +#Pillow==6.1.0 psycopg2-binary==2.8.3 -pyproj==2.2.1 -PyYAML==5.1.1 +#pyproj==2.2.1 +#PyYAML==5.1.1 rasterio==1.1.2 -rio-mbtiles==1.4.2 +#rio-mbtiles==1.4.2 sentinelhub==2.5.3 -Shapely==1.6.4.post2 -six==1.12.0 -tqdm==4.29.1 +#Shapely==1.6.4.post2 +#six==1.12.0 +#tqdm==4.29.1 uWSGI==2.0.18 From 383845d87117e8b071441984be7b6fe2b194dd77 Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Tue, 26 Oct 2021 18:16:40 +0300 Subject: [PATCH 21/23] fix_numpy --- clearcut_detection_backend/django.Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/clearcut_detection_backend/django.Dockerfile b/clearcut_detection_backend/django.Dockerfile index 9fcb1b3..f34a3b3 100644 --- a/clearcut_detection_backend/django.Dockerfile +++ b/clearcut_detection_backend/django.Dockerfile @@ -19,6 +19,8 @@ RUN apt-get update -y && apt-get install -y \ build-essential \ python-dev \ python3-dev \ + python-numpy \ + python3-numpy \ libspatialite-dev \ sqlite3 \ libpq-dev \ @@ -34,7 +36,7 @@ RUN apt-get update -y && apt-get install -y \ bash-completion \ cmake -RUN /bin/bash -c "pip3 install numpy==1.16.4" +RUN /bin/bash -c "pip install numpy==1.16.4" # Compile and install OpenJPEG RUN cd src && tar -xvf openjpeg-${OPENJPEG_VERSION}.tar.gz && cd openjpeg-${OPENJPEG_VERSION}/ \ @@ -45,7 +47,7 @@ RUN cd src && tar -xvf openjpeg-${OPENJPEG_VERSION}.tar.gz && cd openjpeg-${OPEN # Compile and install GDAL RUN cd src && tar -xvf gdal-${GDAL_VERSION}.tar.gz && cd gdal-${GDAL_VERSION} \ - && ./configure --with-python=python3 --with-proj=/usr/local --with-spatialite --with-pg --with-curl --with-openjpeg=$ROOTDIR \ + && ./configure --with-python --with-spatialite --with-pg --with-curl --with-openjpeg=$ROOTDIR \ && make && make install && ldconfig \ && apt-get update -y \ && apt-get remove -y --purge build-essential wget \ @@ -62,7 +64,7 @@ RUN apt-get install -y libgdal-dev RUN apt-get update -y \ && apt-get upgrade -y \ && apt-get install -y postgresql-client locales \ - && apt-get install -y gdal-bin python3-gdal \ + && apt-get install -y gdal-bin python-gdal python3-gdal \ && apt-get update && apt-get install -y gettext libgettextpo-dev \ # Cleanup && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ From 3ed13a7098b868836992ae214af886ea3dfca45a Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Thu, 28 Oct 2021 12:28:16 +0300 Subject: [PATCH 22/23] updated django.Dockerfile --- clearcut_detection_backend/base.Dockerfile | 71 ------------ clearcut_detection_backend/django.Dockerfile | 114 +++++++++---------- 2 files changed, 53 insertions(+), 132 deletions(-) delete mode 100644 clearcut_detection_backend/base.Dockerfile diff --git a/clearcut_detection_backend/base.Dockerfile b/clearcut_detection_backend/base.Dockerfile deleted file mode 100644 index 39b69d4..0000000 --- a/clearcut_detection_backend/base.Dockerfile +++ /dev/null @@ -1,71 +0,0 @@ -FROM python:3.7.7 - -ENV PYTHONUNBUFFERED 1 -ENV ROOTDIR /usr/local/ -ARG GDAL_VERSION=3.1.4 - -WORKDIR ${ROOTDIR}/ - -RUN apt-get update && apt-get upgrade -y\ - && apt-get install software-properties-common -y\ - && add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -sc)-pgdg main"\ - && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -\ - && apt-get update\ - && apt-get install postgresql-client-13 -y\ - && apt-get install sqlite3 -y - -# See https://docs.djangoproject.com/en/3.1/ref/contrib/gis/install/geolibs/ - -RUN wget https://download.osgeo.org/geos/geos-3.8.1.tar.bz2 \ - && tar -xjf geos-3.8.1.tar.bz2 \ - && cd geos-3.8.1 \ - && ./configure \ - && make \ - && make install \ - && cd .. \ - && rm -rf geos-3.8.1 geos-3.8.1.tar.bz2 - -RUN wget https://download.osgeo.org/proj/proj-6.3.2.tar.gz \ - && wget https://download.osgeo.org/proj/proj-datumgrid-1.8.tar.gz \ - && tar -xzf proj-6.3.2.tar.gz \ - && mkdir proj-6.3.2/nad && cd proj-6.3.2/nad \ - && tar -xzf ../../proj-datumgrid-1.8.tar.gz \ - && cd .. \ - && ./configure \ - && make \ - && make install \ - && cd .. \ - && rm -rf proj-6.3.2 proj-6.3.2.tar.gz proj-datumgrid-1.8.tar.gz - -RUN apt-get -y install python3-pip -RUN pip3 install numpy==1.17.3 - -RUN wget https://download.osgeo.org/gdal/${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz \ - && tar -xzf gdal-${GDAL_VERSION}.tar.gz \ - && cd gdal-${GDAL_VERSION} \ - && ./configure --with-python=python3.7 --with-proj=${ROOTDIR} \ - && make \ - && make install \ - && cd swig/python \ - && python3 setup.py build \ - && python3 setup.py install \ - && cd ${ROOTDIR} \ - && rm -rf gdal-${GDAL_VERSION} gdal-${GDAL_VERSION}.tar.gz - -RUN git clone https://github.com/mapbox/tippecanoe.git \ - && cd tippecanoe \ - && git checkout tags/1.36.0 \ - && make -j \ - && make install \ - && cd .. \ - && rm -rf tippecanoe - -RUN ldconfig - -RUN mkdir /code - -WORKDIR /code - -COPY requirements.txt /code/ - -RUN pip3 install -r requirements.txt \ No newline at end of file diff --git a/clearcut_detection_backend/django.Dockerfile b/clearcut_detection_backend/django.Dockerfile index f34a3b3..39b69d4 100644 --- a/clearcut_detection_backend/django.Dockerfile +++ b/clearcut_detection_backend/django.Dockerfile @@ -1,79 +1,71 @@ -FROM python:3.6 +FROM python:3.7.7 -# Set environment variables -#ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 ENV ROOTDIR /usr/local/ -ENV GDAL_VERSION 2.2.4 -ENV OPENJPEG_VERSION 2.2.0 +ARG GDAL_VERSION=3.1.4 -WORKDIR $ROOTDIR/ +WORKDIR ${ROOTDIR}/ -ADD http://download.osgeo.org/gdal/${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz $ROOTDIR/src/ -ADD https://github.com/uclouvain/openjpeg/archive/v${OPENJPEG_VERSION}.tar.gz $ROOTDIR/src/openjpeg-${OPENJPEG_VERSION}.tar.gz +RUN apt-get update && apt-get upgrade -y\ + && apt-get install software-properties-common -y\ + && add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -sc)-pgdg main"\ + && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -\ + && apt-get update\ + && apt-get install postgresql-client-13 -y\ + && apt-get install sqlite3 -y +# See https://docs.djangoproject.com/en/3.1/ref/contrib/gis/install/geolibs/ -RUN apt-get update -y && apt-get install -y \ - software-properties-common \ - python3-software-properties \ - build-essential \ - python-dev \ - python3-dev \ - python-numpy \ - python3-numpy \ - libspatialite-dev \ - sqlite3 \ - libpq-dev \ - libcurl4-gnutls-dev \ - libproj-dev \ - libxml2-dev \ - libgeos-dev \ - libnetcdf-dev \ - libpoppler-dev \ - libspatialite-dev \ - libhdf4-alt-dev \ - libhdf5-serial-dev \ - bash-completion \ - cmake +RUN wget https://download.osgeo.org/geos/geos-3.8.1.tar.bz2 \ + && tar -xjf geos-3.8.1.tar.bz2 \ + && cd geos-3.8.1 \ + && ./configure \ + && make \ + && make install \ + && cd .. \ + && rm -rf geos-3.8.1 geos-3.8.1.tar.bz2 -RUN /bin/bash -c "pip install numpy==1.16.4" +RUN wget https://download.osgeo.org/proj/proj-6.3.2.tar.gz \ + && wget https://download.osgeo.org/proj/proj-datumgrid-1.8.tar.gz \ + && tar -xzf proj-6.3.2.tar.gz \ + && mkdir proj-6.3.2/nad && cd proj-6.3.2/nad \ + && tar -xzf ../../proj-datumgrid-1.8.tar.gz \ + && cd .. \ + && ./configure \ + && make \ + && make install \ + && cd .. \ + && rm -rf proj-6.3.2 proj-6.3.2.tar.gz proj-datumgrid-1.8.tar.gz -# Compile and install OpenJPEG -RUN cd src && tar -xvf openjpeg-${OPENJPEG_VERSION}.tar.gz && cd openjpeg-${OPENJPEG_VERSION}/ \ - && mkdir build && cd build \ - && cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$ROOTDIR \ - && make && make install && make clean \ - && cd $ROOTDIR && rm -Rf src/openjpeg* +RUN apt-get -y install python3-pip +RUN pip3 install numpy==1.17.3 -# Compile and install GDAL -RUN cd src && tar -xvf gdal-${GDAL_VERSION}.tar.gz && cd gdal-${GDAL_VERSION} \ - && ./configure --with-python --with-spatialite --with-pg --with-curl --with-openjpeg=$ROOTDIR \ - && make && make install && ldconfig \ - && apt-get update -y \ - && apt-get remove -y --purge build-essential wget \ - && cd $ROOTDIR && cd src/gdal-${GDAL_VERSION}/swig/python \ +RUN wget https://download.osgeo.org/gdal/${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz \ + && tar -xzf gdal-${GDAL_VERSION}.tar.gz \ + && cd gdal-${GDAL_VERSION} \ + && ./configure --with-python=python3.7 --with-proj=${ROOTDIR} \ + && make \ + && make install \ + && cd swig/python \ && python3 setup.py build \ && python3 setup.py install \ - && cd $ROOTDIR && rm -Rf src/gdal* + && cd ${ROOTDIR} \ + && rm -rf gdal-${GDAL_VERSION} gdal-${GDAL_VERSION}.tar.gz -# Output version and capabilities by default. -CMD gdalinfo --version && gdalinfo --formats && ogrinfo --formats +RUN git clone https://github.com/mapbox/tippecanoe.git \ + && cd tippecanoe \ + && git checkout tags/1.36.0 \ + && make -j \ + && make install \ + && cd .. \ + && rm -rf tippecanoe -RUN apt-get install -y libgdal-dev +RUN ldconfig -RUN apt-get update -y \ - && apt-get upgrade -y \ - && apt-get install -y postgresql-client locales \ - && apt-get install -y gdal-bin python-gdal python3-gdal \ - && apt-get update && apt-get install -y gettext libgettextpo-dev \ - # Cleanup - && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ - # mkdir for app - && mkdir /code +RUN mkdir /code -# Set work directory WORKDIR /code -# Install dependencies -COPY requirements.txt /code -RUN pip3 install -r requirements.txt +COPY requirements.txt /code/ + +RUN pip3 install -r requirements.txt \ No newline at end of file From 7aaad0a0233ce14664893c716de56167553439dd Mon Sep 17 00:00:00 2001 From: DmitriyLugovoy Date: Fri, 29 Oct 2021 11:12:28 +0300 Subject: [PATCH 23/23] requirements.txt was cleaned out --- clearcut_detection_backend/requirements.txt | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/clearcut_detection_backend/requirements.txt b/clearcut_detection_backend/requirements.txt index e4cdd7a..2d94037 100644 --- a/clearcut_detection_backend/requirements.txt +++ b/clearcut_detection_backend/requirements.txt @@ -1,7 +1,5 @@ celery==4.4.6 -# Django==2.2.14 Django==3.1 -# django-celery-results==1.2.1 SQLAlchemy==1.3.18 django-cors-headers==3.5.0 django-rest-swagger==2.2.0 @@ -15,20 +13,9 @@ google-cloud-core==1.0.3 google-cloud-storage==1.23.0 google-resumable-media==0.5.0 googleapis-common-protos==1.6.0 -#gsutil==4.47 -#imageio==2.5.0 invoke==1.2.0 -#numpy==1.16.4 oauth2client==4.1.3 -#pandas==0.24.2 -#Pillow==6.1.0 psycopg2-binary==2.8.3 -#pyproj==2.2.1 -#PyYAML==5.1.1 rasterio==1.1.2 -#rio-mbtiles==1.4.2 sentinelhub==2.5.3 -#Shapely==1.6.4.post2 -#six==1.12.0 -#tqdm==4.29.1 uWSGI==2.0.18