Merge pull request #103 from QuantuMobileSoftware/#14329-Please-fix-C…

clearcut_detection_backend/clearcut_detection_backend/prod_settings.py

@@ -9,3 +9,7 @@
 EMAIL_HOST_USER = 'SET_ME_PLEASE_SMTP_USERNAME'
 EMAIL_HOST_PASSWORD = 'SET_ME_PLEASE_SMTP_PASSWORD'
 EMAIL_ADMIN_MAIL = ['SET_ME_PLEASE_SMTP_ADMIN_MAIL']
+
+DOWNLOADED_IMAGES_DIR = Path('/data/source_images/')
+MODEL_TIFFS_DIR = Path('/data/model_tiffs')
+MAPBOX_TIFFS_DIR = Path('/data/mapbox_tiffs')

clearcut_detection_backend/cron_tab_prod

@@ -1,3 +1,3 @@
 # Cron tab prod
 
-0 0,12 * * * python update.py
+#0 0,12 * * * python update_new.py

clearcut_detection_backend/django.Dockerfile

@@ -1,79 +1,71 @@
-FROM python:3.6
+FROM python:3.7.7
 
-# Set environment variables
-#ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 ENV ROOTDIR /usr/local/
-ENV GDAL_VERSION 2.2.4
-ENV OPENJPEG_VERSION 2.2.0
+ARG GDAL_VERSION=3.1.4
 
-WORKDIR $ROOTDIR/
+WORKDIR ${ROOTDIR}/
 
-ADD http://download.osgeo.org/gdal/${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz $ROOTDIR/src/
-ADD https://github.com/uclouvain/openjpeg/archive/v${OPENJPEG_VERSION}.tar.gz $ROOTDIR/src/openjpeg-${OPENJPEG_VERSION}.tar.gz
+RUN apt-get update && apt-get upgrade -y\
+    && apt-get install software-properties-common -y\
+    && add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -sc)-pgdg main"\
+    && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -\
+    && apt-get update\
+    && apt-get install postgresql-client-13 -y\
+    && apt-get install sqlite3 -y
 
+# See https://docs.djangoproject.com/en/3.1/ref/contrib/gis/install/geolibs/
 
-RUN apt-get update -y && apt-get install -y \
-    software-properties-common \
-    python3-software-properties \
-    build-essential \
-    python-dev \
-    python3-dev \
-    python-numpy \
-    python3-numpy \
-    libspatialite-dev \
-    sqlite3 \
-    libpq-dev \
-    libcurl4-gnutls-dev \
-    libproj-dev \
-    libxml2-dev \
-    libgeos-dev \
-    libnetcdf-dev \
-    libpoppler-dev \
-    libspatialite-dev \
-    libhdf4-alt-dev \
-    libhdf5-serial-dev \
-    bash-completion \
-    cmake
+RUN wget https://download.osgeo.org/geos/geos-3.8.1.tar.bz2 \
+    && tar -xjf geos-3.8.1.tar.bz2 \
+    && cd geos-3.8.1 \
+    && ./configure \
+    && make \
+    && make install \
+    && cd .. \
+    && rm -rf geos-3.8.1 geos-3.8.1.tar.bz2
 
-RUN /bin/bash -c "pip install numpy==1.16.4"
+RUN wget https://download.osgeo.org/proj/proj-6.3.2.tar.gz \
+    && wget https://download.osgeo.org/proj/proj-datumgrid-1.8.tar.gz \
+    && tar -xzf proj-6.3.2.tar.gz \
+    && mkdir proj-6.3.2/nad && cd proj-6.3.2/nad \
+    && tar -xzf ../../proj-datumgrid-1.8.tar.gz \
+    && cd .. \
+    && ./configure \
+    && make \
+    && make install \
+    && cd .. \
+    && rm -rf proj-6.3.2 proj-6.3.2.tar.gz proj-datumgrid-1.8.tar.gz
 
-# Compile and install OpenJPEG
-RUN cd src && tar -xvf openjpeg-${OPENJPEG_VERSION}.tar.gz && cd openjpeg-${OPENJPEG_VERSION}/ \
-    && mkdir build && cd build \
-    && cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$ROOTDIR \
-    && make && make install && make clean \
-    && cd $ROOTDIR && rm -Rf src/openjpeg*
+RUN apt-get -y install python3-pip
+RUN pip3 install numpy==1.17.3
 
-# Compile and install GDAL
-RUN cd src && tar -xvf gdal-${GDAL_VERSION}.tar.gz && cd gdal-${GDAL_VERSION} \
-    && ./configure --with-python --with-spatialite --with-pg --with-curl --with-openjpeg=$ROOTDIR \
-    && make && make install && ldconfig \
-    && apt-get update -y \
-    && apt-get remove -y --purge build-essential wget \
-    && cd $ROOTDIR && cd src/gdal-${GDAL_VERSION}/swig/python \
+RUN wget https://download.osgeo.org/gdal/${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz \
+    && tar -xzf gdal-${GDAL_VERSION}.tar.gz \
+    && cd gdal-${GDAL_VERSION} \
+    && ./configure --with-python=python3.7 --with-proj=${ROOTDIR} \
+    && make \
+    && make install \
+    && cd swig/python \
     && python3 setup.py build \
     && python3 setup.py install \
-    && cd $ROOTDIR && rm -Rf src/gdal*
+    && cd ${ROOTDIR} \
+    && rm -rf gdal-${GDAL_VERSION} gdal-${GDAL_VERSION}.tar.gz
 
-# Output version and capabilities by default.
-CMD gdalinfo --version && gdalinfo --formats && ogrinfo --formats
+RUN git clone https://github.com/mapbox/tippecanoe.git \
+    && cd tippecanoe \
+    && git checkout tags/1.36.0 \
+    && make -j \
+    && make install \
+    && cd .. \
+    && rm -rf tippecanoe
 
-RUN apt-get install -y libgdal-dev
+RUN ldconfig
 
-RUN apt-get update -y \
-    && apt-get upgrade -y \
-    && apt-get install -y postgresql-client locales \
-    && apt-get install -y gdal-bin python-gdal python3-gdal \
-    && apt-get update && apt-get install -y gettext libgettextpo-dev \
-    # Cleanup
-    && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
-    # mkdir for app
-    && mkdir /code
+RUN mkdir /code
 
-# Set work directory
 WORKDIR /code
 
-# Install dependencies
-COPY requirements.txt /code
-RUN pip3 install -r requirements.txt
+COPY requirements.txt /code/
+
+RUN pip3 install -r requirements.txt

clearcut_detection_backend/docker-compose-prod.yml

@@ -1,101 +1,102 @@
-db_prod:
-  image: clearcut_detection/postgis_prod
-  expose:
-    - 5432
-#  ports:
-#    - "5433:5432"
-  env_file:
-    - db.env
-  volumes:
-    - ./db-data_prod/data:/var/lib/postgresql/data
-    - ./:/code
+services:
+  db_prod:
+    image: clearcut_detection/postgis_prod
+    expose:
+      - 5432
+    ports:
+      - "5433:5432"
+    env_file:
+      - db.env
+    volumes:
+      - /home/clearcut/prod/db-data_prod/data:/var/lib/postgresql/data
+      - ./:/code
 
-model_prod_2:
-  image: clearcut_detection/model2_prod
-  env_file:
-    - ./model2/model.env
-#    - ./model2/db.env
-#    - ./model2/rabbitmq.env
-  volumes:
-    - ./model2/:/model
-    - ./data/:/model/data
-  working_dir: /model
-  environment:
-    - DB_HOST=db_prod
-    - RABBITMQ_HOST=rabbitmq_prod
-    - CUDA_VISIBLE_DEVICES=0
-    - PREDICT=1
-  links:
-    - "rabbitmq_prod"
-    - "db_prod"
-  command: /bin/bash -c "./run_predict.sh"  # todo
+  model_prod_2:
+    image: clearcut_detection/model2_prod
+    env_file:
+      - ./model2/model.env
+    volumes:
+      - ./model2/:/model
+      - /home/clearcut/prod/data_prod:/data
+    working_dir: /model
+    environment:
+      - DB_HOST=db_prod
+      - RABBITMQ_HOST=rabbitmq_prod
+      - CUDA_VISIBLE_DEVICES=0
+      - PREDICT=1
+    links:
+      - "rabbitmq_prod"
+      - "db_prod"
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - capabilities: ["gpu"]
+              driver: "nvidia"
+              device_ids: ["GPU-b51efca2-01e2-2215-9434-5cc94ec7716e"]
 
-react_prod:
-  image: node:10.15.0
-  volumes:
-    - ./frontend:/frontend
-  working_dir: /frontend
-  expose:
-    - "3001"
-  links:
-    - "django_prod"
-  command: /bin/bash -c "npm install && npm run-script build"
+    command: /bin/bash -c "./run_predict.sh"  # todo
 
-django_prod:
-  image: clearcut_detection/backend_prod
-  env_file:
-    - db.env
-    - django.env
-    - rabbitmq.env
-  environment:
-    - DJANGO_SETTINGS_MODULE=clearcut_detection_backend.prod_settings
-    - DB_HOST=db_prod
-    - RABBITMQ_HOST=rabbitmq_prod
-#    - SENTINEL_DOWNLOAD=1  # prevent requesting to sentinel
-#    - DOWNLOAD_IMG=0  # skip downloading images if SENTINEL_DOWNLOAD=true
-#    - PREPARE_TIF=1  # skip preprocessing images
-#    - MAKE_PREDICT=0  # skip predicting of clearcuts
-#    - CONVERT_TO_TIFF=0  # skip converting to tiff for uploading
-#    - UPLOAD_TO_MAPBOX=1  # skip uploading to mapbox server
-  volumes:
-    - ./:/code
-  working_dir: /code
-  links:
-    - "db_prod"
-#    - "model_prod_2"
-    - "rabbitmq_prod"
-  expose:
-    - 9000
-#  ports:
-#    - "9001:9000"
-  command: /bin/bash -c "exec invoke runbackend"
+  react_prod:
+    image: node:10.15.0
+    volumes:
+      - ./frontend:/frontend
+    working_dir: /frontend
+    command: /bin/bash -c "npm install && npm run-script build"
 
-nginx_prod:
-  image: nginx:1.15.8
-  links:
-    - "django_prod"
-  expose:
-    - "8080"
-#  ports:
-#    - "8082:8082"
-  environment:
-    - VIRTUAL_HOST=clearcut.quantumobile.com
-    - VIRTUAL_PORT=8080
-    - LETSENCRYPT_HOST=clearcut.quantumobile.com
-    - LETSENCRYPT_EMAIL=o.tsiapa@quantumobile.com
-  volumes:
-    - ./nginx/nginx_prod.conf:/etc/nginx/conf.d/default.conf
-    - ./frontend/build:/frontend/build
+  django_prod:
+    image: clearcut_detection/backend_prod
+    env_file:
+      - db.env
+      - django.env
+      - rabbitmq.env
+    environment:
+      - DJANGO_SETTINGS_MODULE=clearcut_detection_backend.prod_settings
+      - DB_HOST=db_prod
+      - RABBITMQ_HOST=rabbitmq_prod
+  #    - SENTINEL_DOWNLOAD=1  # prevent requesting to sentinel
+  #    - DOWNLOAD_IMG=0  # skip downloading images if SENTINEL_DOWNLOAD=true
+  #    - PREPARE_TIF=1  # skip preprocessing images
+  #    - MAKE_PREDICT=0  # skip predicting of clearcuts
+  #    - CONVERT_TO_TIFF=0  # skip converting to tiff for uploading
+  #    - UPLOAD_TO_MAPBOX=1  # skip uploading to mapbox server
+    volumes:
+      - ./:/code
+      - /home/clearcut/prod/data_prod:/data
+    working_dir: /code
+    links:
+      - "db_prod"
+      - "rabbitmq_prod"
+    expose:
+      - 9000
+    command: /bin/bash -c "exec invoke runbackend"
 
-rabbitmq_prod:
-  hostname: "rabbitmq_prod"
-  labels:
-    NAME: "rabbitmq_prod"
-  image: "rabbitmq"
-  env_file:
-    - rabbitmq.env
-  expose:
-    - 15672
-    - 5672
-  restart: always
+  nginx_prod:
+    image: nginx:1.15.8
+    links:
+      - "django_prod"
+    expose:
+      - "8080"
+    ports:
+      - "8080:8080"
+    environment:
+      - VIRTUAL_HOST=clearcut.quantumobile.com
+      - VIRTUAL_PORT=8080
+      - LETSENCRYPT_HOST=clearcut.quantumobile.com
+      - LETSENCRYPT_EMAIL=o.tsiapa@quantumobile.com
+    volumes:
+      - ./nginx/nginx_prod.conf:/etc/nginx/conf.d/default.conf
+      - ./frontend/build:/frontend/build
+
+  rabbitmq_prod:
+    hostname: "rabbitmq_prod"
+    labels:
+      NAME: "rabbitmq_prod"
+    image: "rabbitmq"
+    env_file:
+      - rabbitmq.env
+    expose:
+      - 15672
+      - 5672
+    restart: always
 

clearcut_detection_backend/model2/celeryconfig.py

@@ -11,7 +11,7 @@
 worker_prefetch_multiplier = 1
 worker_max_tasks_per_child = 1
 task_acks_late = True
-task_time_limit = 60 * 15
+task_time_limit = 60 * 60
 # database_table_names = {
 #     'task': 'django_celery_results_taskresult',
 #     'group': 'myapp_groupmeta',

clearcut_detection_backend/model2/config/__init__.py

@@ -1,5 +1,6 @@
 import os
 import yaml
+from pathlib import Path
 
 with open('/model/predict_config.yml', 'r') as config:
     cfg = yaml.load(config, Loader=yaml.SafeLoader)
@@ -18,3 +19,16 @@
 RABBITMQ_PASS = os.environ.get('RABBITMQ_DEFAULT_PASS', 'guest')
 RABBITMQ_HOST = os.environ.get('RABBITMQ_HOST', 'rabbitmq_prod')
 RABBITMQ_PORT_NUMBER = os.environ.get('RABBITMQ_NODE_PORT_NUMBER', 5672)
+CUDA_VISIBLE_DEVICES = os.environ.get('CUDA_VISIBLE_DEVICES', '0')
+
+SCOPES = ['https://www.googleapis.com/auth/drive.file']
+
+LANDCOVER_POLYGONS_PATH = Path('/data/landcover')
+PREDICTED_PATH = Path('/data/predicted')
+LANDCOVER_FILENAME = 'S2A_OPER_GIP_TILPAR_MPC__20151209T095117_V20150622T000000_21000101T000000_B00.kml'
+SENTINEL_TILES = LANDCOVER_POLYGONS_PATH / LANDCOVER_FILENAME
+LANDCOVER_GEOJSON = LANDCOVER_POLYGONS_PATH / 'landcover_polygons.geojson'
+
+CLOUDS_PROBABILITY_THRESHOLD = 15
+NEAREST_POLYGONS_NUMBER = 10
+DATES_FOR_TILE = 2