From 7a7379a3f473f3c71d7d875c735aa200fa0aef43 Mon Sep 17 00:00:00 2001 From: Jay Beavers Date: Thu, 29 Jul 2021 11:39:48 -0700 Subject: [PATCH 1/2] WIP: Docker Debugging and VSCode Tasks --- .dockerignore | 26 ++++++++++++++++++++++ .gitignore | 1 - .vscode/launch.json | 19 ++++++++++++++++ .vscode/tasks.json | 54 +++++++++++++++++++++++++++++++++++++++++++++ Dockerfile | 29 ++++++++++++++++++------ taskCreator.py | 5 ++++- 6 files changed, 125 insertions(+), 9 deletions(-) create mode 100644 .dockerignore create mode 100644 .vscode/launch.json create mode 100644 .vscode/tasks.json diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..da7fe8e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,26 @@ +**/__pycache__ +**/.classpath +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/azds.yaml +**/bin +**/charts +**/docker-compose* +**/compose* +**/Dockerfile* +**/node_modules +**/npm-debug.log +**/obj +**/secrets.dev.yaml +**/values.dev.yaml +README.md diff --git a/.gitignore b/.gitignore index 7fc2a18..9d18e98 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -*.json *.tar checkpoints* venv* diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..4141a95 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,19 @@ +{ + "configurations": [ + { + "name": "Prepare Eye Catcher (Docker)", + "type": "docker", + "request": "launch", + "preLaunchTask": "Prepare Eye Catcher", + "python": { + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "/app" + } + ], + "projectType": "general" + } + } + ] +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..696c47f --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,54 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "docker-build", + "type": "docker-build", + "platform": "python", + "dockerBuild": { + "tag": "gazecapture:latest", + "dockerfile": "${workspaceFolder}/Dockerfile", + "context": "${workspaceFolder}", + "pull": true + } + }, + { + "type": "docker-run", + "label": "Prepare Eye Catcher", + "dependsOn": [ + "docker-build" + ], + "python": { + "args": ["--task", "'prepareEyeCatcherTask'", "--input", "/data/200407", "--output", "/data/out"], + "file": "taskCreator.py" + }, + "dockerRun": { + "volumes": [ + { + "localPath": "/var/run/docker.sock", + "containerPath": "/var/run/docker.sock" + }, + { + "localPath": "/data", + "containerPath": "/data" + }, + { + "localPath": "${workspaceFolder}", + "containerPath": "/app" + } + ] + } + }, + { + "type": "docker-run", + "label": "train (reset)", + "dependsOn": ["docker-build"], + "python": { + "args": ["--data_path", "/data/out-prepped", "--reset"], + "file": "main.py" + } + } + ] +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 6e7b2b8..60a64a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,34 @@ FROM nvidia/cuda:11.2.2-devel-ubuntu20.04 FROM python:3.7 -Maintainer MSREnable +LABEL org.opencontainers.image.authors="MSREnable@Microsoft.Com" + +# Keeps Python from generating .pyc files in the container +ENV PYTHONDONTWRITEBYTECODE=1 +# Turns off buffering for easier container logging +ENV PYTHONUNBUFFERED=1 +# nvidia-container-runtime +ENV NVIDIA_VISIBLE_DEVICES all +ENV NVIDIA_DRIVER_CAPABILITIES all RUN apt-get update && apt-get install -y build-essential cmake apt-utils RUN pip3 install --upgrade pip -COPY ./requirements.txt ./requirements.txt +COPY requirements.txt . RUN pip3 install -r requirements.txt +VOLUME [ "/app" ] # Dynamically mounted from the host +VOLUME [ "/data" ] # Dynamically mounted from the host + +# COPY . /app # Now dynamically mounted from the host +WORKDIR /app + +# Creates a non-root user with an explicit UID and adds permission to access the /app folder +# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers +RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /app +USER appuser + EXPOSE 8097 # Execute the python -ENTRYPOINT ["python"] - -# nvidia-container-runtime -ENV NVIDIA_VISIBLE_DEVICES all -ENV NVIDIA_DRIVER_CAPABILITIES all \ No newline at end of file +ENTRYPOINT ["python"] \ No newline at end of file diff --git a/taskCreator.py b/taskCreator.py index b25d12b..be1858d 100644 --- a/taskCreator.py +++ b/taskCreator.py @@ -1680,7 +1680,10 @@ def userCalibrationTask(filepath): dataLoader = None # run the job - output = taskManager.job(taskFunction, taskData, dataLoader) + # output = taskManager.job(taskFunction, taskData, dataLoader, 1) + + # temporarily single thread this to simplify debugging + output = taskManager.job(taskFunction, taskData, dataLoader, 1) # post-processing after the task has completed if args.task == "CaptureDataDistributionTask": From b987b6ee09f0d02ac41313965e018b409e2882f4 Mon Sep 17 00:00:00 2001 From: Jay Beavers Date: Thu, 29 Jul 2021 13:59:45 -0700 Subject: [PATCH 2/2] WIP: Update tasks to enable debugging of ROIDetection --- .vscode/launch.json | 4 ++-- .vscode/tasks.json | 37 +++++++++++++++++++++++++++++++------ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 4141a95..6338a2e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,10 +1,10 @@ { "configurations": [ { - "name": "Prepare Eye Catcher (Docker)", + "name": "Step 1: ROI Detection", "type": "docker", "request": "launch", - "preLaunchTask": "Prepare Eye Catcher", + "preLaunchTask": "Step 1: ROI Detection", "python": { "pathMappings": [ { diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 696c47f..00de118 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -1,6 +1,4 @@ { - // See https://go.microsoft.com/fwlink/?LinkId=733558 - // for the documentation about the tasks.json format "version": "2.0.0", "tasks": [ { @@ -16,12 +14,12 @@ }, { "type": "docker-run", - "label": "Prepare Eye Catcher", + "label": "Step 1: ROI Detection", "dependsOn": [ "docker-build" ], "python": { - "args": ["--task", "'prepareEyeCatcherTask'", "--input", "/data/200407", "--output", "/data/out"], + "args": ["--task", "'ROIDetectionTask'", "--input", "/data/200407", "--output", "/data/meta"], "file": "taskCreator.py" }, "dockerRun": { @@ -43,10 +41,37 @@ }, { "type": "docker-run", - "label": "train (reset)", + "label": "Step 2: ROI Extraction", + "dependsOn": [ + "docker-build" + ], + "python": { + "args": ["--task", "'ROIExtraction'", "--input", "/data/200407", "--metapath", "/data/meta", "--output", "/data/prepped"], + "file": "taskCreator.py" + }, + "dockerRun": { + "volumes": [ + { + "localPath": "/var/run/docker.sock", + "containerPath": "/var/run/docker.sock" + }, + { + "localPath": "/data", + "containerPath": "/data" + }, + { + "localPath": "${workspaceFolder}", + "containerPath": "/app" + } + ] + } + }, + { + "type": "docker-run", + "label": "Step 3: train (reset)", "dependsOn": ["docker-build"], "python": { - "args": ["--data_path", "/data/out-prepped", "--reset"], + "args": ["--data_path", "/data/prepped", "--reset"], "file": "main.py" } }