From 896c80cb6476a1efd14340eb669f15a9923f12de Mon Sep 17 00:00:00 2001 From: NickNickGo Date: Wed, 28 Oct 2020 00:15:24 +0000 Subject: [PATCH 1/9] azure pipeline spec --- azure-pipelines.yml | 104 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000..d140ccf --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,104 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Azure pipeline for continuous integration""" + +trigger: + paths: + exclude: + - docs +jobs: + - job: Fairseq_unittests + timeoutInMinutes: 360 + #pool: + # vmImage: 'ubuntu-18.04' + pool: + name: default + demands: + - agent.name -equals gpu4 + steps: + - script: | + #set up docker + sudo docker run --gpus all --privileged -v '/datadrive/:/datadrive' -it adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash + #install fastseq + pip install --editable . + + #show environment + which python + python --version + which nvcc + nvcc --version + which fastseq + python -c "import torch; print('torch:', torch.__version__, torch)" + python -c "import torch; print('CUDA available:', torch.cuda.is_available())" + + #files chnaged in current PR + files_changed=$(git diff HEAD HEAD~ --name-only) + echo "Files Edited in Current PR" + echo $files_changed + + export CUDA_VISIBLE_DEVICES=2,3 + #check whether this PR is specific to fairseq/transformers/both. + #run PR specific unittests. + is_fairseq=$(echo $files_changed|grep fairseq) + is_transformers=$(echo $files_changed|grep transformers) + is_py_change=$(echo $files_changed|grep .py) + is_cpp_change=$(echo $files_changed|grep -e .cu -e .cpp) + + #purely fairseq change + if [ -n "$is_fairseq" -a -z "$is_transformers" ] + then + echo "Running Only fairseq unittests" + bash tests/run_fairseq_tests.sh + #purely transformers change + else if [ -z "$is_fairseq" -a -n "$is_transformers" ] + then + echo "Running Only transformers unittests" + bash tests/run_transformers_tests.sh + else if [ -n "$is_py_change" -o -n "$is_cpp_change" ] + then + echo "Running fairseq and transformers unittests" + bash tests/run_fairseq_tests.sh + bash tests/run_transformers_tests.sh + else + #bash tests/run_fairseq_tests.sh + #bash tests/run_transformers_tests.sh + echo " *** Skipping all unittests *** " + : + fi + fi + fi + + #Linting checks for python files + if [ -n "$is_py_change" ] + then + #install pylint, wrapt update is essential. + conda update --name --yes base conda + conda update --yes wrapt + pip install pylint + echo $files_changed|grep .py|while read file; do + pylint --rcfile=.pylintrc $file + done + else + : + fi + + #Linting checks for c++/cuda files + if [ -n "$is_cpp_change" ] + then + #install cpplint + pip install cpplint + echo $files_changed| grep .cpp|while read file; do + cpplint $file + done + echo $files_changed| grep .cu|while read file; do + cpplint $file + done + else + : + fi + + #run benchmarks + #cd benchmarks/ + #CUDA_VISIBLE_DEVICES=3 run_all_benchmarks.sh + displayName: 'setup environment and run fairseq unit tests' From c2290cddd0f2dc1f59090e2b46bd13cf35144363 Mon Sep 17 00:00:00 2001 From: NickNickGo <66033489+NickNickGo@users.noreply.github.com> Date: Tue, 27 Oct 2020 17:31:32 -0700 Subject: [PATCH 2/9] Update azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d140ccf..d0f13c4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -"""Azure pipeline for continuous integration""" +#Azure pipeline for continuous integration trigger: paths: From f012a98dc585bc1e7007ae7921dbd769c50991b5 Mon Sep 17 00:00:00 2001 From: NickNickGo <66033489+NickNickGo@users.noreply.github.com> Date: Wed, 28 Oct 2020 15:25:37 -0700 Subject: [PATCH 3/9] Update azure-pipelines.yml --- azure-pipelines.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d0f13c4..e426176 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,18 +8,16 @@ trigger: exclude: - docs jobs: - - job: Fairseq_unittests + - job: Fastseq_unittests timeoutInMinutes: 360 - #pool: - # vmImage: 'ubuntu-18.04' pool: name: default demands: - - agent.name -equals gpu4 + - agent.name -equals gpu0 steps: - script: | #set up docker - sudo docker run --gpus all --privileged -v '/datadrive/:/datadrive' -it adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash + sudo docker run --gpus all --privileged -it adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash #install fastseq pip install --editable . @@ -28,16 +26,15 @@ jobs: python --version which nvcc nvcc --version - which fastseq python -c "import torch; print('torch:', torch.__version__, torch)" python -c "import torch; print('CUDA available:', torch.cuda.is_available())" - #files chnaged in current PR - files_changed=$(git diff HEAD HEAD~ --name-only) + #files changed in current PR + files_changed=$(git diff --name-only main) echo "Files Edited in Current PR" echo $files_changed - export CUDA_VISIBLE_DEVICES=2,3 + export CUDA_VISIBLE_DEVICES=3 #check whether this PR is specific to fairseq/transformers/both. #run PR specific unittests. is_fairseq=$(echo $files_changed|grep fairseq) @@ -68,7 +65,6 @@ jobs: fi fi fi - #Linting checks for python files if [ -n "$is_py_change" ] then From 56bfda8dae7cf27dfbc992e6e3f8eba09598f5c8 Mon Sep 17 00:00:00 2001 From: NickNickGo <66033489+NickNickGo@users.noreply.github.com> Date: Wed, 28 Oct 2020 15:31:24 -0700 Subject: [PATCH 4/9] Update azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e426176..4ad3fe1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -17,7 +17,7 @@ jobs: steps: - script: | #set up docker - sudo docker run --gpus all --privileged -it adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash + sudo docker run --gpus all -d --privileged -v '/datadrive/:/datadrive' -it adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash #install fastseq pip install --editable . From 132e50e85e99125227475a04e3e29c9db838c489 Mon Sep 17 00:00:00 2001 From: NickNickGo <66033489+NickNickGo@users.noreply.github.com> Date: Wed, 28 Oct 2020 15:34:42 -0700 Subject: [PATCH 5/9] Update azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4ad3fe1..747d076 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -17,7 +17,7 @@ jobs: steps: - script: | #set up docker - sudo docker run --gpus all -d --privileged -v '/datadrive/:/datadrive' -it adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash + sudo docker run --gpus all --privileged -v '/datadrive/:/datadrive' -it adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash #install fastseq pip install --editable . From da5797400d436aa70144d639545135a01776ec54 Mon Sep 17 00:00:00 2001 From: NickNickGo <66033489+NickNickGo@users.noreply.github.com> Date: Wed, 28 Oct 2020 15:38:35 -0700 Subject: [PATCH 6/9] Update azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 747d076..5a70603 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,7 +13,7 @@ jobs: pool: name: default demands: - - agent.name -equals gpu0 + - agent.name -equals gpu4 steps: - script: | #set up docker From 561b2d0834c1927f2bba941493b889a3cb597dfb Mon Sep 17 00:00:00 2001 From: NickNickGo <66033489+NickNickGo@users.noreply.github.com> Date: Wed, 28 Oct 2020 16:42:17 -0700 Subject: [PATCH 7/9] Update azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5a70603..747d076 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,7 +13,7 @@ jobs: pool: name: default demands: - - agent.name -equals gpu4 + - agent.name -equals gpu0 steps: - script: | #set up docker From 9892e044f209a3f9c86bd0b96cef31cb2618d0fd Mon Sep 17 00:00:00 2001 From: NickNickGo <66033489+NickNickGo@users.noreply.github.com> Date: Wed, 28 Oct 2020 16:47:16 -0700 Subject: [PATCH 8/9] Update azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 747d076..1fa2d1b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -17,7 +17,7 @@ jobs: steps: - script: | #set up docker - sudo docker run --gpus all --privileged -v '/datadrive/:/datadrive' -it adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash + sudo docker run --gpus all --privileged -it adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash #install fastseq pip install --editable . From 8fb48abb6ad97d5e9dd0d8e8dd515d2c36bcd61b Mon Sep 17 00:00:00 2001 From: NickNickGo <66033489+NickNickGo@users.noreply.github.com> Date: Wed, 28 Oct 2020 16:51:24 -0700 Subject: [PATCH 9/9] Update beam_search_optimizer.py --- fastseq/optimizer/fairseq/beam_search_optimizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastseq/optimizer/fairseq/beam_search_optimizer.py b/fastseq/optimizer/fairseq/beam_search_optimizer.py index 743e632..c2fef33 100644 --- a/fastseq/optimizer/fairseq/beam_search_optimizer.py +++ b/fastseq/optimizer/fairseq/beam_search_optimizer.py @@ -1,6 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. - +# """Import the optimization for beam search related parts in FairSeq.""" from packaging import version