diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000..26e9fb3 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,100 @@ +# +trigger: + paths: + exclude: + - docs +jobs: + - job: Fairseq_unittests + timeoutInMinutes: 360 + #pool: + # vmImage: 'ubuntu-18.04' + pool: + name: default + demands: + - agent.name -equals gpu4 + steps: + - script: | + #set up docker + sudo docker run --gpus all --privileged -v '/datadrive/:/datadrive' -it adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash + #install fastseq + pip install --editable . + + #show environment + which python + python --version + which nvcc + nvcc --version + which fastseq + python -c "import torch; print('torch:', torch.__version__, torch)" + python -c "import torch; print('CUDA available:', torch.cuda.is_available())" + + #files chnaged in current PR + files_changed=$(git diff HEAD HEAD~ --name-only) + echo "Files Edited in Current PR" + echo $files_changed + + export CUDA_VISIBLE_DEVICES=2,3 + #check whether this PR is specific to fairseq/transformers/both. + #run PR specific unittests. + is_fairseq=$(echo $files_changed|grep fairseq) + is_transformers=$(echo $files_changed|grep transformers) + is_py_change=$(echo $files_changed|grep .py) + is_cpp_change=$(echo $files_changed|grep -e .cu -e .cpp) + + #purely fairseq change + if [ -n "$is_fairseq" -a -z "$is_transformers" ] + then + echo "Running Only fairseq unittests" + bash tests/run_fairseq_tests.sh + #purely transformers change + else if [ -z "$is_fairseq" -a -n "$is_transformers" ] + then + echo "Running Only transformers unittests" + bash tests/run_transformers_tests.sh + else if [ -n "$is_py_change" -o -n "$is_cpp_change" ] + then + echo "Running fairseq and transformers unittests" + bash tests/run_fairseq_tests.sh + bash tests/run_transformers_tests.sh + else + #bash tests/run_fairseq_tests.sh + #bash tests/run_transformers_tests.sh + echo " *** Skipping all unittests *** " + : + fi + fi + fi + + #Linting checks for python files + if [ -n "$is_py_change" ] + then + #install pylint, wrapt update is essential. + conda update --name --yes base conda + conda update --yes wrapt + pip install pylint + echo $files_changed|grep .py|while read file; do + pylint --rcfile=.pylintrc $file + done + else + : + fi + + #Linting checks for c++/cuda files + if [ -n "$is_cpp_change" ] + then + #install cpplint + pip install cpplint + echo $files_changed| grep .cpp|while read file; do + cpplint $file + done + echo $files_changed| grep .cu|while read file; do + cpplint $file + done + else + : + fi + + #run benchmarks + #cd benchmarks/ + #CUDA_VISIBLE_DEVICES=3 run_all_benchmarks.sh + displayName: 'setup environment and run fairseq unit tests' diff --git a/fastseq/optimizer/fairseq/beam_search_optimizer.py b/fastseq/optimizer/fairseq/beam_search_optimizer.py index 743e632..9dba51d 100644 --- a/fastseq/optimizer/fairseq/beam_search_optimizer.py +++ b/fastseq/optimizer/fairseq/beam_search_optimizer.py @@ -1,6 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. - +### """Import the optimization for beam search related parts in FairSeq.""" from packaging import version diff --git a/tests/utils/test_file_utils.py b/tests/utils/test_file_utils.py index 2b48a30..7910dd8 100644 --- a/tests/utils/test_file_utils.py +++ b/tests/utils/test_file_utils.py @@ -1,5 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +# """Test the file_util functions."""