From 896c80cb6476a1efd14340eb669f15a9923f12de Mon Sep 17 00:00:00 2001
From: NickNickGo <nibhenda@microsoft.com>
Date: Wed, 28 Oct 2020 00:15:24 +0000
Subject: [PATCH 01/16] azure pipeline spec

---
 azure-pipelines.yml | 104 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 104 insertions(+)
 create mode 100644 azure-pipelines.yml

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
new file mode 100644
index 0000000..d140ccf
--- /dev/null
+++ b/azure-pipelines.yml
@@ -0,0 +1,104 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""Azure pipeline for continuous integration"""
+
+trigger:
+  paths:
+    exclude:
+      - docs      
+jobs:
+  - job: Fairseq_unittests
+    timeoutInMinutes: 360
+    #pool:
+    #  vmImage: 'ubuntu-18.04'
+    pool: 
+      name: default
+      demands:
+        - agent.name -equals gpu4 
+    steps:
+    - script: |
+        #set up docker
+        sudo docker run --gpus all --privileged   -v '/datadrive/:/datadrive'  -it adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash
+        #install fastseq
+        pip install --editable .
+      
+        #show environment
+        which python
+        python --version
+        which nvcc
+        nvcc --version
+        which fastseq
+        python -c "import torch; print('torch:', torch.__version__, torch)"
+        python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
+        
+        #files chnaged in current PR
+        files_changed=$(git diff HEAD HEAD~ --name-only)        
+        echo "Files Edited in Current PR"
+        echo $files_changed
+        
+        export CUDA_VISIBLE_DEVICES=2,3
+        #check whether this PR is specific to fairseq/transformers/both.
+        #run PR specific unittests.
+        is_fairseq=$(echo $files_changed|grep fairseq)
+        is_transformers=$(echo $files_changed|grep transformers)
+        is_py_change=$(echo $files_changed|grep .py)
+        is_cpp_change=$(echo $files_changed|grep -e .cu -e .cpp)
+              
+        #purely fairseq change
+        if [ -n "$is_fairseq" -a -z "$is_transformers" ]
+        then
+          echo "Running Only fairseq unittests"
+          bash tests/run_fairseq_tests.sh
+        #purely transformers change
+        else if [ -z "$is_fairseq" -a -n "$is_transformers" ]
+        then
+          echo "Running Only transformers unittests"
+          bash tests/run_transformers_tests.sh
+        else if [ -n "$is_py_change" -o -n "$is_cpp_change" ]
+        then
+          echo "Running fairseq and transformers unittests"
+          bash tests/run_fairseq_tests.sh
+          bash tests/run_transformers_tests.sh
+        else
+          #bash tests/run_fairseq_tests.sh
+          #bash tests/run_transformers_tests.sh
+          echo " *** Skipping all unittests *** "
+          :
+        fi
+        fi
+        fi
+
+        #Linting checks for python files
+        if [ -n "$is_py_change" ]
+        then
+          #install pylint, wrapt update is essential.
+          conda update --name --yes base conda
+          conda update --yes wrapt
+          pip install pylint
+          echo $files_changed|grep .py|while read file; do
+            pylint --rcfile=.pylintrc $file
+          done
+        else 
+          :
+        fi
+        
+        #Linting checks for c++/cuda files
+        if [ -n "$is_cpp_change" ]
+        then
+          #install cpplint
+          pip install cpplint
+          echo $files_changed| grep .cpp|while read file; do
+            cpplint $file
+          done
+          echo $files_changed| grep .cu|while read file; do
+            cpplint $file
+          done
+        else 
+          :
+        fi
+        
+        #run benchmarks 
+        #cd benchmarks/
+        #CUDA_VISIBLE_DEVICES=3 run_all_benchmarks.sh
+      displayName: 'setup environment and run fairseq unit tests'  

From c2290cddd0f2dc1f59090e2b46bd13cf35144363 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 27 Oct 2020 17:31:32 -0700
Subject: [PATCH 02/16] Update azure-pipelines.yml

---
 azure-pipelines.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index d140ccf..d0f13c4 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -1,7 +1,7 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
-"""Azure pipeline for continuous integration"""
+#Azure pipeline for continuous integration
 
 trigger:
   paths:

From f012a98dc585bc1e7007ae7921dbd769c50991b5 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Wed, 28 Oct 2020 15:25:37 -0700
Subject: [PATCH 03/16] Update azure-pipelines.yml

---
 azure-pipelines.yml | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index d0f13c4..e426176 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -8,18 +8,16 @@ trigger:
     exclude:
       - docs      
 jobs:
-  - job: Fairseq_unittests
+  - job: Fastseq_unittests
     timeoutInMinutes: 360
-    #pool:
-    #  vmImage: 'ubuntu-18.04'
     pool: 
       name: default
       demands:
-        - agent.name -equals gpu4 
+        - agent.name -equals gpu0 
     steps:
     - script: |
         #set up docker
-        sudo docker run --gpus all --privileged   -v '/datadrive/:/datadrive'  -it adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash
+        sudo docker run --gpus all --privileged -it adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash
         #install fastseq
         pip install --editable .
       
@@ -28,16 +26,15 @@ jobs:
         python --version
         which nvcc
         nvcc --version
-        which fastseq
         python -c "import torch; print('torch:', torch.__version__, torch)"
         python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
         
-        #files chnaged in current PR
-        files_changed=$(git diff HEAD HEAD~ --name-only)        
+        #files changed in current PR
+        files_changed=$(git diff --name-only main)
         echo "Files Edited in Current PR"
         echo $files_changed
         
-        export CUDA_VISIBLE_DEVICES=2,3
+        export CUDA_VISIBLE_DEVICES=3
         #check whether this PR is specific to fairseq/transformers/both.
         #run PR specific unittests.
         is_fairseq=$(echo $files_changed|grep fairseq)
@@ -68,7 +65,6 @@ jobs:
         fi
         fi
         fi
-
         #Linting checks for python files
         if [ -n "$is_py_change" ]
         then

From 56bfda8dae7cf27dfbc992e6e3f8eba09598f5c8 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Wed, 28 Oct 2020 15:31:24 -0700
Subject: [PATCH 04/16] Update azure-pipelines.yml

---
 azure-pipelines.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index e426176..4ad3fe1 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -17,7 +17,7 @@ jobs:
     steps:
     - script: |
         #set up docker
-        sudo docker run --gpus all --privileged -it adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash
+        sudo docker run --gpus all  -d  --privileged   -v '/datadrive/:/datadrive'  -it adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash
         #install fastseq
         pip install --editable .
       

From 132e50e85e99125227475a04e3e29c9db838c489 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Wed, 28 Oct 2020 15:34:42 -0700
Subject: [PATCH 05/16] Update azure-pipelines.yml

---
 azure-pipelines.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 4ad3fe1..747d076 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -17,7 +17,7 @@ jobs:
     steps:
     - script: |
         #set up docker
-        sudo docker run --gpus all  -d  --privileged   -v '/datadrive/:/datadrive'  -it adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash
+        sudo docker run --gpus all --privileged   -v '/datadrive/:/datadrive'  -it adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash
         #install fastseq
         pip install --editable .
       

From da5797400d436aa70144d639545135a01776ec54 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Wed, 28 Oct 2020 15:38:35 -0700
Subject: [PATCH 06/16] Update azure-pipelines.yml

---
 azure-pipelines.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 747d076..5a70603 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -13,7 +13,7 @@ jobs:
     pool: 
       name: default
       demands:
-        - agent.name -equals gpu0 
+        - agent.name -equals gpu4 
     steps:
     - script: |
         #set up docker

From 561b2d0834c1927f2bba941493b889a3cb597dfb Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Wed, 28 Oct 2020 16:42:17 -0700
Subject: [PATCH 07/16] Update azure-pipelines.yml

---
 azure-pipelines.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 5a70603..747d076 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -13,7 +13,7 @@ jobs:
     pool: 
       name: default
       demands:
-        - agent.name -equals gpu4 
+        - agent.name -equals gpu0 
     steps:
     - script: |
         #set up docker

From 9892e044f209a3f9c86bd0b96cef31cb2618d0fd Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Wed, 28 Oct 2020 16:47:16 -0700
Subject: [PATCH 08/16] Update azure-pipelines.yml

---
 azure-pipelines.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 747d076..1fa2d1b 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -17,7 +17,7 @@ jobs:
     steps:
     - script: |
         #set up docker
-        sudo docker run --gpus all --privileged   -v '/datadrive/:/datadrive'  -it adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash
+        sudo docker run --gpus all --privileged  -it adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash
         #install fastseq
         pip install --editable .
       

From dd2049d1f4303b8d8c29279929140113e7d3c1e5 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Wed, 28 Oct 2020 16:59:38 -0700
Subject: [PATCH 09/16] Update azure-pipelines.yml

---
 azure-pipelines.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 1fa2d1b..a362896 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -29,6 +29,11 @@ jobs:
         python -c "import torch; print('torch:', torch.__version__, torch)"
         python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
         
+        echo "*****************"
+        ls
+        echo "*****************"
+        git branch 
+        
         #files changed in current PR
         files_changed=$(git diff --name-only main)
         echo "Files Edited in Current PR"

From 6e3e23a73f5be310013c6b0dc4aaf32a5146b2e1 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Wed, 28 Oct 2020 17:08:56 -0700
Subject: [PATCH 10/16] Update azure-pipelines.yml

---
 azure-pipelines.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index a362896..2490554 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -33,6 +33,7 @@ jobs:
         ls
         echo "*****************"
         git branch 
+        git diff origin/main --name-only 
         
         #files changed in current PR
         files_changed=$(git diff --name-only main)

From 6d725c44d8012483e0496e3aa9e540f8a5bb1c2e Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Wed, 28 Oct 2020 17:33:35 -0700
Subject: [PATCH 11/16] Update azure-pipelines.yml

---
 azure-pipelines.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 2490554..77fa742 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -34,6 +34,9 @@ jobs:
         echo "*****************"
         git branch 
         git diff origin/main --name-only 
+        git remote add upstream_repo https://github.com/microsoft/fastseq.git
+        git fetch upstream_repo main:main
+        git diff upstream_repo/main
         
         #files changed in current PR
         files_changed=$(git diff --name-only main)

From 3646df41ab028ee10823c1ca1a13790b62b14fc7 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Wed, 28 Oct 2020 17:48:46 -0700
Subject: [PATCH 12/16] Update azure-pipelines.yml

---
 azure-pipelines.yml | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 77fa742..b1c92f9 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -30,13 +30,9 @@ jobs:
         python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
         
         echo "*****************"
-        ls
-        echo "*****************"
-        git branch 
-        git diff origin/main --name-only 
-        git remote add upstream_repo https://github.com/microsoft/fastseq.git
-        git fetch upstream_repo main:main
-        git diff upstream_repo/main
+        git branch  
+        git fetch https://NickNickGo:b501a939f028eaa8cdbcafed173a9fe13f8c3c66@github.com/microsoft/fastseq.git main:main
+        git diff main --name-only
         
         #files changed in current PR
         files_changed=$(git diff --name-only main)

From 49aa7f86af9d2213752a6f0a8b69f393831ac133 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Wed, 28 Oct 2020 17:59:57 -0700
Subject: [PATCH 13/16] Update azure-pipelines.yml

---
 azure-pipelines.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index b1c92f9..6254c46 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -3,10 +3,11 @@
 
 #Azure pipeline for continuous integration
 
-trigger:
-  paths:
-    exclude:
-      - docs      
+pr:
+  branches:
+    include:
+    - '*'  # must quote since "*" is a YAML reserved character; we want a string
+
 jobs:
   - job: Fastseq_unittests
     timeoutInMinutes: 360

From 0f9fb4ef3d3f2b668fab1bdbe35e19f2ac506b78 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Wed, 28 Oct 2020 18:01:01 -0700
Subject: [PATCH 14/16] Update azure-pipelines.yml

---
 azure-pipelines.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 6254c46..039ab66 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -3,10 +3,11 @@
 
 #Azure pipeline for continuous integration
 
-pr:
-  branches:
-    include:
-    - '*'  # must quote since "*" is a YAML reserved character; we want a string
+trigger:
+  pr:
+    branches:
+      include:
+      - '*'  # must quote since "*" is a YAML reserved character; we want a string
 
 jobs:
   - job: Fastseq_unittests

From 757113390405be1f5c74473d80dae79977b18b4e Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Wed, 28 Oct 2020 18:06:26 -0700
Subject: [PATCH 15/16] Update azure-pipelines.yml

---
 azure-pipelines.yml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 039ab66..78c6ce2 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -4,11 +4,9 @@
 #Azure pipeline for continuous integration
 
 trigger:
-  pr:
-    branches:
-      include:
-      - '*'  # must quote since "*" is a YAML reserved character; we want a string
-
+  branches:
+    include:
+    - '*'  # must
 jobs:
   - job: Fastseq_unittests
     timeoutInMinutes: 360

From 800bf305e6ae278617521fb9fff7801f45505c1a Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Thu, 29 Oct 2020 09:58:24 -0700
Subject: [PATCH 16/16] Update azure-pipelines.yml

---
 azure-pipelines.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 78c6ce2..b56433a 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -1,6 +1,6 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
-
+#
 #Azure pipeline for continuous integration
 
 trigger: