From 4e895d86aea9e9917735744190bd4c271d3f25e4 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 20 Oct 2020 16:01:25 -0700
Subject: [PATCH 01/39] Set up CI with Azure Pipelines

[skip ci]
---
 azure-pipelines.yml | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 azure-pipelines.yml

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
new file mode 100644
index 0000000..aa91291
--- /dev/null
+++ b/azure-pipelines.yml
@@ -0,0 +1,19 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+trigger:
+- master
+
+pool:
+  vmImage: 'ubuntu-latest'
+
+steps:
+- script: echo Hello, world!
+  displayName: 'Run a one-line script'
+
+- script: |
+    echo Add other tasks to build, test, and deploy your project.
+    echo See https://aka.ms/yaml
+  displayName: 'Run a multi-line script'

From 675589ade5dba1daff898f7171e03dab993e426c Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 20 Oct 2020 16:16:18 -0700
Subject: [PATCH 02/39] Update azure-pipelines.yml

initial pipeline
---
 azure-pipelines.yml | 88 +++++++++++++++++++++++++++++++++++++--------
 1 file changed, 74 insertions(+), 14 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index aa91291..4270bb9 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -1,19 +1,79 @@
-# Starter pipeline
-# Start with a minimal pipeline that you can customize to build and deploy your code.
-# Add steps that build, run tests, deploy, and more:
-# https://aka.ms/yaml
+# Python package
+# Create and test a Python package on multiple Python versions.
+# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
+# https://docs.microsoft.com/azure/devops/pipelines/languages/python
 
 trigger:
-- master
+  branches:
+    include:
+    - master
 
-pool:
-  vmImage: 'ubuntu-latest'
+jobs:
+  - job: DeepSpeed_Tests
+    timeoutInMinutes: 360
+    pool:
+      vmImage: 'ubuntu-18.04'
+    #container: pytorch/pytorch:latest
+    container:
+      image: adsbrainwestus2.azurecr.io/fastseq:dev-py3
+      endpoint: acr-westus2
+    
+    #container: adsbrainwestus2.azurecr.io/fastseq:dev-py3
+      
+    # strategy:
+    #   matrix:
+    #     PyTorch12-CUDA100:
+    #       python.version: '3.6'
+    #       cuda.version: '10.0'
+    #       pytorch.version: '0.9.0'
 
-steps:
-- script: echo Hello, world!
-  displayName: 'Run a one-line script'
+    steps:
+        # Unfortunately nvidia's nvcc_linux-64=<version> seems to install 10.1 regardless?
+        # Most of this complexity is a workaround to get the compiler toolchain to match the
+        # cudatoolkit runtime
+    - script: |
+        #conda create --force --yes -n $(conda_env) python=$(python.version) cudatoolkit=$(cuda.version)
+        #conda init bash
+        #conda activate $(conda_env)
+        #conda install -q --yes conda
+        #conda install -q --yes pip
+        #conda install -q --yes gxx_linux-64
+        #if [[ $(cuda.version) != "10.2" ]]; then conda install --yes -c conda-forge cudatoolkit-dev=$(cuda.version) ; fi
+        #echo "PATH=$PATH, LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
+        python --version
+        which nvcc
+        nvcc --version
+        which fastseq
+        python test.py
+      displayName: 'Setup environment python=$(python.version) pytorch=$(pytorch.version) cuda=$(cuda.version)'
 
-- script: |
-    echo Add other tasks to build, test, and deploy your project.
-    echo See https://aka.ms/yaml
-  displayName: 'Run a multi-line script'
+    # - script: |
+    #     source activate $(conda_env)
+    #     pip install --progress-bar=off torch==$(pytorch.version)
+    #     #-f https://download.pytorch.org/whl/torch_stable.html
+    #     a=$(python --version)
+    #     echo $a
+    #     pip install fastseq
+    #     #python -I basic_install_test.py
+    #   displayName: 'Install Fastseq'
+    # - script: |
+    #     sudo docker run --gpus all  --network=host --privileged -w '/root'  --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864  adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash
+    #     python --version
+    #     which nvcc
+    #     nvcc --version
+    #     which fastseq
+    #     ls
+    #   displayName: 'docker'
+  
+    # - script: |
+    #     source activate $(conda_env)
+    #     which python
+    #     python --version
+    #     which nvcc
+    #     nvcc --version
+    #     which fastseq
+    #     python -c "import torch; print('torch:', torch.__version__, torch)"
+    #     python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
+    #     #python -c "import deepspeed; print('fastseq:', fastseq.__version__)"
+    #   displayName: 'Show environment'
+  

From 9e90413db719e5cc508225db074247f3aa9a87db Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 20 Oct 2020 16:18:46 -0700
Subject: [PATCH 03/39] Update azure-pipelines.yml

---
 azure-pipelines.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 4270bb9..513d5a0 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -6,7 +6,7 @@
 trigger:
   branches:
     include:
-    - master
+    - main
 
 jobs:
   - job: DeepSpeed_Tests

From 28fc09910ab32a95d852c3f73d649f5c30a8bcbc Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 20 Oct 2020 16:35:04 -0700
Subject: [PATCH 04/39] Update azure-pipelines.yml

---
 azure-pipelines.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 513d5a0..cfda3b6 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -44,7 +44,7 @@ jobs:
         which nvcc
         nvcc --version
         which fastseq
-        python test.py
+        bash tests/run_fairseq_tests.sh
       displayName: 'Setup environment python=$(python.version) pytorch=$(pytorch.version) cuda=$(cuda.version)'
 
     # - script: |

From 1f34332e491ad44c03d9e1c02a76da8db6d08f4a Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 20 Oct 2020 17:55:35 -0700
Subject: [PATCH 05/39] Update azure-pipelines.yml

---
 azure-pipelines.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index cfda3b6..ba54c59 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -11,8 +11,9 @@ trigger:
 jobs:
   - job: DeepSpeed_Tests
     timeoutInMinutes: 360
-    pool:
-      vmImage: 'ubuntu-18.04'
+    #pool:
+    #  vmImage: 'ubuntu-18.04'
+    pool: gpu4
     #container: pytorch/pytorch:latest
     container:
       image: adsbrainwestus2.azurecr.io/fastseq:dev-py3

From e1f154b0ca33728a6f264e42d28ed624ff499267 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 20 Oct 2020 18:07:45 -0700
Subject: [PATCH 06/39] Update azure-pipelines.yml

---
 azure-pipelines.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index ba54c59..e16fe6d 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -13,7 +13,7 @@ jobs:
     timeoutInMinutes: 360
     #pool:
     #  vmImage: 'ubuntu-18.04'
-    pool: gpu4
+    pool:'gpu4'
     #container: pytorch/pytorch:latest
     container:
       image: adsbrainwestus2.azurecr.io/fastseq:dev-py3

From 62f6a294a82a3122e2b0d8ee820c3259269150cf Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 20 Oct 2020 18:27:58 -0700
Subject: [PATCH 07/39] Update azure-pipelines.yml

---
 azure-pipelines.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index e16fe6d..458d973 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -13,6 +13,7 @@ jobs:
     timeoutInMinutes: 360
     #pool:
     #  vmImage: 'ubuntu-18.04'
+    #haha
     pool:'gpu4'
     #container: pytorch/pytorch:latest
     container:

From 33f90502d312076792995d44358bb850efc137dd Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 20 Oct 2020 18:32:37 -0700
Subject: [PATCH 08/39] Update azure-pipelines.yml

---
 azure-pipelines.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 458d973..e16fe6d 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -13,7 +13,6 @@ jobs:
     timeoutInMinutes: 360
     #pool:
     #  vmImage: 'ubuntu-18.04'
-    #haha
     pool:'gpu4'
     #container: pytorch/pytorch:latest
     container:

From c2a5a9eabd87d452fa4552184fb63b13c1944ac6 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 20 Oct 2020 18:43:34 -0700
Subject: [PATCH 09/39] Update azure-pipelines.yml

---
 azure-pipelines.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index e16fe6d..458d973 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -13,6 +13,7 @@ jobs:
     timeoutInMinutes: 360
     #pool:
     #  vmImage: 'ubuntu-18.04'
+    #haha
     pool:'gpu4'
     #container: pytorch/pytorch:latest
     container:

From 650fe2bd4a07ac917da1bb153abc768a30cb3ca0 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 20 Oct 2020 18:50:56 -0700
Subject: [PATCH 10/39] Update azure-pipelines.yml

---
 azure-pipelines.yml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 458d973..627f3d1 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -14,7 +14,13 @@ jobs:
     #pool:
     #  vmImage: 'ubuntu-18.04'
     #haha
-    pool:'gpu4'
+    pool: 
+      name: default
+      demands:
+        - agent.name -equals gpu4 
+
+
+    #pool:'gpu4'
     #container: pytorch/pytorch:latest
     container:
       image: adsbrainwestus2.azurecr.io/fastseq:dev-py3

From ff8fd8c60c3bccbacb564faa2f34e5d7850aef3f Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 20 Oct 2020 19:01:23 -0700
Subject: [PATCH 11/39] Update azure-pipelines.yml

---
 azure-pipelines.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 627f3d1..9e6cc2d 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -13,7 +13,6 @@ jobs:
     timeoutInMinutes: 360
     #pool:
     #  vmImage: 'ubuntu-18.04'
-    #haha
     pool: 
       name: default
       demands:

From c91ba67aea7bdfdb88ce439a6f34869568155e67 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 20 Oct 2020 19:08:22 -0700
Subject: [PATCH 12/39] Update azure-pipelines.yml

---
 azure-pipelines.yml | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 9e6cc2d..6dcbe0a 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -19,11 +19,9 @@ jobs:
         - agent.name -equals gpu4 
 
 
-    #pool:'gpu4'
-    #container: pytorch/pytorch:latest
-    container:
-      image: adsbrainwestus2.azurecr.io/fastseq:dev-py3
-      endpoint: acr-westus2
+    #container:
+    #  image: adsbrainwestus2.azurecr.io/fastseq:dev-py3
+    #  endpoint: acr-westus2
     
     #container: adsbrainwestus2.azurecr.io/fastseq:dev-py3
       
@@ -48,10 +46,10 @@ jobs:
         #if [[ $(cuda.version) != "10.2" ]]; then conda install --yes -c conda-forge cudatoolkit-dev=$(cuda.version) ; fi
         #echo "PATH=$PATH, LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
         python --version
-        which nvcc
-        nvcc --version
-        which fastseq
-        bash tests/run_fairseq_tests.sh
+        #which nvcc
+        #nvcc --version
+        #which fastseq
+        #bash tests/run_fairseq_tests.sh
       displayName: 'Setup environment python=$(python.version) pytorch=$(pytorch.version) cuda=$(cuda.version)'
 
     # - script: |

From 4b79eb18fd46d4fd957c7f0ccafcb32cc8d8c5fa Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 20 Oct 2020 19:11:28 -0700
Subject: [PATCH 13/39] Update azure-pipelines.yml

---
 azure-pipelines.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 6dcbe0a..d2ac6c6 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -50,6 +50,8 @@ jobs:
         #nvcc --version
         #which fastseq
         #bash tests/run_fairseq_tests.sh
+        sudo docker run --gpus all  --network=host --privileged -w '/root'  --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864  adsbrainwestus2.azurecr.io/fastseq:dev
+        python --version
       displayName: 'Setup environment python=$(python.version) pytorch=$(pytorch.version) cuda=$(cuda.version)'
 
     # - script: |

From ec43d27b880373c8c89d5986f0d680482307692e Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 20 Oct 2020 19:15:32 -0700
Subject: [PATCH 14/39] Update azure-pipelines.yml

---
 azure-pipelines.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index d2ac6c6..2b9be13 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -50,7 +50,7 @@ jobs:
         #nvcc --version
         #which fastseq
         #bash tests/run_fairseq_tests.sh
-        sudo docker run --gpus all  --network=host --privileged -w '/root'  --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864  adsbrainwestus2.azurecr.io/fastseq:dev
+        sudo docker run --gpus all  --network=host --privileged -w '/root'  --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864  adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash
         python --version
       displayName: 'Setup environment python=$(python.version) pytorch=$(pytorch.version) cuda=$(cuda.version)'
 

From 7bd64a81dabbd2be1bada8a06f0cdfc32ec85abd Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 20 Oct 2020 19:18:57 -0700
Subject: [PATCH 15/39] Update azure-pipelines.yml

---
 azure-pipelines.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 2b9be13..79bb3e5 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -51,6 +51,7 @@ jobs:
         #which fastseq
         #bash tests/run_fairseq_tests.sh
         sudo docker run --gpus all  --network=host --privileged -w '/root'  --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864  adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash
+        bash tests/run_fairseq_tests.sh
         python --version
       displayName: 'Setup environment python=$(python.version) pytorch=$(pytorch.version) cuda=$(cuda.version)'
 

From 6d98718367b3965f0eba315c75132695b9765013 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 20 Oct 2020 20:30:23 -0700
Subject: [PATCH 16/39] Update azure-pipelines.yml

---
 azure-pipelines.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 79bb3e5..b9a9c19 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -51,6 +51,8 @@ jobs:
         #which fastseq
         #bash tests/run_fairseq_tests.sh
         sudo docker run --gpus all  --network=host --privileged -w '/root'  --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864  adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash
+        ls
+        pip install --editable .
         bash tests/run_fairseq_tests.sh
         python --version
       displayName: 'Setup environment python=$(python.version) pytorch=$(pytorch.version) cuda=$(cuda.version)'

From 57fb6d9559b02f43536cc50291364ea84ab024e5 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 20 Oct 2020 20:40:04 -0700
Subject: [PATCH 17/39] Update beam_search_optimizer_v1.py

---
 fastseq/optimizer/fairseq/beam_search_optimizer_v1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fastseq/optimizer/fairseq/beam_search_optimizer_v1.py b/fastseq/optimizer/fairseq/beam_search_optimizer_v1.py
index 83adba4..ec4d7b0 100644
--- a/fastseq/optimizer/fairseq/beam_search_optimizer_v1.py
+++ b/fastseq/optimizer/fairseq/beam_search_optimizer_v1.py
@@ -53,7 +53,7 @@ class MultiheadAttentionV2(MultiheadAttention):
     """
 
     def __init__(self,
-                 embed_dim,
+                 embed_dim____,
                  num_heads,
                  kdim=None,
                  vdim=None,

From ed3497d29ef3cf9585511a31f9c29b167d81baa5 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 20 Oct 2020 20:46:49 -0700
Subject: [PATCH 18/39] Update beam_search_optimizer_v1.py

---
 fastseq/optimizer/fairseq/beam_search_optimizer_v1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fastseq/optimizer/fairseq/beam_search_optimizer_v1.py b/fastseq/optimizer/fairseq/beam_search_optimizer_v1.py
index ec4d7b0..83adba4 100644
--- a/fastseq/optimizer/fairseq/beam_search_optimizer_v1.py
+++ b/fastseq/optimizer/fairseq/beam_search_optimizer_v1.py
@@ -53,7 +53,7 @@ class MultiheadAttentionV2(MultiheadAttention):
     """
 
     def __init__(self,
-                 embed_dim____,
+                 embed_dim,
                  num_heads,
                  kdim=None,
                  vdim=None,

From 83943babd71075073051ccfacc24aaa9bf7c8d7b Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 20 Oct 2020 20:55:30 -0700
Subject: [PATCH 19/39] Update azure-pipelines.yml

---
 azure-pipelines.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index b9a9c19..2ea54ae 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -4,10 +4,9 @@
 # https://docs.microsoft.com/azure/devops/pipelines/languages/python
 
 trigger:
-  branches:
-    include:
-    - main
-
+  paths:
+    exclude:
+      - docs      
 jobs:
   - job: DeepSpeed_Tests
     timeoutInMinutes: 360

From 489662fa55061eebe0d375e6ae69e0aa162b6e4c Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Tue, 20 Oct 2020 21:26:15 -0700
Subject: [PATCH 20/39] Update azure-pipelines.yml

---
 azure-pipelines.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 2ea54ae..c62e911 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -8,7 +8,7 @@ trigger:
     exclude:
       - docs      
 jobs:
-  - job: DeepSpeed_Tests
+  - job: Fastseq_Tests
     timeoutInMinutes: 360
     #pool:
     #  vmImage: 'ubuntu-18.04'

From f5d524d2fc55174106b2c9d967b3d66af14a5b3a Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Thu, 22 Oct 2020 14:44:47 -0700
Subject: [PATCH 21/39] Set up CI with Azure Pipelines

[skip ci]
---
 azure-pipelines.yml | 44 +-------------------------------------------
 1 file changed, 1 insertion(+), 43 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index c62e911..ef664a7 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -17,20 +17,6 @@ jobs:
       demands:
         - agent.name -equals gpu4 
 
-
-    #container:
-    #  image: adsbrainwestus2.azurecr.io/fastseq:dev-py3
-    #  endpoint: acr-westus2
-    
-    #container: adsbrainwestus2.azurecr.io/fastseq:dev-py3
-      
-    # strategy:
-    #   matrix:
-    #     PyTorch12-CUDA100:
-    #       python.version: '3.6'
-    #       cuda.version: '10.0'
-    #       pytorch.version: '0.9.0'
-
     steps:
         # Unfortunately nvidia's nvcc_linux-64=<version> seems to install 10.1 regardless?
         # Most of this complexity is a workaround to get the compiler toolchain to match the
@@ -56,33 +42,5 @@ jobs:
         python --version
       displayName: 'Setup environment python=$(python.version) pytorch=$(pytorch.version) cuda=$(cuda.version)'
 
-    # - script: |
-    #     source activate $(conda_env)
-    #     pip install --progress-bar=off torch==$(pytorch.version)
-    #     #-f https://download.pytorch.org/whl/torch_stable.html
-    #     a=$(python --version)
-    #     echo $a
-    #     pip install fastseq
-    #     #python -I basic_install_test.py
-    #   displayName: 'Install Fastseq'
-    # - script: |
-    #     sudo docker run --gpus all  --network=host --privileged -w '/root'  --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864  adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash
-    #     python --version
-    #     which nvcc
-    #     nvcc --version
-    #     which fastseq
-    #     ls
-    #   displayName: 'docker'
-  
-    # - script: |
-    #     source activate $(conda_env)
-    #     which python
-    #     python --version
-    #     which nvcc
-    #     nvcc --version
-    #     which fastseq
-    #     python -c "import torch; print('torch:', torch.__version__, torch)"
-    #     python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
-    #     #python -c "import deepspeed; print('fastseq:', fastseq.__version__)"
-    #   displayName: 'Show environment'
+
   

From 29aa0cdc51e2046e802a0a418090cce642aa93c8 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Thu, 22 Oct 2020 14:56:13 -0700
Subject: [PATCH 22/39] Update azure-pipelines.yml

---
 azure-pipelines.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index ef664a7..974eab7 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -1,7 +1,6 @@
 # Python package
 # Create and test a Python package on multiple Python versions.
 # Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
-# https://docs.microsoft.com/azure/devops/pipelines/languages/python
 
 trigger:
   paths:

From 6da9b087f75a5d996d7e54923bc86b6d23fda0f6 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Thu, 22 Oct 2020 15:03:40 -0700
Subject: [PATCH 23/39] Update azure-pipelines.yml

---
 azure-pipelines.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 974eab7..cf53b66 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -1,6 +1,5 @@
 # Python package
 # Create and test a Python package on multiple Python versions.
-# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
 
 trigger:
   paths:

From 2d1b6d3f343d1af6900ef924fca552da7c66fbcc Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Thu, 22 Oct 2020 15:09:32 -0700
Subject: [PATCH 24/39] Update README.md

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index b84972c..b2f66ce 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,4 @@
+[![Build Status](https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main)](https://dev.azure.com/adsbrain/t1mp2/_build/latest?definitionId=11&branchName=main)
 <h1 align="Center"> <p> FastSeq </p> </h1>
 
 # Introduction

From 651da923e97c12ee1aa84d903a921217ca07e432 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Thu, 22 Oct 2020 15:10:22 -0700
Subject: [PATCH 25/39] Update README.md

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index b2f66ce..d2684f6 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,5 @@
 [![Build Status](https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main)](https://dev.azure.com/adsbrain/t1mp2/_build/latest?definitionId=11&branchName=main)
+https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main
 <h1 align="Center"> <p> FastSeq </p> </h1>
 
 # Introduction

From 347f2b8da68a648c97a63d0fcbae0c52335ad4ee Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Thu, 22 Oct 2020 15:10:53 -0700
Subject: [PATCH 26/39] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d2684f6..472d9f3 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,5 @@
 [![Build Status](https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main)](https://dev.azure.com/adsbrain/t1mp2/_build/latest?definitionId=11&branchName=main)
-https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main
+(https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main)
 <h1 align="Center"> <p> FastSeq </p> </h1>
 
 # Introduction

From cce3d05618054717e8006797d4296afaaf816aa0 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Thu, 22 Oct 2020 15:13:00 -0700
Subject: [PATCH 27/39] Update README.md

---
 README.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/README.md b/README.md
index 472d9f3..b2f66ce 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,4 @@
 [![Build Status](https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main)](https://dev.azure.com/adsbrain/t1mp2/_build/latest?definitionId=11&branchName=main)
-(https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main)
 <h1 align="Center"> <p> FastSeq </p> </h1>
 
 # Introduction

From 57f08017576c06e3f3fbddfaf099d0d9201cddc6 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Thu, 22 Oct 2020 15:13:09 -0700
Subject: [PATCH 28/39] Update README.md


From c7db14769d63c33d6c0137a73fa59db884491a81 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Thu, 22 Oct 2020 15:14:05 -0700
Subject: [PATCH 29/39] Update README.md

---
 README.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b2f66ce..3177984 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,7 @@
-[![Build Status](https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main)](https://dev.azure.com/adsbrain/t1mp2/_build/latest?definitionId=11&branchName=main)
+[![Build Status](https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main)](https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main)
+
+
+
 <h1 align="Center"> <p> FastSeq </p> </h1>
 
 # Introduction

From 4226d310c7949ee575e9d6f1682fe23f59773840 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Thu, 22 Oct 2020 15:15:56 -0700
Subject: [PATCH 30/39] Update README.md

---
 README.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/README.md b/README.md
index 3177984..a6ea67d 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,5 @@
 [![Build Status](https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main)](https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main)
 
-
-
 <h1 align="Center"> <p> FastSeq </p> </h1>
 
 # Introduction

From 9069c928e89a907f690c954d633cc955088f4795 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Thu, 22 Oct 2020 15:19:01 -0700
Subject: [PATCH 31/39] Update README.md

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a6ea67d..f2bdc38 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,5 @@
-[![Build Status](https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main)](https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main)
+(https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main)
+[![Build Status](https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main)]
 
 <h1 align="Center"> <p> FastSeq </p> </h1>
 

From 50e9ea686c9a616c149313add97d975cd3c94aab Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Thu, 22 Oct 2020 15:19:25 -0700
Subject: [PATCH 32/39] Update README.md

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index f2bdc38..7195c8e 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,6 @@
-(https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main)
 [![Build Status](https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main)]
+(https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main)
+
 
 <h1 align="Center"> <p> FastSeq </p> </h1>
 

From f15428e4bad0f4f32db605cd3dcca5d9682163a8 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Thu, 22 Oct 2020 15:19:47 -0700
Subject: [PATCH 33/39] Update README.md

---
 README.md | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/README.md b/README.md
index 7195c8e..b84972c 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,3 @@
-[![Build Status](https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main)]
-(https://dev.azure.com/adsbrain/t1mp2/_apis/build/status/NickNickGo.fastseq%20(1)?branchName=main)
-
-
 <h1 align="Center"> <p> FastSeq </p> </h1>
 
 # Introduction

From b8b60b491230f9e08331e10fdcd8d373fe0b0c02 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Thu, 22 Oct 2020 15:19:54 -0700
Subject: [PATCH 34/39] Update README.md


From 719aef6f6b13763d13db4de42b805f741b20731c Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Thu, 22 Oct 2020 15:19:55 -0700
Subject: [PATCH 35/39] Update README.md


From bcf6fb4a8ab24785ac9966231e5833ea69478d11 Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Thu, 22 Oct 2020 16:01:53 -0700
Subject: [PATCH 36/39] Set up CI with Azure Pipelines

[skip ci]
---
 azure-pipelines.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index cf53b66..29a38ad 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -2,9 +2,7 @@
 # Create and test a Python package on multiple Python versions.
 
 trigger:
-  paths:
-    exclude:
-      - docs      
+- main
 jobs:
   - job: Fastseq_Tests
     timeoutInMinutes: 360

From d5ea0179a98e34cd43df36ee58ce9f5a73736fcf Mon Sep 17 00:00:00 2001
From: NickNickGo <66033489+NickNickGo@users.noreply.github.com>
Date: Fri, 23 Oct 2020 10:57:07 -0700
Subject: [PATCH 37/39] Update azure-pipelines.yml

---
 azure-pipelines.yml | 53 +++++++++++++++++++++------------------------
 1 file changed, 25 insertions(+), 28 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 29a38ad..1635a2e 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -1,10 +1,10 @@
-# Python package
-# Create and test a Python package on multiple Python versions.
-
+#
 trigger:
-- main
+  paths:
+    exclude:
+      - docs      
 jobs:
-  - job: Fastseq_Tests
+  - job: Fairseq_unittests
     timeoutInMinutes: 360
     #pool:
     #  vmImage: 'ubuntu-18.04'
@@ -12,31 +12,28 @@ jobs:
       name: default
       demands:
         - agent.name -equals gpu4 
-
     steps:
-        # Unfortunately nvidia's nvcc_linux-64=<version> seems to install 10.1 regardless?
-        # Most of this complexity is a workaround to get the compiler toolchain to match the
-        # cudatoolkit runtime
     - script: |
-        #conda create --force --yes -n $(conda_env) python=$(python.version) cudatoolkit=$(cuda.version)
-        #conda init bash
-        #conda activate $(conda_env)
-        #conda install -q --yes conda
-        #conda install -q --yes pip
-        #conda install -q --yes gxx_linux-64
-        #if [[ $(cuda.version) != "10.2" ]]; then conda install --yes -c conda-forge cudatoolkit-dev=$(cuda.version) ; fi
-        #echo "PATH=$PATH, LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
-        python --version
-        #which nvcc
-        #nvcc --version
-        #which fastseq
-        #bash tests/run_fairseq_tests.sh
+        #set up docker
         sudo docker run --gpus all  --network=host --privileged -w '/root'  --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864  adsbrainwestus2.azurecr.io/fastseq:dev-py3 /bin/bash
-        ls
+        #install fastseq
         pip install --editable .
-        bash tests/run_fairseq_tests.sh
+      
+        #show environment
+        which python
         python --version
-      displayName: 'Setup environment python=$(python.version) pytorch=$(pytorch.version) cuda=$(cuda.version)'
-
-
-  
+        which nvcc
+        nvcc --version
+        which fastseq
+        python -c "import torch; print('torch:', torch.__version__, torch)"
+        python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
+        python -c "import fastseq; print('fastseq:', fastseq.__version__)"
+        
+        #run unit tests
+        CUDA_VISIBLE_DEVICES=3 bash tests/run_fairseq_tests.sh
+        CUDA_VISIBLE_DEVICES=3 bash tests/run_transformers_tests.sh
+        
+        #run benchmarks 
+        #cd benchmarks/
+        #CUDA_VISIBLE_DEVICES=3 run_all_benchmarks.sh
+      displayName: 'setup environment and run fairseq unit tests'  

From b7f983144f3c2e89225c208dc8fcd3665e966f67 Mon Sep 17 00:00:00 2001
From: root <root@gpu4.fzppkdkor00e3cjx1piteeqqpe.xx.internal.cloudapp.net>
Date: Fri, 23 Oct 2020 19:36:24 +0000
Subject: [PATCH 38/39] junk

---
 junk | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 junk

diff --git a/junk b/junk
new file mode 100644
index 0000000..a941931
--- /dev/null
+++ b/junk
@@ -0,0 +1 @@
+junk

From 057d6f2e3bc50b46b54e75684436918771f0be18 Mon Sep 17 00:00:00 2001
From: root <root@gpu4.fzppkdkor00e3cjx1piteeqqpe.xx.internal.cloudapp.net>
Date: Fri, 23 Oct 2020 20:05:46 +0000
Subject: [PATCH 39/39] junk2

---
 fastseq/optimizer/fairseq/beam_search_optimizer_v1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fastseq/optimizer/fairseq/beam_search_optimizer_v1.py b/fastseq/optimizer/fairseq/beam_search_optimizer_v1.py
index 83adba4..a73b166 100644
--- a/fastseq/optimizer/fairseq/beam_search_optimizer_v1.py
+++ b/fastseq/optimizer/fairseq/beam_search_optimizer_v1.py
@@ -1,6 +1,6 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
-
+#
 """Apply the beam search optimizations to fairseq-v0.9.0"""
 
 import math