From 9acbacda80bff6de6fd2c1a28286e6b1a7ea76dd Mon Sep 17 00:00:00 2001 From: Charles Doutriaux Date: Wed, 29 May 2019 13:08:17 -0700 Subject: [PATCH 1/3] caching conda env for faster build times --- .circleci/config.yml | 80 ++++++++++++++++++----------- ci-support/checkout_merge_commit.sh | 29 ----------- 2 files changed, 50 insertions(+), 59 deletions(-) delete mode 100755 ci-support/checkout_merge_commit.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index a0fb2ac70..114be365a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,16 +1,6 @@ version: 2 -checkout: - post: - - ./ci-support/checkout_merge_commit.sh - aliases: - - &setup_miniconda - name: setup_miniconda - command: | - mkdir -p workspace - git clone -b validateNightly git@github.com:CDAT/cdat workspace/cdat - python workspace/cdat/scripts/install_miniconda.py -w $WORKDIR -p 'py3' - &get_testdata name: get_testdata @@ -24,30 +14,40 @@ aliases: CHANNELS: "-c cdat/label/nightly -c conda-forge" PKGS: "cdms2 cdat_info udunits2 testsrunner mesalib matplotlib image-compare genutil dv3d cdutil cdtime nbformat 'proj4<5' numpy ghostscript vtk-cdat" command: | - export PATH=$WORKDIR/miniconda/bin:$PATH - conda config --set always_yes yes --set changeps1 no - conda update -y -q conda - conda config --set anaconda_upload no - if [[ $PY_VER = "py2" ]]; then - conda create -q -n $PY_VER $CUSTOM_CHANNELS $CHANNELS $PKGS $TEMP_PKGS "python<3" - else - conda create -q -n $PY_VER $CUSTOM_CHANNELS $CHANNELS $PKGS $TEMP_PKGS "python=3.7" sphinxcontrib-websupport nbsphinx easydev $COVERAGE_PKGS - fi + if [ -d ${HOME}/miniconda ]; then + export PATH=$HOME/miniconda/bin:$PATH + echo "Found miniconda directory" + else + echo "miniconda directory not found, installing conda" + mkdir -p workspace + git clone -b validateNightly --depth 1 git@github.com:CDAT/cdat workspace/cdat + ls workspace/cdat + python workspace/cdat/scripts/install_miniconda.py -w $HOME -p 'py3.7' + export PATH=$HOME/miniconda/bin:$PATH + conda config --set always_yes yes --set changeps1 no + conda update -y -q conda + conda config --set anaconda_upload no + if [[ $PY_VER = "py2" ]]; then + conda create -q -n cdat $CUSTOM_CHANNELS $CHANNELS $PKGS $TEMP_PKGS "python<3" + else + conda create -q -n cdat $CUSTOM_CHANNELS $CHANNELS $PKGS $TEMP_PKGS "python=3.7" sphinxcontrib-websupport nbsphinx easydev $COVERAGE_PKGS + fi + fi - &setup_vcs name: setup_vcs command: | - export PATH=$WORKDIR/miniconda/bin:$PATH - source activate $PY_VER + export PATH=$HOME/miniconda/bin:$PATH + source activate cdat python setup.py install - &run_vcs_tests name: run_vcs_tests command: | - export PATH=$WORKDIR/miniconda/bin:$PATH + export PATH=$HOME/miniconda/bin:$PATH export CDAT_ANONYMOUS_LOG=False export CDAT_SETUP_PATH=${HOME} - source activate $PY_VER + source activate cdat # possibly turn off coverage first=$(git cherry master -v | head -n1) current=$(git log -n1 --pretty=oneline) @@ -58,7 +58,7 @@ aliases: echo "Running w/o coverage" ACTUAL_COVERAGE="" fi - python run_tests.py -n 4 -H -v2 --timeout=100000 --checkout-baseline --no-vtk-ui $ACTUAL_COVERAGE + python run_tests.py -H -v2 --timeout=100000 --checkout-baseline --no-vtk-ui $ACTUAL_COVERAGE RESULT=$? echo "**** $PY_VER test result: "${RESULT} if [[ $PY_VER = 'py3' ]]; then @@ -78,7 +78,7 @@ aliases: if [[ $CIRCLE_BRANCH != 'master' ]]; then exit 0 fi - export PATH=${HOME}/project/$WORKDIR/miniconda/bin:$PATH + export PATH=${HOME}/miniconda/bin:$PATH conda install conda-build anaconda-client conda config --set anaconda_upload no export CONDA_BLD_PATH=`pwd`/conda_build @@ -99,7 +99,7 @@ aliases: - &run_coveralls name: run_coveralls command: | - export PATH=$WORKDIR/miniconda/bin:$PATH + export PATH=$HOME/miniconda/bin:$PATH source activate $PY_VER # possibly turn off coverage first=$(git cherry master -v | head -n1) @@ -124,9 +124,14 @@ jobs: CUSTOM_CHANNELS: "" steps: - checkout - - run: *setup_miniconda + - restore_cache: + keys: + - macos_py2-2019-05-29 - run: *get_testdata - run: *create_conda_env + - save_cache: + key: macos_py2-2019-05-29 + paths: /Users/distiller/miniconda - run: *setup_vcs - run: *run_vcs_tests - store_artifacts: @@ -147,9 +152,14 @@ jobs: CUSTOM_CHANNELS: "" steps: - checkout - - run: *setup_miniconda + - restore_cache: + keys: + - macos_py3-2019-05-29 - run: *get_testdata - run: *create_conda_env + - save_cache: + key: macos_py3-2019-05-29 + paths: /Users/distiller/miniconda - run: *setup_vcs - run: *run_vcs_tests - store_artifacts: @@ -170,9 +180,14 @@ jobs: CUSTOM_CHANNELS: "" steps: - checkout - - run: *setup_miniconda + - restore_cache: + keys: + - linux_py2-2019-05-29 - run: *get_testdata - run: *create_conda_env + - save_cache: + key: linux_py2-2019-05-29 + paths: /home/circleci/miniconda - run: *setup_vcs - run: *run_vcs_tests - store_artifacts: @@ -195,9 +210,14 @@ jobs: CUSTOM_CHANNELS: "" steps: - checkout - - run: *setup_miniconda + - restore_cache: + keys: + - linux_py3-2019-05-29 - run: *get_testdata - run: *create_conda_env + - save_cache: + key: linux_py3-2019-05-29 + paths: /home/circleci/miniconda - run: *setup_vcs - run: *run_vcs_tests - run: *run_coveralls diff --git a/ci-support/checkout_merge_commit.sh b/ci-support/checkout_merge_commit.sh deleted file mode 100755 index 0d82d172e..000000000 --- a/ci-support/checkout_merge_commit.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - - -# Add `master` branch to the update list. -# Otherwise CircleCI will give us a cached one. -FETCH_REFS="+master:master" - -# Update PR refs for testing. -if [[ -n "${CIRCLE_PR_NUMBER}" ]] -then - FETCH_REFS="${FETCH_REFS} +refs/pull/${CIRCLE_PR_NUMBER}/head:pr/${CIRCLE_PR_NUMBER}/head" - FETCH_REFS="${FETCH_REFS} +refs/pull/${CIRCLE_PR_NUMBER}/merge:pr/${CIRCLE_PR_NUMBER}/merge" -fi - -# Retrieve the refs. -git fetch -u origin ${FETCH_REFS} - -# Checkout the PR merge ref. -if [[ -n "${CIRCLE_PR_NUMBER}" ]] -then - git checkout -qf "pr/${CIRCLE_PR_NUMBER}/merge" -fi - -# Check for merge conflicts. -if [[ -n "${CIRCLE_PR_NUMBER}" ]] -then - git branch --merged | grep master > /dev/null - git branch --merged | grep "pr/${CIRCLE_PR_NUMBER}/head" > /dev/null -fi From 0cdf0efa32605de739e9833b3c335d76f5ede6ba Mon Sep 17 00:00:00 2001 From: Charles Doutriaux Date: Wed, 29 May 2019 13:09:44 -0700 Subject: [PATCH 2/3] not sure why ls is here for --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 114be365a..e51f37df6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,6 @@ aliases: - &get_testdata name: get_testdata command: | - ls -l workspace git clone git://github.com/cdat/uvcdat-testdata - &create_conda_env From cf9a0f3233239d9c25661e2e75f1f4770a6ec44c Mon Sep 17 00:00:00 2001 From: Charles Doutriaux Date: Wed, 29 May 2019 13:22:26 -0700 Subject: [PATCH 3/3] forgot on spot for new conda env name --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e51f37df6..6b9b1158b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -99,7 +99,7 @@ aliases: name: run_coveralls command: | export PATH=$HOME/miniconda/bin:$PATH - source activate $PY_VER + source activate cdat # possibly turn off coverage first=$(git cherry master -v | head -n1) current=$(git log -n1 --pretty=oneline)