Ubuntu/devel

.travis.yml

@@ -3,18 +3,25 @@ dist: bionic
 
 # We use two different caching strategies.  The default is to cache pip
 # packages (as most of our jobs use pip packages), which is configured here.
-# For the integration tests, we instead want to cache the lxd images.  The
-# directory in which the images are stored (/var/snap/lxd/common/lxd/images/)
-# is not readable/writeable by the default user (which is a requirement for
-# caching), so we instead cache the `lxd_images/` directory.  We move lxd
-# images out of there before we run tests and back in once tests are complete.
-# We _move_ the images out and only copy the most recent lxd image back into
-# the cache, to avoid our cache growing without bound.  (We only need the most
-# recent lxd image because the integration tests only use a single image.)
+# For the integration tests, we instead want to cache the lxd images and
+# package build schroot.
 #
 # We cache the lxd images because this saves a few seconds in the general
 # case, but provides substantial speed-ups when cloud-images.ubuntu.com, the
-# source of the images, is under heavy load.
+# source of the images, is under heavy load.  The directory in which the lxd
+# images are stored (/var/snap/lxd/common/lxd/images/) is not
+# readable/writeable by the default user (which is a requirement for caching),
+# so we instead cache the `lxd_images/` directory.  We move lxd images out of
+# there before we run tests and back in once tests are complete.  We _move_ the
+# images out and only copy the most recent lxd image back into the cache, to
+# avoid our cache growing without bound.  (We only need the most recent lxd
+# image because the integration tests only use a single image.)
+#
+# We cache the package build schroot because it saves 2-3 minutes per build.
+# Without caching, we have to perform a debootstrap for every build.  We update
+# the schroot before storing it back in the cache, to ensure that we aren't
+# just using an increasingly-old schroot as time passes.  The cached schroot is
+# stored as a tarball, to preserve permissions/ownership.
 cache: pip
 
 install:
@@ -37,20 +44,18 @@ matrix:
           cache:
               - directories:
                   - lxd_images
+                  - chroots
           before_cache:
               - |
                   # Find the most recent image file
                   latest_file="$(sudo ls -Art /var/snap/lxd/common/lxd/images/ | tail -n 1)"
                   # This might be <hash>.rootfs or <hash>, normalise
                   latest_file="$(basename $latest_file .rootfs)"
                   # Find all files with that prefix and copy them to our cache dir
-                  sudo find /var/snap/lxd/common/lxd/images/ -name $latest_file* -print -exec cp {} lxd_images/ \;
+                  sudo find /var/snap/lxd/common/lxd/images/ -name $latest_file* -print -exec cp {} "$TRAVIS_BUILD_DIR/lxd_images/" \;
           install:
             - git fetch --unshallow
-            - sudo apt-get build-dep -y cloud-init
-            - sudo apt-get install -y --install-recommends sbuild ubuntu-dev-tools fakeroot tox
-            # These are build deps but not pulled in by the build-dep call above
-            - sudo apt-get install -y --install-recommends dh-systemd python3-coverage python3-pytest python3-pytest-cov
+            - sudo apt-get install -y --install-recommends sbuild ubuntu-dev-tools fakeroot tox debhelper
             - pip install .
             - pip install tox
             # bionic has lxd from deb installed, remove it first to ensure
@@ -61,15 +66,56 @@ matrix:
             - sudo lxd init --auto
             - sudo mkdir --mode=1777 -p /var/snap/lxd/common/consoles
             # Move any cached lxd images into lxd's image dir
-            - sudo find lxd_images/ -type f -print -exec mv {} /var/snap/lxd/common/lxd/images/ \;
+            - sudo find "$TRAVIS_BUILD_DIR/lxd_images/" -type f -print -exec mv {} /var/snap/lxd/common/lxd/images/ \;
             - sudo usermod -a -G lxd $USER
             - sudo sbuild-adduser $USER
             - cp /usr/share/doc/sbuild/examples/example.sbuildrc /home/$USER/.sbuildrc
           script:
             # Ubuntu LTS: Build
-            - ./packages/bddeb -S
-            # Use this to get a new shell where we're in the sbuild group
-            - sudo -E su $USER -c 'mk-sbuild xenial'
+            - ./packages/bddeb -S -d --release xenial
+            - |
+                needs_caching=false
+                if [ -e "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" ]; then
+                    # If we have a cached chroot, move it into place
+                    sudo mkdir -p /var/lib/schroot/chroots/xenial-amd64
+                    sudo tar --sparse --xattrs --preserve-permissions --numeric-owner -xf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64
+                    # Write its configuration
+                    cat > sbuild-xenial-amd64 << EOM
+                [xenial-amd64]
+                description=xenial-amd64
+                groups=sbuild,root,admin
+                root-groups=sbuild,root,admin
+                # Uncomment these lines to allow members of these groups to access
+                # the -source chroots directly (useful for automated updates, etc).
+                #source-root-users=sbuild,root,admin
+                #source-root-groups=sbuild,root,admin
+                type=directory
+                profile=sbuild
+                union-type=overlay
+                directory=/var/lib/schroot/chroots/xenial-amd64
+                EOM
+                    sudo mv sbuild-xenial-amd64 /etc/schroot/chroot.d/
+                    sudo chown root /etc/schroot/chroot.d/sbuild-xenial-amd64
+                    # And ensure it's up-to-date.
+                    before_pkgs="$(sudo schroot -c source:xenial-amd64 -d / dpkg -l | sha256sum)"
+                    sudo schroot -c source:xenial-amd64 -d / -- sh -c "apt-get update && apt-get -qqy upgrade"
+                    after_pkgs=$(sudo schroot -c source:xenial-amd64 -d / dpkg -l | sha256sum)
+                    if [ "$before_pkgs" != "$after_pkgs" ]; then
+                        needs_caching=true
+                    fi
+                else
+                    # Otherwise, create the chroot
+                    sudo -E su $USER -c 'mk-sbuild xenial'
+                    needs_caching=true
+                fi
+                # If there are changes to the schroot (or it's entirely new),
+                # tar up the schroot (to preserve ownership/permissions) and
+                # move it into the cached dir; no need to compress it because
+                # Travis will do that anyway
+                if [ "$needs_caching" = "true" ]; then
+                    sudo tar --sparse --xattrs --xattrs-include=* -cf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64 .
+                fi
+            # Use sudo to get a new shell where we're in the sbuild group
             - sudo -E su $USER -c 'sbuild --nolog --no-run-lintian --verbose --dist=xenial cloud-init_*.dsc'
             # Ubuntu LTS: Integration
             - sg lxd -c 'tox -e citest -- run --verbose --preserve-data --data-dir results --os-name xenial --test modules/apt_configure_sources_list.yaml --test modules/ntp_servers --test modules/set_password_list --test modules/user_groups --deb cloud-init_*_all.deb'