The container images provided by Travis only support Ubuntu 14.04,
however, Travis has ability to run docker, which allows the build
script to use arbitrary OS images. This takes advantage of that to
convert the build over to Ubuntu 16.04 and 18.04

This is using the official Ubuntu provided images and installing
extra build deps required, as we previously did with Travis container
images. With the switch to Docker though, this can be improved, by
building custom Docker images with all the deps pre-installed which
will cut down build time. This can be driven from the package lists
in libvirt-jenkins-ci repo, to remove the duplication. This work
for future improvement though, this just does the minimal conversion
to match what we already do, but with newer distro.

Signed-off-by: Daniel P. Berrangé <berrange@redhat.com>
---
 .travis.yml | 217 +++++++++++++++++++++++++++++-----------------------
 1 file changed, 121 insertions(+), 96 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index c71411dcae..1e2f946352 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,6 +1,5 @@
-sudo: false
-language: c
-cache: ccache
+sudo: required
+language: generic
 
 branches:
   except:
@@ -8,17 +7,41 @@ branches:
 
 matrix:
   include:
-    - compiler: gcc
-      dist: trusty
+    - services:
+        - docker
       env:
-        - PYTHON=$(which python2)
-        - DISTCHECK_CONFIGURE_FLAGS="--with-init-script=upstart"
-    - compiler: clang
-      dist: trusty
-      env:
-        - PYTHON=$(which python3)
+        - IMAGE_NAME=ubuntu:18.04
+        - PYTHON=python3
         - DISTCHECK_CONFIGURE_FLAGS="--with-init-script=systemd"
+      before_script:
+        - docker run
+            --privileged
+            -v $(pwd):/build
+            -w /build
+            -e VIR_TEST_DEBUG="$VIR_TEST_DEBUG"
+            -e PACKAGES="$PACKAGES"
+            -e DISTCHECK_FLAGS="$DISTCHECK_CONFIGURE_FLAGS"
+            $IMAGE_NAME
+            /bin/sh -c "$DOCKER_CMD"
+    - services:
+        - docker
+      env:
+        - IMAGE_NAME=ubuntu:16.04
+        - PYTHON=python2
+        - DISTCHECK_CONFIGURE_FLAGS="--with-init-script=upstart"
+      before_script:
+        - docker pull berrange/test
+        - docker run
+            --privileged
+            -v $(pwd):/build
+            -w /build
+            -e VIR_TEST_DEBUG="$VIR_TEST_DEBUG"
+            -e PACKAGES="$PACKAGES"
+            -e DISTCHECK_FLAGS="$DISTCHECK_CONFIGURE_FLAGS"
+            $IMAGE_NAME
+            /bin/sh -c "$DOCKER_CMD"
     - compiler: clang
+      language: c
       os: osx
       env:
         - PATH="/usr/local/opt/gettext/bin:/usr/local/opt/ccache/libexec:/usr/local/opt/rpcgen/bin:$PATH"
@@ -27,80 +50,18 @@ matrix:
         - brew update
         - brew upgrade
         - brew install python ccache rpcgen yajl
+      before_script:
+        - ./autogen.sh --prefix=$(pwd)/install-root
       script:
         # We can't run make distcheck/syntax-check because they
         # fail on macOS, but doing 'install' and 'dist' gives us
         # some useful coverage
         - make -j3 && make -j3 install && make -j3 dist
+      after_failure:
+        - echo '=== LOG FILE(S) START ==='
+        - find -name test-suite.log | xargs cat
+        - echo '=== LOG FILE(S) END ==='
 
-addons:
-  apt:
-    # Please keep this list sorted alphabetically
-    packages:
-      - augeas-tools
-      - autoconf
-      - automake
-      - autopoint
-      - bash-completion
-      - ccache
-      - dnsmasq-base
-      - dwarves
-      - ebtables
-      - gcc
-      - gettext
-      - glusterfs-client
-      - libacl1-dev
-      - libapparmor-dev
-      - libattr1-dev
-      - libaudit-dev
-      - libavahi-client-dev
-      - libblkid-dev
-      - libc6-dev
-      - libcap-ng-dev
-      - libc-dev-bin
-      - libcurl4-gnutls-dev
-      - libdbus-1-dev
-      - libdevmapper-dev
-      - libfuse-dev
-      - libgnutls-dev
-      - libnetcf-dev
-      - libnl-3-dev
-      - libnl-route-3-dev
-      - libnuma-dev
-      - libopenwsman-dev
-      - libparted-dev
-      - libpcap-dev
-      - libpciaccess-dev
-      - librbd-dev
-      - libreadline-dev
-      - libsanlock-dev
-      - libsasl2-dev
-      - libselinux1-dev
-      - libssh2-1-dev
-      - libssh-dev
-      - libtirpc-dev
-      - libtool
-      - libudev-dev
-      - libxen-dev
-      - libxml2-dev
-      - libxml2-utils
-      - libyajl-dev
-      - lvm2
-      - make
-#     - nfs-common    # broken on trusty since build #807 (2018-01-09)
-      - open-iscsi
-      - parted
-      - patch
-      - perl
-      - pkgconf
-      - policykit-1
-      - qemu-utils
-      - radvd
-      - scrub
-      - sheepdog
-      - systemtap-sdt-dev
-      - xsltproc
-      - zfs-fuse
 
 git:
   submodules: true
@@ -108,23 +69,87 @@ git:
 env:
   global:
     - VIR_TEST_DEBUG=1
-
-before_script:
-  - ./autogen.sh --prefix=$(pwd)/install-root
-
-script:
-  - make -j3 && make -j3 syntax-check && make -j3 distcheck DISTCHECK_CONFIGURE_FLAGS=$DISTCHECK_CONFIGURE_FLAGS
-
-after_failure:
-  - echo '============================================================================'
-  - 'ver=$(sed -n "s/AC_INIT(\[libvirt\], \[\([0-9.]*\)\],.*/\1/p" configure.ac);
-    if [ -f $(pwd)/tests/test-suite.log ]; then
-        cat $(pwd)/tests/test-suite.log;
-    elif [ -f libvirt-$ver/_build/tests/test-suite.log ]; then
-        cat libvirt-$ver/_build/tests/test-suite.log;
-    else
-        echo "=== NO LOG FILE FOUND ===";
-    fi'
+    - DOCKER_CMD="
+        apt-get update &&
+        apt-get install -y \$PACKAGES &&
+        ./autogen.sh --prefix=$(pwd)/install-root &&
+        make -j3 &&
+        make -j3 syntax-check &&
+        make -j3 distcheck DISTCHECK_CONFIGURE_FLAGS=\"\$DISTCHECK_FLAGS\" ||
+        (
+          echo '=== LOG FILE(S) START ==='
+          find -name test-suite.log | xargs cat
+          echo '=== LOG FILE(S) END ==='
+          exit 1
+        )
+      "
+    # Please keep this list sorted alphabetically
+    - PACKAGES="
+        augeas-tools
+        autoconf
+        automake
+        autopoint
+        bash-completion
+        ccache
+        dnsmasq-base
+        dwarves
+        ebtables
+        gcc
+        gettext
+        git
+        glusterfs-client
+        libacl1-dev
+        libapparmor-dev
+        libattr1-dev
+        libaudit-dev
+        libavahi-client-dev
+        libblkid-dev
+        libc6-dev
+        libcap-ng-dev
+        libc-dev-bin
+        libdbus-1-dev
+        libdevmapper-dev
+        libfuse-dev
+        libgnutls28-dev
+        libnetcf-dev
+        libnl-3-dev
+        libnl-route-3-dev
+        libnuma-dev
+        libopenwsman-dev
+        libparted-dev
+        libpcap-dev
+        libpciaccess-dev
+        librbd-dev
+        libreadline-dev
+        libsanlock-dev
+        libsasl2-dev
+        libselinux1-dev
+        libssh2-1-dev
+        libssh-dev
+        libtirpc-dev
+        libtool
+        libudev-dev
+        libxen-dev
+        libxml2-dev
+        libxml2-utils
+        libyajl-dev
+        lvm2
+        make
+        nfs-common
+        open-iscsi
+        parted
+        patch
+        perl
+        pkgconf
+        policykit-1
+        qemu-utils
+        radvd
+        scrub
+        sheepdog
+        systemtap-sdt-dev
+        xsltproc
+        zfs-fuse
+      "
 
 notifications:
   irc:
-- 
2.17.0

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list

On Wed, 2018-06-06 at 17:24 +0100, Daniel P. Berrangé wrote:
> The container images provided by Travis only support Ubuntu 14.04,
> however, Travis has ability to run docker, which allows the build
> script to use arbitrary OS images. This takes advantage of that to
> convert the build over to Ubuntu 16.04 and 18.04
> 
> This is using the official Ubuntu provided images and installing
> extra build deps required, as we previously did with Travis container
> images.

As mentioned during the review of v1, this bit:

> With the switch to Docker though, this can be improved, by
> building custom Docker images with all the deps pre-installed which
> will cut down build time. This can be driven from the package lists
> in libvirt-jenkins-ci repo, to remove the duplication. This work
> for future improvement though, this just does the minimal conversion
> to match what we already do, but with newer distro.

doesn't belong to the commit message in my opinion.

[...]
> +    - services:
> +        - docker
>        env:
> +        - IMAGE_NAME=ubuntu:18.04

This can be just IMAGE or even OS.

IMAGE is probably better because it sorts very early, thus
ensuring it will be displayed prominently in the build summary.

> +        - PYTHON=python3

Since we're not actually passing this to configure after all (and
neither could we, due to the fact that $PYTHON needs to be a full
path and we set the variable outside of the container where Python
could be installed in a different location than inside it) it
doesn't really make a lot of sense to have this.

Let's just get rid of it and rely on configure picking up python3
on modern operating systems and python2 otherwise, same way we
handle the issue on CentOS CI.

[...]
> +      before_script:

I thought we agreed on using script instead of before_script here?

> +        - docker run
> +            --privileged
> +            -v $(pwd):/build
> +            -w /build
> +            -e VIR_TEST_DEBUG="$VIR_TEST_DEBUG"
> +            -e PACKAGES="$PACKAGES"
> +            -e DISTCHECK_FLAGS="$DISTCHECK_CONFIGURE_FLAGS"

We should keep the names of variables consistent between the host
and container environment.

> +            $IMAGE_NAME

$IMAGE_NAME (or rather $IMAGE) should be quoted.

> +            /bin/sh -c "$DOCKER_CMD"

Something I didn't spot the first time around: using

  /bin/sh -xc "$DOCKER_CMD"

here will be very helpful to figure out exactly what is happening
during the build, so please add that.

[...]
> +    - services:
> +        - docker
> +      env:
> +        - IMAGE_NAME=ubuntu:16.04
> +        - PYTHON=python2
> +        - DISTCHECK_CONFIGURE_FLAGS="--with-init-script=upstart"
> +      before_script:
> +        - docker pull berrange/test

You forgot to delete this one :)

[...]
>          - brew update
>          - brew upgrade
>          - brew install python ccache rpcgen yajl
> +      before_script:
> +        - ./autogen.sh --prefix=$(pwd)/install-root

Another bit I missed the first time around (sorry): before these
changes, we took advantage of the before_script/script split to
avoid typing out the autogen.sh call twice, but that's no longer
relevant now, so you can simplify things by folding it into script.

[...]
>  env:
>    global:
>      - VIR_TEST_DEBUG=1
> +    - DOCKER_CMD="
> +        apt-get update &&
> +        apt-get install -y \$PACKAGES &&
> +        ./autogen.sh --prefix=$(pwd)/install-root &&

You didn't escape '$(pwd)' correctly here, so it will be expanded
in the host rather than inside the container.

That said, we don't actually call 'make install' for builds
happening inside Docker since 'make distcheck' works there, so you
might as well drop --prefix altogether.

-- 
Andrea Bolognani / Red Hat / Virtualization

--
libvir-list mailing list
libvir-list@redhat.com
https://www.redhat.com/mailman/listinfo/libvir-list