From 59273a0c53d6b62b19e0765a5e8837f31686f7d5 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Fri, 1 Mar 2019 10:01:09 +0100 Subject: [PATCH] semaphoreci: Run subset of autopkgtests in LXC (#11814) Run build/test in LXC for now, as full nested QEMU is too brittle right now: https://github.com/semaphoreci/semaphore/issues/37 But this at least runs some tests. It ensures that systemd generally works in containers, as well as provides some backup results if the main Ubuntu CI is down. --- semaphoreci/gcc-compilation.sh | 43 +++++++++++++++++++++--- semaphoreci/setup.sh | 60 +++++++++++++++++++++------------- 2 files changed, 76 insertions(+), 27 deletions(-) diff --git a/semaphoreci/gcc-compilation.sh b/semaphoreci/gcc-compilation.sh index 0e0c83788a..8755b850b6 100755 --- a/semaphoreci/gcc-compilation.sh +++ b/semaphoreci/gcc-compilation.sh @@ -2,7 +2,42 @@ set -ex -meson build -Dtests=unsafe -Dsplit-usr=true -Dslow-tests=true -ninja -C build -ninja -C build test -DESTDIR=/var/tmp/inst1 ninja -C build install +# keep this in sync with setup.sh +CONTAINER=${RELEASE:-buster}-${ARCH:-amd64} +AUTOPKGTESTDIR=${SEMAPHORE_CACHE_DIR:-/tmp}/autopkgtest +# semaphore cannot expose these, but useful for interactive/local runs +ARTIFACTS_DIR=/tmp/artifacts + +# add current debian/ packaging +git fetch --depth=1 https://salsa.debian.org/systemd-team/systemd.git master +git checkout FETCH_HEAD debian + +# craft changelog +UPSTREAM_VER=$(git describe | sed 's/^v//') +cat << EOF > debian/changelog.new +systemd (${UPSTREAM_VER}-0) UNRELEASED; urgency=low + + * Automatic build for upstream test + + -- systemd test $(date -R) + +EOF +cat debian/changelog >> debian/changelog.new +mv debian/changelog.new debian/changelog + +# clean out patches +rm -rf debian/patches +# disable autopkgtests which are not for upstream +sed -i '/# NOUPSTREAM/ q' debian/tests/control +# enable more unit tests +sed -i '/^CONFFLAGS =/ s/=/= -Dtests=unsafe -Dsplit-usr=true -Dslow-tests=true /' debian/rules +# no orig tarball +echo '1.0' > debian/source/format + +# build source package +dpkg-buildpackage -S -I -I$(basename "$SEMAPHORE_CACHE_DIR") -d -us -uc -nc + +# now build the package and run the tests +rm -rf "$ARTIFACTS_DIR" +# autopkgtest exits with 2 for "some tests skipped", accept that +$AUTOPKGTESTDIR/runner/autopkgtest --apt-upgrade --env DEB_BUILD_OPTIONS=noudeb --env TEST_UPSTREAM=1 ../systemd_*.dsc -o "$ARTIFACTS_DIR" -- lxc -s $CONTAINER || [ $? -eq 2 ] diff --git a/semaphoreci/setup.sh b/semaphoreci/setup.sh index d9a7b50685..b8cdba8033 100755 --- a/semaphoreci/setup.sh +++ b/semaphoreci/setup.sh @@ -2,26 +2,40 @@ set -ex -sudo add-apt-repository ppa:upstream-systemd-ci/systemd-ci -y -sudo rm -rf /etc/apt/sources.list.d/beineri* /etc/apt/sources.list.d/google-chrome* /etc/apt/sources.list.d/heroku* /etc/apt/sources.list.d/mongodb* /etc/apt/sources.list.d/webupd8team* /etc/apt/sources.list.d/rwky* /etc/apt/sources.list.d/rethinkdb* /etc/apt/sources.list.d/cassandra* /etc/apt/sources.list.d/cwchien* /etc/apt/sources.list.d/rabbitmq* /etc/apt/sources.list.d/docker* /home/runner/{.npm,.phpbrew,.phpunit,.kerl,.kiex,.lein,.nvm,.npm,.phpbrew,.rbenv} -sudo bash -c "echo 'deb-src http://de.archive.ubuntu.com/ubuntu/ xenial main restricted universe multiverse' >>/etc/apt/sources.list" -sudo apt-get update -qq -sudo apt-get build-dep systemd -y -sudo apt-get install --force-yes -y util-linux libmount-dev libblkid-dev liblzma-dev libqrencode-dev libmicrohttpd-dev iptables-dev liblz4-dev libcurl4-gnutls-dev unifont clang-3.6 libasan0 itstool kbd cryptsetup-bin net-tools isc-dhcp-client iputils-ping strace qemu-system-x86 linux-image-virtual mount libgpg-error-dev libxkbcommon-dev python-lxml python3-lxml python3-pip libcap-dev -# curl -s https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - -# sudo add-apt-repository -y 'deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty main' -# sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test -sudo apt-get update -sudo apt-get install --force-yes -y gettext python3-evdev python3-pyparsing libmount-dev -# sudo apt-get install -y clang-6.0 -sudo sh -c 'echo 01010101010101010101010101010101 >/etc/machine-id' -sudo mount -t tmpfs none /tmp -test -d /run/mount || sudo mkdir /run/mount -sudo adduser --system --no-create-home nfsnobody -sudo rm -f /etc/mtab -git clone https://github.com/ninja-build/ninja -cd ninja -./configure.py --bootstrap -sudo cp ninja /usr/bin/ -cd .. -pip3 install --user 'meson == 0.46.1' +# default to Debian testing +DISTRO=${DISTRO:-debian} +RELEASE=${RELEASE:-buster} +ARCH=${ARCH:-amd64} +CONTAINER=${RELEASE}-${ARCH} + +# remove semaphore repos, some of them don't work and cause error messages +sudo rm -f /etc/apt/sources.list.d/* + +# enable backports for latest LXC +echo 'deb http://archive.ubuntu.com/ubuntu xenial-backports main restricted universe multiverse' | sudo tee -a /etc/apt/sources.list.d/backports.list +sudo apt-get -q update +sudo apt-get install -y -t xenial-backports lxc +sudo apt-get install -y python3-debian git dpkg-dev fakeroot + +AUTOPKGTESTDIR=${SEMAPHORE_CACHE_DIR:-/tmp}/autopkgtest +[ -d $AUTOPKGTESTDIR ] || git clone --quiet --depth=1 https://salsa.debian.org/ci-team/autopkgtest.git "$AUTOPKGTESTDIR" + +# TODO: cache container image (though downloading/building it takes < 1 min) +# create autopkgtest LXC image +sudo lxc-create -n $CONTAINER -t download -- -d $DISTRO -r $RELEASE -a $ARCH + +# unconfine the container, otherwise some tests fail +echo 'lxc.apparmor.profile = unconfined' | sudo tee -a /var/lib/lxc/$CONTAINER/config + +sudo lxc-start -n $CONTAINER + +# enable source repositories so that apt-get build-dep works +sudo lxc-attach -n $CONTAINER -- sh -ex <> /etc/apt/sources.list.d/sources.list +# wait until online +while [ -z "\$(ip route list 0/0)" ]; do sleep 1; done +apt-get -q update +apt-get -y dist-upgrade +apt-get install -y eatmydata +EOF +sudo lxc-stop -n $CONTAINER