semaphoreci: Run subset of autopkgtests in LXC (#11814)

Run build/test in LXC for now, as full nested QEMU is too brittle right
now: https://github.com/semaphoreci/semaphore/issues/37
But this at least runs some tests. It ensures that systemd generally
works in containers, as well as provides some backup results if the main
Ubuntu CI is down.

semaphoreci/gcc-compilation.sh

@@ -2,7 +2,42 @@
 
 set -ex
 
-meson build -Dtests=unsafe -Dsplit-usr=true -Dslow-tests=true
-ninja -C build
-ninja -C build test
-DESTDIR=/var/tmp/inst1 ninja -C build install
+# keep this in sync with setup.sh
+CONTAINER=${RELEASE:-buster}-${ARCH:-amd64}
+AUTOPKGTESTDIR=${SEMAPHORE_CACHE_DIR:-/tmp}/autopkgtest
+# semaphore cannot expose these, but useful for interactive/local runs
+ARTIFACTS_DIR=/tmp/artifacts
+
+# add current debian/ packaging
+git fetch --depth=1 https://salsa.debian.org/systemd-team/systemd.git master
+git checkout FETCH_HEAD debian
+
+# craft changelog
+UPSTREAM_VER=$(git describe | sed 's/^v//')
+cat << EOF > debian/changelog.new
+systemd (${UPSTREAM_VER}-0) UNRELEASED; urgency=low
+
+  * Automatic build for upstream test
+
+ -- systemd test <pkg-systemd-maintainers@lists.alioth.debian.org>  $(date -R)
+
+EOF
+cat debian/changelog >> debian/changelog.new
+mv debian/changelog.new debian/changelog
+
+# clean out patches
+rm -rf debian/patches
+# disable autopkgtests which are not for upstream
+sed -i '/# NOUPSTREAM/ q' debian/tests/control
+# enable more unit tests
+sed -i '/^CONFFLAGS =/ s/=/= -Dtests=unsafe -Dsplit-usr=true -Dslow-tests=true /' debian/rules
+# no orig tarball
+echo '1.0' > debian/source/format
+
+# build source package
+dpkg-buildpackage -S -I -I$(basename "$SEMAPHORE_CACHE_DIR") -d -us -uc -nc
+
+# now build the package and run the tests
+rm -rf "$ARTIFACTS_DIR"
+# autopkgtest exits with 2 for "some tests skipped", accept that
+$AUTOPKGTESTDIR/runner/autopkgtest --apt-upgrade --env DEB_BUILD_OPTIONS=noudeb --env TEST_UPSTREAM=1 ../systemd_*.dsc -o "$ARTIFACTS_DIR" -- lxc -s $CONTAINER || [ $? -eq 2 ]

semaphoreci/setup.sh

@@ -2,26 +2,40 @@
 
 set -ex
 
-sudo add-apt-repository ppa:upstream-systemd-ci/systemd-ci -y
-sudo rm -rf /etc/apt/sources.list.d/beineri* /etc/apt/sources.list.d/google-chrome* /etc/apt/sources.list.d/heroku* /etc/apt/sources.list.d/mongodb* /etc/apt/sources.list.d/webupd8team* /etc/apt/sources.list.d/rwky* /etc/apt/sources.list.d/rethinkdb* /etc/apt/sources.list.d/cassandra* /etc/apt/sources.list.d/cwchien* /etc/apt/sources.list.d/rabbitmq* /etc/apt/sources.list.d/docker* /home/runner/{.npm,.phpbrew,.phpunit,.kerl,.kiex,.lein,.nvm,.npm,.phpbrew,.rbenv}
-sudo bash -c "echo 'deb-src http://de.archive.ubuntu.com/ubuntu/ xenial main restricted universe multiverse' >>/etc/apt/sources.list"
-sudo apt-get update -qq
-sudo apt-get build-dep systemd -y
-sudo apt-get install --force-yes -y util-linux libmount-dev libblkid-dev liblzma-dev libqrencode-dev libmicrohttpd-dev iptables-dev liblz4-dev libcurl4-gnutls-dev unifont clang-3.6 libasan0 itstool kbd cryptsetup-bin net-tools isc-dhcp-client iputils-ping strace qemu-system-x86 linux-image-virtual mount libgpg-error-dev libxkbcommon-dev python-lxml python3-lxml python3-pip libcap-dev
-# curl -s https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
-# sudo add-apt-repository -y 'deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty main'
-# sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
-sudo apt-get update
-sudo apt-get install --force-yes -y gettext python3-evdev python3-pyparsing libmount-dev
-# sudo apt-get install -y clang-6.0
-sudo sh -c 'echo 01010101010101010101010101010101 >/etc/machine-id'
-sudo mount -t tmpfs none /tmp
-test -d /run/mount || sudo mkdir /run/mount
-sudo adduser --system --no-create-home nfsnobody
-sudo rm -f /etc/mtab
-git clone https://github.com/ninja-build/ninja
-cd ninja
-./configure.py --bootstrap
-sudo cp ninja /usr/bin/
-cd ..
-pip3 install --user 'meson == 0.46.1'
+# default to Debian testing
+DISTRO=${DISTRO:-debian}
+RELEASE=${RELEASE:-buster}
+ARCH=${ARCH:-amd64}
+CONTAINER=${RELEASE}-${ARCH}
+
+# remove semaphore repos, some of them don't work and cause error messages
+sudo rm -f /etc/apt/sources.list.d/*
+
+# enable backports for latest LXC
+echo 'deb http://archive.ubuntu.com/ubuntu xenial-backports main restricted universe multiverse' | sudo tee -a /etc/apt/sources.list.d/backports.list
+sudo apt-get -q update
+sudo apt-get install -y -t xenial-backports lxc
+sudo apt-get install -y python3-debian git dpkg-dev fakeroot
+
+AUTOPKGTESTDIR=${SEMAPHORE_CACHE_DIR:-/tmp}/autopkgtest
+[ -d $AUTOPKGTESTDIR ] || git clone --quiet --depth=1 https://salsa.debian.org/ci-team/autopkgtest.git "$AUTOPKGTESTDIR"
+
+# TODO: cache container image (though downloading/building it takes < 1 min)
+# create autopkgtest LXC image
+sudo lxc-create -n $CONTAINER -t download -- -d $DISTRO -r $RELEASE -a $ARCH
+
+# unconfine the container, otherwise some tests fail
+echo 'lxc.apparmor.profile = unconfined' | sudo tee -a /var/lib/lxc/$CONTAINER/config
+
+sudo lxc-start -n $CONTAINER
+
+# enable source repositories so that apt-get build-dep works
+sudo lxc-attach -n $CONTAINER -- sh -ex <<EOF
+sed 's/^deb/deb-src/' /etc/apt/sources.list >> /etc/apt/sources.list.d/sources.list
+# wait until online
+while [ -z "\$(ip route list 0/0)" ]; do sleep 1; done
+apt-get -q update
+apt-get -y dist-upgrade
+apt-get install -y eatmydata
+EOF
+sudo lxc-stop -n $CONTAINER