From d7707faec204874b021566ce10373462f1c4bc62 Mon Sep 17 00:00:00 2001 From: Frantisek Sumsal Date: Wed, 24 Apr 2019 07:52:49 +0200 Subject: [PATCH] semaphore: merge semaphore scripts to avoid code duplication (#12365) --- semaphoreci/gcc-compilation.sh | 47 -------------- semaphoreci/semaphore-runner.sh | 111 ++++++++++++++++++++++++++++++++ semaphoreci/setup.sh | 57 ---------------- 3 files changed, 111 insertions(+), 104 deletions(-) delete mode 100755 semaphoreci/gcc-compilation.sh create mode 100755 semaphoreci/semaphore-runner.sh delete mode 100755 semaphoreci/setup.sh diff --git a/semaphoreci/gcc-compilation.sh b/semaphoreci/gcc-compilation.sh deleted file mode 100755 index ef499b8a35..0000000000 --- a/semaphoreci/gcc-compilation.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -ex - -# keep this in sync with setup.sh -CONTAINER=${RELEASE:-buster}-${ARCH:-amd64} -AUTOPKGTESTDIR=${SEMAPHORE_CACHE_DIR:-/tmp}/autopkgtest -# semaphore cannot expose these, but useful for interactive/local runs -ARTIFACTS_DIR=/tmp/artifacts - -# add current debian/ packaging -git fetch --depth=1 https://salsa.debian.org/systemd-team/systemd.git master -git checkout FETCH_HEAD debian - -# craft changelog -UPSTREAM_VER=$(git describe | sed 's/^v//') -cat << EOF > debian/changelog.new -systemd (${UPSTREAM_VER}-0) UNRELEASED; urgency=low - - * Automatic build for upstream test - - -- systemd test $(date -R) - -EOF -cat debian/changelog >> debian/changelog.new -mv debian/changelog.new debian/changelog - -# clean out patches -rm -rf debian/patches -# disable autopkgtests which are not for upstream -sed -i '/# NOUPSTREAM/ q' debian/tests/control -# enable more unit tests -sed -i '/^CONFFLAGS =/ s/=/= -Dtests=unsafe -Dsplit-usr=true -Dslow-tests=true /' debian/rules -# no orig tarball -echo '1.0' > debian/source/format - -# build source package -dpkg-buildpackage -S -I -I$(basename "$SEMAPHORE_CACHE_DIR") -d -us -uc -nc - -# now build the package and run the tests -rm -rf "$ARTIFACTS_DIR" -# autopkgtest exits with 2 for "some tests skipped", accept that -$AUTOPKGTESTDIR/runner/autopkgtest --apt-upgrade \ - --env DEB_BUILD_OPTIONS=noudeb \ - --env TEST_UPSTREAM=1 ../systemd_*.dsc \ - -o "$ARTIFACTS_DIR" \ - -- lxc -s $CONTAINER \ - || [ $? -eq 2 ] diff --git a/semaphoreci/semaphore-runner.sh b/semaphoreci/semaphore-runner.sh new file mode 100755 index 0000000000..5117bb4861 --- /dev/null +++ b/semaphoreci/semaphore-runner.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +set -eux + +# default to Debian testing +DISTRO=${DISTRO:-debian} +RELEASE=${RELEASE:-buster} +ARCH=${ARCH:-amd64} +CONTAINER=${RELEASE}-${ARCH} +MAX_CACHE_AGE=604800 # one week +CACHE_DIR=${SEMAPHORE_CACHE_DIR:=/tmp} +CACHE="${CACHE_DIR}/${CONTAINER}.img.tar.gz" +AUTOPKGTEST_DIR="${CACHE_DIR}/autopkgtest" +# semaphore cannot expose these, but useful for interactive/local runs +ARTIFACTS_DIR=/tmp/artifacts +PHASES=(${@:-SETUP RUN}) + +create_container() { + # create autopkgtest LXC image; this sometimes fails with "Unable to fetch + # GPG key from keyserver", so retry a few times + for retry in $(seq 5); do + sudo lxc-create -n $CONTAINER -t download -- -d $DISTRO -r $RELEASE -a $ARCH && break + sleep $((retry*retry)) + done + + # unconfine the container, otherwise some tests fail + echo 'lxc.apparmor.profile = unconfined' | sudo tee -a /var/lib/lxc/$CONTAINER/config + + sudo lxc-start -n $CONTAINER + + # enable source repositories so that apt-get build-dep works + sudo lxc-attach -n $CONTAINER -- sh -ex <> /etc/apt/sources.list.d/sources.list +# wait until online +while [ -z "\$(ip route list 0/0)" ]; do sleep 1; done +apt-get -q update +apt-get -y dist-upgrade +apt-get install -y eatmydata +EOF + sudo lxc-stop -n $CONTAINER + + # cache it + sudo tar cpzf "$CACHE" /var/lib/lxc/$CONTAINER +} + +for phase in "${PHASES[@]}"; do + case $phase in + SETUP) + # remove semaphore repos, some of them don't work and cause error messages + sudo rm -f /etc/apt/sources.list.d/* + + # enable backports for latest LXC + echo 'deb http://archive.ubuntu.com/ubuntu xenial-backports main restricted universe multiverse' | sudo tee -a /etc/apt/sources.list.d/backports.list + sudo apt-get -q update + sudo apt-get install -y -t xenial-backports lxc + sudo apt-get install -y python3-debian git dpkg-dev fakeroot + + [ -d $AUTOPKGTEST_DIR ] || git clone --quiet --depth=1 https://salsa.debian.org/ci-team/autopkgtest.git "$AUTOPKGTEST_DIR" + + # use cached container image, unless older than a week + if [ -e "$CACHE" ] && [ $(( $(date +%s) - $(stat -c %Y "$CACHE") )) -le $MAX_CACHE_AGE ]; then + sudo tar -C / -xpzf "$CACHE" + else + create_container + fi + ;; + RUN) + # add current debian/ packaging + git fetch --depth=1 https://salsa.debian.org/systemd-team/systemd.git master + git checkout FETCH_HEAD debian + + # craft changelog + UPSTREAM_VER=$(git describe | sed 's/^v//') + cat << EOF > debian/changelog.new +systemd (${UPSTREAM_VER}-0) UNRELEASED; urgency=low + + * Automatic build for upstream test + + -- systemd test $(date -R) + +EOF + cat debian/changelog >> debian/changelog.new + mv debian/changelog.new debian/changelog + + # clean out patches + rm -rf debian/patches + # disable autopkgtests which are not for upstream + sed -i '/# NOUPSTREAM/ q' debian/tests/control + # enable more unit tests + sed -i '/^CONFFLAGS =/ s/=/= -Dtests=unsafe -Dsplit-usr=true -Dslow-tests=true /' debian/rules + # no orig tarball + echo '1.0' > debian/source/format + + # build source package + dpkg-buildpackage -S -I -I$(basename "$CACHE_DIR") -d -us -uc -nc + + # now build the package and run the tests + rm -rf "$ARTIFACTS_DIR" + # autopkgtest exits with 2 for "some tests skipped", accept that + $AUTOPKGTEST_DIR/runner/autopkgtest --apt-upgrade \ + --env DEB_BUILD_OPTIONS=noudeb \ + --env TEST_UPSTREAM=1 ../systemd_*.dsc \ + -o "$ARTIFACTS_DIR" \ + -- lxc -s $CONTAINER \ + || [ $? -eq 2 ] + ;; + *) + echo >&2 "Unknown phase '$phase'" + exit 1 + esac +done diff --git a/semaphoreci/setup.sh b/semaphoreci/setup.sh deleted file mode 100755 index c39238a4e9..0000000000 --- a/semaphoreci/setup.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -set -ex - -# default to Debian testing -DISTRO=${DISTRO:-debian} -RELEASE=${RELEASE:-buster} -ARCH=${ARCH:-amd64} -CONTAINER=${RELEASE}-${ARCH} -MAX_CACHE_AGE=604800 # one week -CACHE=${SEMAPHORE_CACHE_DIR:=/tmp}/${CONTAINER}.img.tar.gz - -create_container() { - # create autopkgtest LXC image; this sometimes fails with "Unable to fetch - # GPG key from keyserver", so retry a few times - for retry in $(seq 5); do - sudo lxc-create -n $CONTAINER -t download -- -d $DISTRO -r $RELEASE -a $ARCH && break - sleep $((retry*retry)) - done - - # unconfine the container, otherwise some tests fail - echo 'lxc.apparmor.profile = unconfined' | sudo tee -a /var/lib/lxc/$CONTAINER/config - - sudo lxc-start -n $CONTAINER - - # enable source repositories so that apt-get build-dep works - sudo lxc-attach -n $CONTAINER -- sh -ex <> /etc/apt/sources.list.d/sources.list -# wait until online -while [ -z "\$(ip route list 0/0)" ]; do sleep 1; done -apt-get -q update -apt-get -y dist-upgrade -apt-get install -y eatmydata -EOF - sudo lxc-stop -n $CONTAINER - - # cache it - sudo tar cpzf "$CACHE" /var/lib/lxc/$CONTAINER -} - -# remove semaphore repos, some of them don't work and cause error messages -sudo rm -f /etc/apt/sources.list.d/* - -# enable backports for latest LXC -echo 'deb http://archive.ubuntu.com/ubuntu xenial-backports main restricted universe multiverse' | sudo tee -a /etc/apt/sources.list.d/backports.list -sudo apt-get -q update -sudo apt-get install -y -t xenial-backports lxc -sudo apt-get install -y python3-debian git dpkg-dev fakeroot - -AUTOPKGTESTDIR=$SEMAPHORE_CACHE_DIR/autopkgtest -[ -d $AUTOPKGTESTDIR ] || git clone --quiet --depth=1 https://salsa.debian.org/ci-team/autopkgtest.git "$AUTOPKGTESTDIR" - -# use cached container image, unless older than a week -if [ -e "$CACHE" ] && [ $(( $(date +%s) - $(stat -c %Y "$CACHE") )) -le $MAX_CACHE_AGE ]; then - sudo tar -C / -xpzf "$CACHE" -else - create_container -fi