From a455e75a309814f2b9947a70356dbbb3aad5a982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Wed, 25 Mar 2020 13:43:20 +0100 Subject: [PATCH 01/10] test: use systemd.status-unit-format=name in all but one test We want to test the traditional output, but the non-default one is generally much nicer for debugging (and takes less space...). --- test/test-functions | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/test-functions b/test/test-functions index ead815f2fa..ee2dca64f8 100644 --- a/test/test-functions +++ b/test/test-functions @@ -919,6 +919,12 @@ install_config_files() { # set the hostname echo systemd-testsuite > $initdir/etc/hostname + + # let's set up just one image with the traditional verbose output + if [ ${IMAGE_NAME} != "basic" ]; then + mkdir -p $initdir/etc/systemd/system.conf.d + echo -e '[Manager]\nStatusUnitFormat=name' >$initdir/etc/systemd/system.conf.d/status.conf + fi } install_basic_tools() { From 3597bf75502a607c4cdf10f34f3082b1606d6abd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Wed, 25 Mar 2020 13:46:44 +0100 Subject: [PATCH 02/10] run-integration-tests: add support for blacklisting This was done downstream in debian and ubuntu [1]. I want to change the downstream file to use run-integration-tests so we can change the way tests work more easily. Let's start moving downstream functionality upstream. $ sudo BLACKLIST_MARKERS='blacklist-ubuntu-ci-arm64 blacklist-ubuntu-ci' \ BUILD_DIR=build test/run-integration-tests.sh [1] https://salsa.debian.org/systemd-team/systemd/-/blob/debian/master/debian/tests/upstream --- test/run-integration-tests.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/test/run-integration-tests.sh b/test/run-integration-tests.sh index bcbbb8b7ed..5c6c21ea09 100755 --- a/test/run-integration-tests.sh +++ b/test/run-integration-tests.sh @@ -25,9 +25,21 @@ if [ $do_clean = 1 ]; then done fi +pass_blacklist() { + for marker in $BLACKLIST_MARKERS; do + if [ -f "$1/$marker" ]; then + echo "========== BLACKLISTED: $1 ($marker) ==========" + return 1 + fi + done + return 0 +} + for TEST in TEST-??-* ; do COUNT=$(($COUNT+1)) + pass_blacklist $TEST || continue + echo -e "\n--x-- Running $TEST --x--" set +e ( set -x ; make -C "$TEST" "BUILD_DIR=$BUILD_DIR" $args ) @@ -41,7 +53,7 @@ for TEST in TEST-??-* ; do done if [ $FAILURES -eq 0 -a $do_clean = 1 ]; then - for TEST in TEST-??-* ; do + for TEST in ${!results[@]}; do ( set -x ; make -C "$TEST" "BUILD_DIR=$BUILD_DIR" clean-again ) done fi From 8943daf813221345580abe8ebaddcf10e2752287 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Mon, 30 Mar 2020 16:39:31 +0200 Subject: [PATCH 03/10] test: use a helper function to move the journal files --- test/TEST-02-CRYPTSETUP/test.sh | 5 ++--- test/TEST-24-UNIT-TESTS/test.sh | 6 ++---- test/test-functions | 20 ++++++++++++++------ 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/test/TEST-02-CRYPTSETUP/test.sh b/test/TEST-02-CRYPTSETUP/test.sh index 157a70021b..27b42223d1 100755 --- a/test/TEST-02-CRYPTSETUP/test.sh +++ b/test/TEST-02-CRYPTSETUP/test.sh @@ -13,13 +13,12 @@ check_result_qemu() { [[ -f $initdir/failed ]] && cp -a $initdir/failed $TESTDIR cryptsetup luksOpen ${LOOPDEV}p2 varcrypt <$TESTDIR/keyfile mount /dev/mapper/varcrypt $initdir/var - cp -a $initdir/var/log/journal $TESTDIR - rm -r $initdir/var/log/journal/* + save_journal $initdir/var/log/journal _umount_dir $initdir/var _umount_dir $initdir cryptsetup luksClose /dev/mapper/varcrypt [[ -f $TESTDIR/failed ]] && cat $TESTDIR/failed - ls -l $TESTDIR/journal/*/*.journal + echo $JOURNAL_LIST test -s $TESTDIR/failed && ret=$(($ret+1)) return $ret } diff --git a/test/TEST-24-UNIT-TESTS/test.sh b/test/TEST-24-UNIT-TESTS/test.sh index a3a911c1b2..e5553b0fb5 100755 --- a/test/TEST-24-UNIT-TESTS/test.sh +++ b/test/TEST-24-UNIT-TESTS/test.sh @@ -30,8 +30,7 @@ check_result_nspawn() { cat $1/testok fi fi - cp -a $1/var/log/journal $TESTDIR - rm -r $1/var/log/journal/* + save_journal $1/var/log/journal _umount_dir $initdir [[ -n "$TIMED_OUT" ]] && _ret=$(($_ret+1)) return $_ret @@ -55,8 +54,7 @@ check_result_qemu() { cat $initdir/testok fi fi - cp -a $initdir/var/log/journal $TESTDIR - rm -r $initdir/var/log/journal/* + save_journal $initdir/var/log/journal _umount_dir $initdir [[ -n "$TIMED_OUT" ]] && _ret=$(($_ret+1)) return $_ret diff --git a/test/test-functions b/test/test-functions index ee2dca64f8..b1b69add05 100644 --- a/test/test-functions +++ b/test/test-functions @@ -790,16 +790,25 @@ check_asan_reports() { return $ret } +save_journal() { + dest="${TESTDIR}" + + cp -a $1 "$dest/" + rm -r $1/* + + # we want to print this sometime later, so save this in a variable + JOURNAL_LIST="$(ls -l $dest/*/*/*.journal 2>&1)" +} + check_result_nspawn() { local ret=1 local journald_report="" local pids="" [[ -e $1/testok ]] && ret=0 [[ -f $1/failed ]] && cp -a $1/failed $TESTDIR - cp -a $1/var/log/journal $TESTDIR - rm -r $1/var/log/journal/* + save_journal $1/var/log/journal [[ -f $TESTDIR/failed ]] && cat $TESTDIR/failed - ls -l $TESTDIR/journal/*/*.journal + echo $JOURNAL_LIST test -s $TESTDIR/failed && ret=$(($ret+1)) [ -n "$TIMED_OUT" ] && ret=$(($ret+1)) check_asan_reports "$1" || ret=$(($ret+1)) @@ -813,12 +822,11 @@ check_result_qemu() { mount_initdir [[ -e $initdir/testok ]] && ret=0 [[ -f $initdir/failed ]] && cp -a $initdir/failed $TESTDIR - cp -a $initdir/var/log/journal $TESTDIR - rm -r $initdir/var/log/journal/* + save_journal $initdir/var/log/journal check_asan_reports "$initdir" || ret=$(($ret+1)) _umount_dir $initdir [[ -f $TESTDIR/failed ]] && cat $TESTDIR/failed - ls -l $TESTDIR/journal/*/*.journal + echo $JOURNAL_LIST test -s $TESTDIR/failed && ret=$(($ret+1)) [ -n "$TIMED_OUT" ] && ret=$(($ret+1)) return $ret From f9eb2d51dadc730b5de97e4c7780450ec91c0d2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Mon, 30 Mar 2020 21:29:52 +0200 Subject: [PATCH 04/10] test: use systemd-journal-remote for journal copying, support $ARTIFACT_DIRECTORY Using s-j-remote fixes the following issue: when coalescing files from multiple inputs, simply copying all files with into the the same directory might potentially mess things up, because a newer system.journal might overwrite an older journal. This happens because we run multiple tests from the same image, and need to clean out the directory after each run. By using systemd-journal-remote, we nicely coalesce all files. This has the advantage that if there aren't too many logs, we end up with just one journal file. ARTIFACT_DIRECTORY is for ubuntuautopackagetests, where the journal files are copied to a separate directory to preserve after tests have been run. This functionality can now be recreated by setting ARTIFACT_DIRECTORY=$AUTOPKGTEST_ARTIFACTS. --- test/test-functions | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/test/test-functions b/test/test-functions index b1b69add05..bf21c6312f 100644 --- a/test/test-functions +++ b/test/test-functions @@ -791,13 +791,21 @@ check_asan_reports() { } save_journal() { - dest="${TESTDIR}" + if [ -n "${ARTIFACT_DIRECTORY}" ]; then + dest="${ARTIFACT_DIRECTORY}/${testname}.journal" + else + dest="$TESTDIR/system.journal" + fi - cp -a $1 "$dest/" - rm -r $1/* + for j in $1/*; do + /usr/lib/systemd/systemd-journal-remote \ + -o $dest \ + --getter="journalctl -o export -D $j" + rm -r $j + done # we want to print this sometime later, so save this in a variable - JOURNAL_LIST="$(ls -l $dest/*/*/*.journal 2>&1)" + JOURNAL_LIST="$(ls -l $dest*)" } check_result_nspawn() { From f1416431b6627b0013d8466c8ae5e315ba27d654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Mon, 30 Mar 2020 21:45:21 +0200 Subject: [PATCH 05/10] test: optionally print the journal after tests This is hopefully the last bit of functionality from https://salsa.debian.org/systemd-team/systemd/-/blob/debian/master/debian/tests/upstream. --- test/test-functions | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/test-functions b/test/test-functions index bf21c6312f..3f5d8df966 100644 --- a/test/test-functions +++ b/test/test-functions @@ -801,6 +801,12 @@ save_journal() { /usr/lib/systemd/systemd-journal-remote \ -o $dest \ --getter="journalctl -o export -D $j" + + if [ -n "${TEST_SHOW_JOURNAL}" ]; then + echo "---- $j ----" + journalctl --no-pager -o short-monotonic --no-hostname --priority=${TEST_SHOW_JOURNAL} -D $j + fi + rm -r $j done From 68a7705e351596203a738ca666030dc747077051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Tue, 31 Mar 2020 13:19:13 +0200 Subject: [PATCH 06/10] TEST-06-*: also try the installation path for Debian https://salsa.debian.org/systemd-team/systemd/-/blob/debian/master/debian/tests/upstream used sed to adjust the path. I think it's better to make our script more flexible. --- test/TEST-06-SELINUX/test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/TEST-06-SELINUX/test.sh b/test/TEST-06-SELINUX/test.sh index 7a836bbc03..0acd7a10e8 100755 --- a/test/TEST-06-SELINUX/test.sh +++ b/test/TEST-06-SELINUX/test.sh @@ -46,7 +46,9 @@ test_create_image() { cp systemd_test.if $initdir/systemd-test-module dracut_install -o sesearch dracut_install runcon - dracut_install checkmodule semodule semodule_package m4 make /usr/libexec/selinux/hll/pp load_policy sefcontext_compile + dracut_install checkmodule semodule semodule_package m4 make load_policy sefcontext_compile + dracut_install -o /usr/libexec/selinux/hll/pp # Fedora/RHEL/... + dracut_install -o /usr/lib/selinux/hll/pp # Debian/Ubuntu/... ) } From e00fca8c2eadc51396fbc97c867b8506878e24c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Tue, 31 Mar 2020 11:23:21 +0200 Subject: [PATCH 07/10] TEST-13-*: remove containers created in test before running the test The test would fail when run again from the same image. So let's rename the stuff we create to be more unique, and remove it before running the test. (Removing it after would be more elegant, but it's hard to make sure that everything is removed when things fail halfway. Cleanup *before* tests is much more rebust.) --- test/TEST-13-NSPAWN-SMOKE/test.sh | 4 ++-- test/units/testsuite-13.sh | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/test/TEST-13-NSPAWN-SMOKE/test.sh b/test/TEST-13-NSPAWN-SMOKE/test.sh index c777c166f7..4a08cfd4ec 100755 --- a/test/TEST-13-NSPAWN-SMOKE/test.sh +++ b/test/TEST-13-NSPAWN-SMOKE/test.sh @@ -15,8 +15,8 @@ test_create_image() { setup_basic_environment mask_supporting_services - ../create-busybox-container $initdir/nc-container - initdir="$initdir/nc-container" dracut_install nc ip + ../create-busybox-container $initdir/testsuite-13.nc-container + initdir="$initdir/testsuite-13.nc-container" dracut_install nc ip ) } diff --git a/test/units/testsuite-13.sh b/test/units/testsuite-13.sh index 0ab684732c..e374206102 100755 --- a/test/units/testsuite-13.sh +++ b/test/units/testsuite-13.sh @@ -33,7 +33,8 @@ fi function check_bind_tmp_path { # https://github.com/systemd/systemd/issues/4789 - local _root="/var/lib/machines/bind-tmp-path" + local _root="/var/lib/machines/testsuite-13.bind-tmp-path" + rm -rf "$_root" /usr/lib/systemd/tests/testdata/create-busybox-container "$_root" >/tmp/bind systemd-nspawn --register=no -D "$_root" --bind=/tmp/bind /bin/sh -c 'test -e /tmp/bind' @@ -41,7 +42,8 @@ function check_bind_tmp_path { function check_norbind { # https://github.com/systemd/systemd/issues/13170 - local _root="/var/lib/machines/norbind-path" + local _root="/var/lib/machines/testsuite-13.norbind-path" + rm -rf "$_root" mkdir -p /tmp/binddir/subdir echo -n "outer" > /tmp/binddir/subdir/file mount -t tmpfs tmpfs /tmp/binddir/subdir @@ -53,8 +55,9 @@ function check_norbind { function check_notification_socket { # https://github.com/systemd/systemd/issues/4944 local _cmd='echo a | $(busybox which nc) -U -u -w 1 /run/systemd/nspawn/notify' - systemd-nspawn --register=no -D /nc-container /bin/sh -x -c "$_cmd" - systemd-nspawn --register=no -D /nc-container -U /bin/sh -x -c "$_cmd" + # /testsuite-13.nc-container is prepared by test.sh + systemd-nspawn --register=no -D /testsuite-13.nc-container /bin/sh -x -c "$_cmd" + systemd-nspawn --register=no -D /testsuite-13.nc-container -U /bin/sh -x -c "$_cmd" } function run { @@ -67,7 +70,8 @@ function run { return 0 fi - local _root="/var/lib/machines/unified-$1-cgns-$2-api-vfs-writable-$3" + local _root="/var/lib/machines/testsuite-13.unified-$1-cgns-$2-api-vfs-writable-$3" + rm -rf "$_root" /usr/lib/systemd/tests/testdata/create-busybox-container "$_root" SYSTEMD_NSPAWN_UNIFIED_HIERARCHY="$1" SYSTEMD_NSPAWN_USE_CGNS="$2" SYSTEMD_NSPAWN_API_VFS_WRITABLE="$3" systemd-nspawn --register=no -D "$_root" -b SYSTEMD_NSPAWN_UNIFIED_HIERARCHY="$1" SYSTEMD_NSPAWN_USE_CGNS="$2" SYSTEMD_NSPAWN_API_VFS_WRITABLE="$3" systemd-nspawn --register=no -D "$_root" --private-network -b From 5a89d7b39c2a91052fa9649d12b05b3ee05aa3da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Tue, 31 Mar 2020 11:44:09 +0200 Subject: [PATCH 08/10] run-integration-tests: print times --- test/run-integration-tests.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/run-integration-tests.sh b/test/run-integration-tests.sh index 5c6c21ea09..2435e1894e 100755 --- a/test/run-integration-tests.sh +++ b/test/run-integration-tests.sh @@ -13,6 +13,7 @@ fi ninja -C "$BUILD_DIR" declare -A results +declare -A times COUNT=0 FAILURES=0 @@ -39,6 +40,7 @@ for TEST in TEST-??-* ; do COUNT=$(($COUNT+1)) pass_blacklist $TEST || continue + start=$(date +%s) echo -e "\n--x-- Running $TEST --x--" set +e @@ -48,6 +50,7 @@ for TEST in TEST-??-* ; do echo "--x-- Result of $TEST: $RESULT --x--" results["$TEST"]="$RESULT" + times["$TEST"]=$(( $(date +%s) - $start )) [ "$RESULT" -ne "0" ] && FAILURES=$(($FAILURES+1)) done @@ -62,11 +65,9 @@ echo "" for TEST in ${!results[@]}; do RESULT="${results[$TEST]}" - if [ "$RESULT" -eq "0" ] ; then - echo "$TEST: SUCCESS" - else - echo "$TEST: FAIL" - fi + time="${times[$TEST]}" + string=$([ "$RESULT" = "0" ] && echo "SUCCESS" || echo "FAIL") + printf "%-35s %-8s (%3s s)\n" "${TEST}:" "${string}" "$time" done | sort if [ "$FAILURES" -eq 0 ] ; then From 6e5b51d94a1c87bd3decf0fa1e557f60a9b52017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Tue, 31 Mar 2020 11:46:00 +0200 Subject: [PATCH 09/10] test: run loop device cleanup in more cases --- test/test-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-functions b/test/test-functions index 3f5d8df966..d2c75f51ae 100644 --- a/test/test-functions +++ b/test/test-functions @@ -675,7 +675,7 @@ cleanup_loopdev() { fi } -trap cleanup_loopdev EXIT +trap cleanup_loopdev EXIT INT QUIT PIPE create_empty_image() { if [ -z "$IMAGE_NAME" ]; then From 73abf7ae06a8699e8104cc7d97b5980245ecf04d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Tue, 31 Mar 2020 12:11:14 +0200 Subject: [PATCH 10/10] run-integration-tests: only do the clean operation in the beginning When doing 'make clean', we remove the cached image. So doing 'make -C TEST-NN-foo clean setup run clean-again' in a loop is very slow. Let's filter out the 'clean' target (if specified), and do the cleaning in the beginning, and then run other targets in a loop as before. --- test/run-integration-tests.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/run-integration-tests.sh b/test/run-integration-tests.sh index 2435e1894e..ac7a28cf2c 100755 --- a/test/run-integration-tests.sh +++ b/test/run-integration-tests.sh @@ -4,11 +4,11 @@ set -e BUILD_DIR="$($(dirname "$0")/../tools/find-build-dir.sh)" if [ $# -gt 0 ]; then args="$@" - do_clean=0 else args="setup run clean-again" - do_clean=1 fi +args_no_clean=$(sed -r 's/(^| )clean($| )/ /g' <<<$args) +do_clean=$( [ "$args" = "$args_no_clean" ]; echo $? ) ninja -C "$BUILD_DIR" @@ -20,6 +20,8 @@ FAILURES=0 cd "$(dirname "$0")" +# Let's always do the cleaning operation first, because it destroys the image +# cache. if [ $do_clean = 1 ]; then for TEST in TEST-??-* ; do ( set -x ; make -C "$TEST" "BUILD_DIR=$BUILD_DIR" clean ) @@ -44,7 +46,7 @@ for TEST in TEST-??-* ; do echo -e "\n--x-- Running $TEST --x--" set +e - ( set -x ; make -C "$TEST" "BUILD_DIR=$BUILD_DIR" $args ) + ( set -x ; make -C "$TEST" "BUILD_DIR=$BUILD_DIR" $args_no_clean ) RESULT=$? set -e echo "--x-- Result of $TEST: $RESULT --x--"