From 3762f8e316e9cc5fed0ca0dc810bec0db1cb0632 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= <zbyszek@in.waw.pl>
Date: Thu, 20 Sep 2018 16:34:14 +0200
Subject: [PATCH] tests: add a runner for installed tests

We have "installed tests", but don't provide an easy way to run them.

The protocol is very simple: each test must return 0 for success, 77 means
"skipped", anything else is an error. In addition, we want to print test
output only if the test failed.

I wrote this simple script. It is pretty basic, but implements the functions
listed above. Since it is written in python it should be easy to add option
parsing (like running only specific tests, or running unsafe tests, etc.)

I looked at the following alternatives:
- Ubuntu root-unittests: this works, but just dumps all output to the terminal,
  has no coloring.
- @ssahani's test runner [2]
  It uses the unittest library and the test suite was implented as a class, and
  doesn't implement any of the functions listed above.
- cram [3,4]
  cram runs our tests, but does not understand the "ignore the output" part,
  has not support for our magic skip code (it uses hardcoded 80 instead),
  and seems dead upstream.
- meson test
  Here the idea would be to provide an almost-empty meson.build file under
  /usr/lib/systemd/tests/ that would just define all the tests. This would
  allow us to reuse the test runner we use normally. Unfortunately meson requires
  a build directory and configuration to be done before running tests. This
  would be possible, but seems a lot of effort to just run a few binaries.

[1] https://salsa.debian.org/systemd-team/systemd/blob/242c96addb06480ec9cd75248a5660f37a17b4b9/debian/tests/root-unittests
[2] https://github.com/systemd/systemd-fedora-ci/blob/master/upstream/systemd-upstream-tests.py
[3] https://bitheap.org/cram/
[4] https://pypi.org/project/pytest-cram/

Fixes #10069.
---
 test/meson.build       |  8 +++++++
 test/run-unit-tests.py | 49 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100755 test/run-unit-tests.py

diff --git a/test/meson.build b/test/meson.build
index 9750ff22b9..13896aee50 100644
--- a/test/meson.build
+++ b/test/meson.build
@@ -229,6 +229,14 @@ endif
 
 ############################################################
 
+if install_tests
+        install_data('run-unit-tests.py',
+                     install_mode : 'rwxr-xr-x',
+                     install_dir : testsdir)
+endif
+
+############################################################
+
 # prepare test/sys tree
 sys_script_py = find_program('sys-script.py')
 custom_target(
diff --git a/test/run-unit-tests.py b/test/run-unit-tests.py
new file mode 100755
index 0000000000..4bbc3e2c44
--- /dev/null
+++ b/test/run-unit-tests.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+
+import dataclasses
+import glob
+import os
+import subprocess
+import sys
+try:
+    import colorama as c
+    GREEN = c.Fore.GREEN
+    YELLOW = c.Fore.YELLOW
+    RED = c.Fore.RED
+    RESET_ALL = c.Style.RESET_ALL
+    BRIGHT = c.Style.BRIGHT
+except ImportError:
+    GREEN = YELLOW = RED = RESET_ALL = BRIGHT = ''
+
+@dataclasses.dataclass
+class Total:
+    total:int
+    good:int = 0
+    skip:int = 0
+    fail:int = 0
+
+tests = glob.glob('/usr/lib/systemd/tests/test-*')
+total = Total(total=len(tests))
+for test in tests:
+    name = os.path.basename(test)
+
+    ex = subprocess.run(test, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    if ex.returncode == 0:
+        print(f'{GREEN}PASS: {name}{RESET_ALL}')
+        total.good += 1
+    elif ex.returncode == 77:
+        print(f'{YELLOW}SKIP: {name}{RESET_ALL}')
+        total.skip += 1
+    else:
+        print(f'{RED}FAIL: {name}{RESET_ALL}')
+        total.fail += 1
+
+        # stdout/stderr might not be valid unicode, let's just dump it to the terminal.
+        # Also let's reset the style afterwards, in case our output sets something.
+        sys.stdout.buffer.write(ex.stdout)
+        print(f'{RESET_ALL}{BRIGHT}')
+        sys.stdout.buffer.write(ex.stderr)
+        print(f'{RESET_ALL}')
+
+print(f'{BRIGHT}OK: {total.good} SKIP: {total.skip} FAIL: {total.fail}{RESET_ALL}')
+sys.exit(total.fail > 0)