Introduce notion of a test group, use for CA tests

Grouping our tests should make it easier to understand the intent than
one long poorly-arranged list. It also is convenient for running just
the tests for a specific component when working on that component.

We need at least one test group so this isn't dead code; I decided to
collect the tests for the `ca-derivations` and `dynamic-derivations`
experimental features in groups. Do
```bash
make ca.test-group -jN
```
and
```bash
make dyn-drv.test-group -jN
```
to try running just them.

I originally did this as part of #8397 for being able to just the local
overlay store alone. I am PRing it separately now so we can separate
general infra from new features.

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
This commit is contained in:
John Ericson 2023-07-09 22:24:51 -04:00
parent a8d5bb5e7e
commit 259e328de8
7 changed files with 97 additions and 29 deletions

View file

@ -27,6 +27,8 @@ makefiles += \
src/libstore/tests/local.mk \
src/libexpr/tests/local.mk \
tests/local.mk \
tests/ca/local.mk \
tests/dyn-drv/local.mk \
tests/test-libstoreconsumer/local.mk \
tests/plugins/local.mk
else

View file

@ -14,6 +14,8 @@ You can run the whole testsuite with `make check`, or the tests for a specific c
The functional tests reside under the `tests` directory and are listed in `tests/local.mk`.
Each test is a bash script.
### Running the whole test suite
The whole test suite can be run with:
```shell-session
@ -23,6 +25,33 @@ ran test tests/bar.sh... [PASS]
...
```
### Grouping tests
Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite.
Each test group is in a subdirectory of `tests`.
For example, `tests/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs.
That test group can be run like this:
```shell-session
$ make ca.test-group -j50
ran test tests/ca/nix-run.sh... [PASS]
ran test tests/ca/import-derivation.sh... [PASS]
...
```
The test group is defined in Make like this:
```makefile
$(test-group-name)-tests := \
$(d)/test0.sh \
$(d)/test1.sh \
...
install-tests-groups += $(test-group-name)
```
### Running individual tests
Individual tests can be run with `make`:
```shell-session

View file

@ -10,6 +10,7 @@ bin-scripts :=
noinst-scripts :=
man-pages :=
install-tests :=
install-tests-groups :=
ifdef HOST_OS
HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS)))
@ -121,7 +122,16 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b
$(foreach script, $(bin-scripts), $(eval programs-list += $(script)))
$(foreach script, $(noinst-scripts), $(eval programs-list += $(script)))
$(foreach template, $(template-files), $(eval $(call instantiate-template,$(template))))
$(foreach test, $(install-tests), $(eval $(call run-install-test,$(test))))
$(foreach test, $(install-tests), \
$(eval $(call run-install-test,$(test))) \
$(eval installcheck: $(test).test))
$(foreach test-group, $(install-tests-groups), \
$(eval $(call run-install-test-group,$(test-group))) \
$(eval installcheck: $(test-group).test-group) \
$(foreach test, $($(test-group)-tests), \
$(eval $(call run-install-test,$(test))) \
$(eval $(test-group).test-group: $(test).test)))
$(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file))))))
@ -151,6 +161,14 @@ ifdef libs-list
@echo "The following libraries can be built:"
@echo ""
@for i in $(libs-list); do echo " $$i"; done
endif
ifdef install-tests-groups
@echo ""
@echo "The following groups of functional tests can be run:"
@echo ""
@for i in $(install-tests-groups); do echo " $$i.test-group"; done
@echo ""
@echo "(installcheck includes tests in test groups too.)"
endif
@echo ""
@echo "The following variables control the build:"

View file

@ -4,8 +4,6 @@ test-deps =
define run-install-test
installcheck: $1.test
.PHONY: $1.test
$1.test: $1 $(test-deps)
@env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null
@ -16,6 +14,12 @@ define run-install-test
endef
define run-install-test-group
.PHONY: $1.test-group
endef
.PHONY: check installcheck
print-top-help += \

27
tests/ca/local.mk Normal file
View file

@ -0,0 +1,27 @@
ca-tests := \
$(d)/build-with-garbage-path.sh \
$(d)/build.sh \
$(d)/concurrent-builds.sh \
$(d)/derivation-json.sh \
$(d)/duplicate-realisation-in-closure.sh \
$(d)/gc.sh \
$(d)/import-derivation.sh \
$(d)/new-build-cmd.sh \
$(d)/nix-copy.sh \
$(d)/nix-run.sh \
$(d)/nix-shell.sh \
$(d)/post-hook.sh \
$(d)/recursive.sh \
$(d)/repl.sh \
$(d)/selfref-gc.sh \
$(d)/signatures.sh \
$(d)/substitute.sh \
$(d)/why-depends.sh
install-tests-groups += ca
clean-files += \
$(d)/config.nix
test-deps += \
tests/ca/config.nix

11
tests/dyn-drv/local.mk Normal file
View file

@ -0,0 +1,11 @@
dyn-drv-tests := \
$(d)/text-hashed-output.sh \
$(d)/recursive-mod-json.sh
install-tests-groups += dyn-drv
clean-files += \
$(d)/config.nix
test-deps += \
tests/dyn-drv/config.nix

View file

@ -14,7 +14,6 @@ nix_tests = \
flakes/absolute-paths.sh \
flakes/build-paths.sh \
flakes/flake-in-submodule.sh \
ca/gc.sh \
gc.sh \
nix-collect-garbage-d.sh \
remote-store.sh \
@ -28,8 +27,6 @@ nix_tests = \
user-envs-migration.sh \
binary-cache.sh \
multiple-outputs.sh \
ca/build.sh \
ca/new-build-cmd.sh \
nix-build.sh \
gc-concurrent.sh \
repair.sh \
@ -47,24 +44,17 @@ nix_tests = \
referrers.sh \
optimise-store.sh \
substitute-with-invalid-ca.sh \
ca/concurrent-builds.sh \
signing.sh \
ca/build-with-garbage-path.sh \
hash.sh \
gc-non-blocking.sh \
check.sh \
ca/substitute.sh \
nix-shell.sh \
ca/signatures.sh \
ca/nix-shell.sh \
ca/nix-copy.sh \
check-refs.sh \
build-remote-input-addressed.sh \
secure-drv-outputs.sh \
restricted.sh \
fetchGitSubmodules.sh \
flakes/search-root.sh \
ca/duplicate-realisation-in-closure.sh \
readfile-context.sh \
nix-channel.sh \
recursive.sh \
@ -80,10 +70,7 @@ nix_tests = \
nar-access.sh \
pure-eval.sh \
eval.sh \
ca/post-hook.sh \
repl.sh \
ca/repl.sh \
ca/recursive.sh \
binary-cache-build-remote.sh \
search.sh \
logging.sh \
@ -109,13 +96,8 @@ nix_tests = \
fmt.sh \
eval-store.sh \
why-depends.sh \
ca/why-depends.sh \
derivation-json.sh \
ca/derivation-json.sh \
import-derivation.sh \
ca/import-derivation.sh \
dyn-drv/text-hashed-output.sh \
dyn-drv/recursive-mod-json.sh \
nix_path.sh \
case-hack.sh \
placeholders.sh \
@ -124,8 +106,7 @@ nix_tests = \
build.sh \
build-delete.sh \
output-normalization.sh \
ca/nix-run.sh \
selfref-gc.sh ca/selfref-gc.sh \
selfref-gc.sh \
db-migration.sh \
bash-profile.sh \
pass-as-file.sh \
@ -150,16 +131,12 @@ install-tests += $(foreach x, $(nix_tests), $(d)/$(x))
clean-files += \
$(d)/common/vars-and-functions.sh \
$(d)/config.nix \
$(d)/ca/config.nix \
$(d)/dyn-drv/config.nix
$(d)/config.nix
test-deps += \
tests/common/vars-and-functions.sh \
tests/config.nix \
tests/ca/config.nix \
tests/test-libstoreconsumer/test-libstoreconsumer \
tests/dyn-drv/config.nix
tests/test-libstoreconsumer/test-libstoreconsumer
ifeq ($(BUILD_SHARED_LIBS), 1)
test-deps += tests/plugins/libplugintest.$(SO_EXT)