From 408dcfc0d3898fb855c1038d24c09c82fe58c379 Mon Sep 17 00:00:00 2001 From: Sandro Date: Mon, 5 Sep 2022 15:42:10 +0200 Subject: [PATCH 001/909] Improve experimental-features error wording --- src/libutil/experimental-features.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index fa79cca6b..d8b0d30fa 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -52,7 +52,7 @@ std::set parseFeatures(const std::set & rawFea } MissingExperimentalFeature::MissingExperimentalFeature(ExperimentalFeature feature) - : Error("experimental Nix feature '%1%' is disabled; use '--extra-experimental-features %1%' to override", showExperimentalFeature(feature)) + : Error("experimental Nix feature '%1%' is disabled; pass '--extra-experimental-features %1%' as an argument to enable it", showExperimentalFeature(feature)) , missingFeature(feature) {} From 481e4082bf96f5613693785e97a9a5cec7c09940 Mon Sep 17 00:00:00 2001 From: Sandro Date: Wed, 9 Nov 2022 12:03:53 +0100 Subject: [PATCH 002/909] Update src/libutil/experimental-features.cc Co-authored-by: Valentin Gagarin --- src/libutil/experimental-features.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index d8b0d30fa..052988b34 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -52,7 +52,7 @@ std::set parseFeatures(const std::set & rawFea } MissingExperimentalFeature::MissingExperimentalFeature(ExperimentalFeature feature) - : Error("experimental Nix feature '%1%' is disabled; pass '--extra-experimental-features %1%' as an argument to enable it", showExperimentalFeature(feature)) + : Error("experimental Nix feature '%1%' is disabled; add '--extra-experimental-features %1%' to enable it", showExperimentalFeature(feature)) , missingFeature(feature) {} From c74248c56e6e5558603be6c6eda43eb439772f20 Mon Sep 17 00:00:00 2001 From: Michael Lohmann Date: Thu, 12 Jan 2023 15:45:19 +0100 Subject: [PATCH 003/909] install-multi-user: ignore profile_target backups that have no change If there was a prior nix installation that created this backup file and then you tried to install it again, it would stop to tell you there is this file. But if the file and its backup are identical in content, there is no harm in continuing and in a later step overwriting the existing backup file with the identical one. This is just a convenience feature. --- scripts/install-multi-user.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index f149ea0d7..b2f2a9e45 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -445,6 +445,14 @@ EOF # a row for different files. if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then # this backup process first released in Nix 2.1 + + if diff -q "$profile_target$PROFILE_BACKUP_SUFFIX" "$profile_target" > /dev/null; then + # a backup file for the rc-file exist, but they are identical, + # so we can safely ignore it and overwrite it with the same + # content later + continue + fi + failure < Date: Mon, 23 Jan 2023 10:50:44 +0000 Subject: [PATCH 004/909] improved help command listing. --- src/libcmd/repl.cc | 53 +++++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index e3afb1531..5fc503c2b 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -486,35 +486,40 @@ bool NixRepl::processLine(std::string line) std::cout << "The following commands are available:\n" << "\n" - << " Evaluate and print expression\n" - << " = Bind expression to variable\n" - << " :a Add attributes from resulting set to scope\n" - << " :b Build a derivation\n" - << " :bl Build a derivation, creating GC roots in the working directory\n" - << " :e Open package or function in $EDITOR\n" - << " :i Build derivation, then install result into current profile\n" - << " :l Load Nix expression and add it to scope\n" - << " :lf Load Nix flake and add it to scope\n" - << " :p Evaluate and print expression recursively\n" - << " :q Exit nix-repl\n" - << " :r Reload all files\n" - << " :sh Build dependencies of derivation, then start nix-shell\n" - << " :t Describe result of evaluation\n" - << " :u Build derivation, then start nix-shell\n" - << " :doc Show documentation of a builtin function\n" - << " :log Show logs for a derivation\n" - << " :te [bool] Enable, disable or toggle showing traces for errors\n" + << " Evaluate and print expression\n" + << " = Bind expression to variable\n" + << " :a, :add Add attributes from resulting set to scope\n" + << " :b Build a derivation\n" + << " :bl Build a derivation, creating GC roots in the\n" + << " working directory\n" + << " :e, :edit Open package or function in $EDITOR\n" + << " :i Build derivation, then install result into\n" + << " current profile\n" + << " :l, :load Load Nix expression and add it to scope\n" + << " :lf, :load-flake Load Nix flake and add it to scope\n" + << " :p, :print Evaluate and print expression recursively\n" + << " :q, :quit Exit nix-repl\n" + << " :r, :reload Reload all files\n" + << " :sh Build dependencies of derivation, then start\n" + << " nix-shell\n" + << " :t Describe result of evaluation\n" + << " :u Build derivation, then start nix-shell\n" + << " :doc Show documentation of a builtin function\n" + << " :log Show logs for a derivation\n" + << " :te, :trace-enable [bool] Enable, disable or toggle showing traces for\n" + << " errors\n" + << " :?, :help Brings up this help menu\n" ; if (state->debugRepl) { std::cout << "\n" << " Debug mode commands\n" - << " :env Show env stack\n" - << " :bt Show trace stack\n" - << " :st Show current trace\n" - << " :st Change to another trace in the stack\n" - << " :c Go until end of program, exception, or builtins.break\n" - << " :s Go one step\n" + << " :env Show env stack\n" + << " :bt, :backtrace Show trace stack\n" + << " :st Show current trace\n" + << " :st Change to another trace in the stack\n" + << " :c, :continue Go until end of program, exception, or builtins.break\n" + << " :s, :step Go one step\n" ; } From c4df53f1542086836d6d59cf82cff8f5cbe5d8f1 Mon Sep 17 00:00:00 2001 From: Rynn Blackmon Date: Wed, 19 Apr 2023 16:32:47 -0700 Subject: [PATCH 005/909] Mention `$DRV_PATH` in post-build-hook docs --- doc/manual/src/advanced-topics/post-build-hook.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md index 1479cc3a4..17b67111c 100644 --- a/doc/manual/src/advanced-topics/post-build-hook.md +++ b/doc/manual/src/advanced-topics/post-build-hook.md @@ -69,6 +69,8 @@ exec nix copy --to "s3://example-nix-cache" $OUT_PATHS > store sign`. Nix guarantees the paths will not contain any spaces, > however a store path might contain glob characters. The `set -f` > disables globbing in the shell. +> If you want to upload the .drv file too, the `$DRV_PATH` variable +> is also defined for the script and works just like `$OUT_PATHS`. Then make sure the hook program is executable by the `root` user: From 6d1aa523de8ece115eb0504b2313c12dc5302383 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 May 2023 14:41:47 -0400 Subject: [PATCH 006/909] Create escape hatch for supplementary group sandboxing woes There is no obvious good solution for this that has occured to anyone. --- src/libstore/build/local-derivation-goal.cc | 13 ++++--------- src/libstore/globals.hh | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 21cd6e7ee..6d2d458da 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -907,15 +907,10 @@ void LocalDerivationGoal::startBuilder() openSlave(); /* Drop additional groups here because we can't do it - after we've created the new user namespace. FIXME: - this means that if we're not root in the parent - namespace, we can't drop additional groups; they will - be mapped to nogroup in the child namespace. There does - not seem to be a workaround for this. (But who can tell - from reading user_namespaces(7)?) - See also https://lwn.net/Articles/621612/. */ - if (getuid() == 0 && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); + after we've created the new user namespace. */ + if (settings.dropSupplementaryGroups) + if (setgroups(0, 0) == -1) + throw SysError("setgroups failed"); ProcessOptions options; options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 609cf53b8..0b429cf98 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -515,6 +515,25 @@ public: Setting sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; + Setting dropSupplementaryGroups{this, getuid() == 0, "drop-supplementary-groups", + R"( + Whether to drop supplementary groups when building with sandboxing. + This is normally a good idea if we are root and have the capability to + do so. + + But if this "root" is mapped from a non-root user in a larger + namespace, we won't be able drop additional groups; they will be + mapped to nogroup in the child namespace. There does not seem to be a + workaround for this. + + (But who can tell from reading user_namespaces(7)? See also https://lwn.net/Articles/621612/.) + + TODO: It might be good to create a middle ground option that allows + `setgroups` to fail if all additional groups are "nogroup" / the value + of `/proc/sys/fs/overflowuid`. This would handle the common + nested-sandboxing case identified above. + )"}; + #if __linux__ Setting sandboxShmSize{ this, "50%", "sandbox-dev-shm-size", From 2524a2118647a4125dcae08fe0eb20de5f79a291 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 15 May 2023 12:38:39 -0400 Subject: [PATCH 007/909] Update src/libstore/build/local-derivation-goal.cc Co-authored-by: Guillaume Girol --- src/libstore/build/local-derivation-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6d2d458da..a50fe1a28 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -910,7 +910,7 @@ void LocalDerivationGoal::startBuilder() after we've created the new user namespace. */ if (settings.dropSupplementaryGroups) if (setgroups(0, 0) == -1) - throw SysError("setgroups failed"); + throw SysError("setgroups failed. Set the drop-supplementary-groups option to false to skip this step."); ProcessOptions options; options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; From d8ef0c949523324615b66059b3d48c4c445f478b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 15 May 2023 17:41:51 -0400 Subject: [PATCH 008/909] Add some tests for `drop-supplementary-groups` --- tests/common.sh | 2 +- tests/common/vars-and-functions.sh.in | 2 +- tests/hermetic.nix | 56 +++++++++++++++++++++++++++ tests/local.mk | 1 + tests/supplementary-groups.sh | 33 ++++++++++++++++ 5 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 tests/hermetic.nix create mode 100644 tests/supplementary-groups.sh diff --git a/tests/common.sh b/tests/common.sh index 8941671d6..7b0922c9f 100644 --- a/tests/common.sh +++ b/tests/common.sh @@ -4,7 +4,7 @@ if [[ -z "${COMMON_SH_SOURCED-}" ]]; then COMMON_SH_SOURCED=1 -source "$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")/common/vars-and-functions.sh" +source "$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")/common/vars-and-functions.sh" if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then startDaemon fi diff --git a/tests/common/vars-and-functions.sh.in b/tests/common/vars-and-functions.sh.in index a9e6c802f..dc7ce13cc 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/common/vars-and-functions.sh.in @@ -4,7 +4,7 @@ if [[ -z "${COMMON_VARS_AND_FUNCTIONS_SH_SOURCED-}" ]]; then COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1 -export PS4='+(${BASH_SOURCE[0]}:$LINENO) ' +export PS4='+(${BASH_SOURCE[0]-$0}:$LINENO) ' export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default} export NIX_STORE_DIR diff --git a/tests/hermetic.nix b/tests/hermetic.nix new file mode 100644 index 000000000..4c9d7a51f --- /dev/null +++ b/tests/hermetic.nix @@ -0,0 +1,56 @@ +{ busybox, seed }: + +with import ./config.nix; + +let + contentAddressedByDefault = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT" == "1"; + caArgs = if contentAddressedByDefault then { + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } else {}; + + mkDerivation = args: + derivation ({ + inherit system; + builder = busybox; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + } // removeAttrs args ["builder" "meta" "passthru"] + // caArgs) + // { meta = args.meta or {}; passthru = args.passthru or {}; }; + + input1 = mkDerivation { + shell = busybox; + name = "hermetic-input-1"; + buildCommand = "echo hi-input1 seed=${toString seed}; echo FOO > $out"; + }; + + input2 = mkDerivation { + shell = busybox; + name = "hermetic-input-2"; + buildCommand = "echo hi; echo BAR > $out"; + }; + + input3 = mkDerivation { + shell = busybox; + name = "hermetic-input-3"; + buildCommand = '' + echo hi-input3 + read x < ${input2} + echo $x BAZ > $out + ''; + }; + +in + + mkDerivation { + shell = busybox; + name = "hermetic"; + passthru = { inherit input1 input2 input3; }; + buildCommand = + '' + read x < ${input1} + read y < ${input3} + echo "$x $y" > $out + ''; + } diff --git a/tests/local.mk b/tests/local.mk index 9e340e2e2..9cb81e1f0 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -93,6 +93,7 @@ nix_tests = \ misc.sh \ dump-db.sh \ linux-sandbox.sh \ + supplementary-groups.sh \ build-dry.sh \ structured-attrs.sh \ shell.sh \ diff --git a/tests/supplementary-groups.sh b/tests/supplementary-groups.sh new file mode 100644 index 000000000..fd3da2945 --- /dev/null +++ b/tests/supplementary-groups.sh @@ -0,0 +1,33 @@ +source common.sh + +requireSandboxSupport +[[ $busybox =~ busybox ]] || skipTest "no busybox" +if ! command -p -v unshare; then skipTest "Need unshare"; fi +needLocalStore "The test uses --store always so we would just be bypassing the daemon" + +unshare --mount --map-root-user bash < Date: Mon, 15 May 2023 17:49:04 -0400 Subject: [PATCH 009/909] Avoid out links in supplementary groups test This gets in the way of the tests running in parallel. --- tests/supplementary-groups.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/supplementary-groups.sh b/tests/supplementary-groups.sh index fd3da2945..47debc5e3 100644 --- a/tests/supplementary-groups.sh +++ b/tests/supplementary-groups.sh @@ -13,7 +13,7 @@ unshare --mount --map-root-user bash < Date: Tue, 14 Mar 2023 11:54:04 -0300 Subject: [PATCH 010/909] feat: add always-allow-substitutes This adds a new configuration option to Nix, `always-allow-substitutes`, whose effect is simple: it causes the `allowSubstitutes` attribute in derivations to be ignored, and for substituters to always be used. This is extremely valuable for users of Nix in CI, where usually `nix-build-uncached` is used. There, derivations which disallow substitutes cause headaches as the inputs for building already-cached derivations need to be fetched to spuriously rebuild some simple text file. This option should be a good middle-ground, since it doesn't imply rebuilding the world, such as the approach I took in https://github.com/NixOS/nixpkgs/pull/221048 --- doc/manual/src/language/advanced-attributes.md | 3 +++ src/libstore/globals.hh | 8 ++++++++ src/libstore/parsed-derivations.cc | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 307971434..211d195b2 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -261,6 +261,9 @@ Derivations can declare some infrequently used optional attributes. useful for very trivial derivations (such as `writeText` in Nixpkgs) that are cheaper to build than to substitute from a binary cache. + You may disable the effects of this attibute by enabling the + `always-allow-substitutes` configuration option in Nix. + > **Note** > > You need to have a builder configured which satisfies the diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 31dfe5b4e..8b0e3fc7d 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -255,6 +255,14 @@ public: For the exact format and examples, see [the manual chapter on remote builds](../advanced-topics/distributed-builds.md) )"}; + Setting alwaysAllowSubstitutes{ + this, false, "always-allow-substitutes", + R"( + If set to `true`, Nix will ignore the `allowSubstitutes` attribute in + derivations and always attempt to use available substituters. + For more information on `allowSubstitutes`, see [the manual chapter on advanced attributes](../language/advanced-attributes.md). + )"}; + Setting buildersUseSubstitutes{ this, false, "builders-use-substitutes", R"( diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index cc4a94fab..1d900c272 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -122,7 +122,7 @@ bool ParsedDerivation::willBuildLocally(Store & localStore) const bool ParsedDerivation::substitutesAllowed() const { - return getBoolAttr("allowSubstitutes", true); + return settings.alwaysAllowSubstitutes ? true : getBoolAttr("allowSubstitutes", true); } bool ParsedDerivation::useUidRange() const From b9c2f834ee37c76fcb21bc5dbcd60bb58a229194 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Mon, 22 May 2023 23:39:31 +0200 Subject: [PATCH 011/909] Check exact error codes in linux-sandbox.sh --- tests/linux-sandbox-cert-test.nix | 45 ++++++++++++++++--------------- tests/linux-sandbox.sh | 18 ++++++++----- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/tests/linux-sandbox-cert-test.nix b/tests/linux-sandbox-cert-test.nix index 2b86dad2e..2fc083ea9 100644 --- a/tests/linux-sandbox-cert-test.nix +++ b/tests/linux-sandbox-cert-test.nix @@ -1,29 +1,30 @@ -{ fixed-output }: +{ mode }: with import ./config.nix; -mkDerivation ({ - name = "ssl-export"; - buildCommand = '' - # Add some indirection, otherwise grepping into the debug output finds the string. - report () { echo CERT_$1_IN_SANDBOX; } +mkDerivation ( + { + name = "ssl-export"; + buildCommand = '' + # Add some indirection, otherwise grepping into the debug output finds the string. + report () { echo CERT_$1_IN_SANDBOX; } - if [ -f /etc/ssl/certs/ca-certificates.crt ]; then - content=$( $TEST_ROOT/log) -if grepQuiet 'error: renaming' $TEST_ROOT/log; then false; fi -grepQuiet 'may not be deterministic' $TEST_ROOT/log +expectStderr 104 nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link --check -K \ + | tee >( grepQuietInverse 'error: renaming' ) \ + | grepQuiet 'may not be deterministic' # Test that sandboxed builds cannot write to /etc easily -(! nix-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' --no-out-link --sandbox-paths /nix/store) +expect 100 nix-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' --no-out-link --sandbox-paths /nix/store ## Test mounting of SSL certificates into the sandbox testCert () { - (! nix-build linux-sandbox-cert-test.nix --argstr fixed-output "$2" --no-out-link --sandbox-paths /nix/store --option ssl-cert-file "$3" 2> $TEST_ROOT/log) - cat $TEST_ROOT/log - grepQuiet "CERT_${1}_IN_SANDBOX" $TEST_ROOT/log + expectation=$1 # "missing" | "present" + mode=$2 # "normal" | "fixed-output" + certFile=$3 # a string that can be the path to a cert file + [ "$mode" == fixed-output ] && ret=1 || ret=100 + expectStderr $ret nix-build linux-sandbox-cert-test.nix --argstr mode "$mode" --no-out-link --sandbox-paths /nix/store --option ssl-cert-file "$certFile" | \ + # tee /dev/stderr | \ + grepQuiet "CERT_${expectation}_IN_SANDBOX" } nocert=$TEST_ROOT/no-cert-file.pem From f0233f3a3fd92d443d2502ed29aea076b6572ab7 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Tue, 23 May 2023 09:59:49 +0200 Subject: [PATCH 012/909] Further refactor linux-sandbox.sh and fix tee usage --- tests/linux-sandbox.sh | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh index 5c4e0cae9..19a7527bf 100644 --- a/tests/linux-sandbox.sh +++ b/tests/linux-sandbox.sh @@ -11,6 +11,8 @@ requireSandboxSupport # otherwise things get complicated (e.g. if it's in /bin, do we need # /lib as well?). if [[ ! $SHELL =~ /nix/store ]]; then skipTest "Shell is not from Nix store"; fi +# An alias to automatically bind-mount the $SHELL on nix-build invocations +nix-sandbox-build () { nix-build --no-out-link --sandbox-paths /nix/store "$@"; } chmod -R u+w $TEST_ROOT/store0 || true rm -rf $TEST_ROOT/store0 @@ -18,7 +20,7 @@ rm -rf $TEST_ROOT/store0 export NIX_STORE_DIR=/my/store export NIX_REMOTE=$TEST_ROOT/store0 -outPath=$(nix-build dependencies.nix --no-out-link --sandbox-paths /nix/store) +outPath=$(nix-sandbox-build dependencies.nix) [[ $outPath =~ /my/store/.*-dependencies ]] @@ -29,17 +31,18 @@ nix store ls -R -l $outPath | grep foobar nix store cat $outPath/foobar | grep FOOBAR # Test --check without hash rewriting. -nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store +nix-sandbox-build dependencies.nix --check # Test that sandboxed builds with --check and -K can move .check directory to store -nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link +nix-sandbox-build check.nix -A nondeterministic -expectStderr 104 nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link --check -K \ - | tee >( grepQuietInverse 'error: renaming' ) \ - | grepQuiet 'may not be deterministic' +expectStderr 104 nix-sandbox-build check.nix -A nondeterministic --check -K > $TEST_ROOT/log +grepQuietInverse 'error: renaming' $TEST_ROOT/log +grepQuiet 'may not be deterministic' $TEST_ROOT/log # Test that sandboxed builds cannot write to /etc easily -expect 100 nix-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' --no-out-link --sandbox-paths /nix/store +expectStderr 100 nix-sandbox-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' | + grepQuiet "/etc/test: Permission denied" ## Test mounting of SSL certificates into the sandbox @@ -48,9 +51,9 @@ testCert () { mode=$2 # "normal" | "fixed-output" certFile=$3 # a string that can be the path to a cert file [ "$mode" == fixed-output ] && ret=1 || ret=100 - expectStderr $ret nix-build linux-sandbox-cert-test.nix --argstr mode "$mode" --no-out-link --sandbox-paths /nix/store --option ssl-cert-file "$certFile" | \ - # tee /dev/stderr | \ - grepQuiet "CERT_${expectation}_IN_SANDBOX" + expectStderr $ret nix-sandbox-build linux-sandbox-cert-test.nix --argstr mode "$mode" --option ssl-cert-file "$certFile" | + # tee /dev/stderr | + grepQuiet "CERT_${expectation}_IN_SANDBOX" } nocert=$TEST_ROOT/no-cert-file.pem From 5454fdcceb2004c67b0c829ae1ee447da9c81d00 Mon Sep 17 00:00:00 2001 From: David McFarland Date: Mon, 12 Jun 2023 23:42:05 -0300 Subject: [PATCH 013/909] Add test of explicit ssh control path in nix-copy test This highlights a problem caused by SSHMaster::isMasterRunning returning false when NIX_SSHOPTS contains -oControlPath. --- tests/nixos/nix-copy.nix | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/nixos/nix-copy.nix b/tests/nixos/nix-copy.nix index 16c477bf9..ef053de03 100644 --- a/tests/nixos/nix-copy.nix +++ b/tests/nixos/nix-copy.nix @@ -79,6 +79,15 @@ in { server.copy_from_host("key.pub", "/root/.ssh/authorized_keys") server.succeed("systemctl restart sshd") client.succeed(f"ssh -o StrictHostKeyChecking=no {server.name} 'echo hello world'") + client.succeed(f"ssh -O check {server.name}") + client.succeed(f"ssh -O exit {server.name}") + client.fail(f"ssh -O check {server.name}") + + # Check that an explicit master will work + client.succeed(f"ssh -MNfS /tmp/master {server.name}") + client.succeed(f"ssh -S /tmp/master -O check {server.name}") + client.succeed("NIX_SSHOPTS='-oControlPath=/tmp/master' nix copy --to ssh://server ${pkgA} >&2") + client.succeed(f"ssh -S /tmp/master -O exit {server.name}") # Copy the closure of package B from the server to the client, using ssh-ng. client.fail("nix-store --check-validity ${pkgB}") From d5e1eb20a2712eb17076c3ca2d30e5ac4a351d00 Mon Sep 17 00:00:00 2001 From: David McFarland Date: Mon, 12 Jun 2023 23:45:53 -0300 Subject: [PATCH 014/909] Pass common ssh options in isMasterRunning --- src/libstore/ssh.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index fae99d75b..da32f1b79 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -42,7 +42,10 @@ void SSHMaster::addCommonSSHOpts(Strings & args) } bool SSHMaster::isMasterRunning() { - auto res = runProgram(RunOptions {.program = "ssh", .args = {"-O", "check", host}, .mergeStderrToStdout = true}); + Strings args = {"-O", "check", host}; + addCommonSSHOpts(args); + + auto res = runProgram(RunOptions {.program = "ssh", .args = args, .mergeStderrToStdout = true}); return res.first == 0; } From e09b40e0d0b68ca7c3646ddffb50e1356daec997 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sun, 5 Mar 2023 02:36:26 +0100 Subject: [PATCH 015/909] reword documentation on trusted users and substituters this is to make it slightly easier to scan over --- src/libstore/globals.hh | 14 ++++++-------- src/nix/daemon.cc | 30 +++++++++++++++--------------- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 820898350..b0c025c8e 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -700,8 +700,8 @@ public: At least one of the following conditions must be met for Nix to use a substituter: - - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list - - the user calling Nix is in the [`trusted-users`](#conf-trusted-users) list + - The substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list + - The user calling Nix is in the [`trusted-users`](#conf-trusted-users) list In addition, each store path should be trusted as described in [`trusted-public-keys`](#conf-trusted-public-keys) )", @@ -710,12 +710,10 @@ public: Setting trustedSubstituters{ this, {}, "trusted-substituters", R"( - A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), - separated by whitespace. These are - not used by default, but can be enabled by users of the Nix daemon - by specifying `--option substituters urls` on the command - line. Unprivileged users are only allowed to pass a subset of the - URLs listed in `substituters` and `trusted-substituters`. + A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace. + These are not used by default, but can be enabled by users of the Nix daemon by specifying [`substituters`](#conf-substituters). + + Unprivileged users are only allowed to pass as `substituters` only those URLs listed in `trusted-substituters`. )", {"trusted-binary-caches"}}; diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 9fe9b3b1e..09adab5d3 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -55,19 +55,16 @@ struct AuthorizationSettings : Config { Setting trustedUsers{ this, {"root"}, "trusted-users", R"( - A list of names of users (separated by whitespace) that have - additional rights when connecting to the Nix daemon, such as the - ability to specify additional binary caches, or to import unsigned - NARs. You can also specify groups by prefixing them with `@`; for - instance, `@wheel` means all users in the `wheel` group. The default - is `root`. + A list of user names, separated by whitespace. + These users will have additional rights when connecting to the Nix daemon, such as the ability to specify additional [substituters](#conf-substituters), or to import unsigned [NAR](@docroot@/glossary.md#gloss-nar)s. + + You can also specify groups by prefixing names with `@`. + For instance, `@wheel` means all users in the `wheel` group. > **Warning** > - > Adding a user to `trusted-users` is essentially equivalent to - > giving that user root access to the system. For example, the user - > can set `sandbox-paths` and thereby obtain read access to - > directories that are otherwise inacessible to them. + > Adding a user to `trusted-users` is essentially equivalent to giving that user root access to the system. + > For example, the user can set [`sandbox-paths`](#conf-sandbox-paths) and thereby obtain read access to directories that are otherwise inacessible to them. )"}; /** @@ -76,12 +73,15 @@ struct AuthorizationSettings : Config { Setting allowedUsers{ this, {"*"}, "allowed-users", R"( - A list of names of users (separated by whitespace) that are allowed - to connect to the Nix daemon. As with the `trusted-users` option, - you can specify groups by prefixing them with `@`. Also, you can - allow all users by specifying `*`. The default is `*`. + A list user names, separated by whitespace. + These users are allowed to connect to the Nix daemon. - Note that trusted users are always allowed to connect. + As with the [`trusted-users`](#conf-trusted-users) option, you can specify groups by prefixing names with `@`. + Also, you can allow all users by specifying `*`. + + > **Note** + > + > Trusted users are always allowed to connect to the Nix daemon. )"}; }; From b7d47e1d22e7ce2785487d325cc3dd35a43f16b5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 15 Jun 2023 04:58:07 +0200 Subject: [PATCH 016/909] fix wording --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index b0c025c8e..f63ec8b50 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -713,7 +713,7 @@ public: A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace. These are not used by default, but can be enabled by users of the Nix daemon by specifying [`substituters`](#conf-substituters). - Unprivileged users are only allowed to pass as `substituters` only those URLs listed in `trusted-substituters`. + Unprivileged users are allowed to pass as `substituters` only those URLs listed in `trusted-substituters`. )", {"trusted-binary-caches"}}; From e1fa48f17c0bb0f73e97e684077cec8dfa1d7a3d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 15 Jun 2023 07:41:37 -0400 Subject: [PATCH 017/909] Update src/nix/daemon.cc Co-authored-by: Valentin Gagarin --- src/nix/daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 09adab5d3..a57070c56 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -81,7 +81,7 @@ struct AuthorizationSettings : Config { > **Note** > - > Trusted users are always allowed to connect to the Nix daemon. + > Trusted users (set in [`trusted-users`](#conf-trusted-users)) can always to connect to the Nix daemon. )"}; }; From 2ceacce484e21ac116a79c74877327355fd153d0 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 15 Jun 2023 15:57:54 +0200 Subject: [PATCH 018/909] Update src/libstore/globals.hh --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index f63ec8b50..46147a5e1 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -713,7 +713,7 @@ public: A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace. These are not used by default, but can be enabled by users of the Nix daemon by specifying [`substituters`](#conf-substituters). - Unprivileged users are allowed to pass as `substituters` only those URLs listed in `trusted-substituters`. + Unprivileged users (those set in only [`allowed-users`](#conf-allowed-users) but not [`trusted-users`](#conf-trusted-users)) can pass as `substituters` only those URLs listed in `trusted-substituters`. )", {"trusted-binary-caches"}}; From 4a33d5fe3549137bacf9373e5ba7bfe11a421099 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 16 Jun 2023 14:33:31 +0200 Subject: [PATCH 019/909] fix link text Co-authored-by: Robert Hensing --- src/nix/daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index a57070c56..d0e70a7b1 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -56,7 +56,7 @@ struct AuthorizationSettings : Config { this, {"root"}, "trusted-users", R"( A list of user names, separated by whitespace. - These users will have additional rights when connecting to the Nix daemon, such as the ability to specify additional [substituters](#conf-substituters), or to import unsigned [NAR](@docroot@/glossary.md#gloss-nar)s. + These users will have additional rights when connecting to the Nix daemon, such as the ability to specify additional [substituters](#conf-substituters), or to import unsigned [NARs](@docroot@/glossary.md#gloss-nar). You can also specify groups by prefixing names with `@`. For instance, `@wheel` means all users in the `wheel` group. From 1a8ca85d488ddacf26f2aeddddab926c0e081d98 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 16 Jun 2023 14:34:11 +0200 Subject: [PATCH 020/909] use "store URLs" consistently --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 46147a5e1..d2efd1505 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -710,7 +710,7 @@ public: Setting trustedSubstituters{ this, {}, "trusted-substituters", R"( - A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace. + A list of [Nix store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace. These are not used by default, but can be enabled by users of the Nix daemon by specifying [`substituters`](#conf-substituters). Unprivileged users (those set in only [`allowed-users`](#conf-allowed-users) but not [`trusted-users`](#conf-trusted-users)) can pass as `substituters` only those URLs listed in `trusted-substituters`. From 126eea48e300ab365c46ce062776e74a3907a7c8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 16 Jun 2023 14:36:53 +0200 Subject: [PATCH 021/909] do not refer to `trusted-users` another time --- src/nix/daemon.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index d0e70a7b1..43d2f8f86 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -76,7 +76,8 @@ struct AuthorizationSettings : Config { A list user names, separated by whitespace. These users are allowed to connect to the Nix daemon. - As with the [`trusted-users`](#conf-trusted-users) option, you can specify groups by prefixing names with `@`. + You can specify groups by prefixing names with `@`. + For instance, `@wheel` means all users in the `wheel` group. Also, you can allow all users by specifying `*`. > **Note** From baef05e6fefb46b3bdddb2785861bd7190920506 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 16 Jun 2023 14:37:08 +0200 Subject: [PATCH 022/909] fix typo Co-authored-by: Robert Hensing --- src/nix/daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 43d2f8f86..6dcd7f94d 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -82,7 +82,7 @@ struct AuthorizationSettings : Config { > **Note** > - > Trusted users (set in [`trusted-users`](#conf-trusted-users)) can always to connect to the Nix daemon. + > Trusted users (set in [`trusted-users`](#conf-trusted-users)) can always connect to the Nix daemon. )"}; }; From f695a74751c314cc426ff7bbc67ce5de8b58bbfd Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 16 Jun 2023 17:58:01 +0200 Subject: [PATCH 023/909] Update src/libstore/globals.hh --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d2efd1505..05aa8288a 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -711,7 +711,7 @@ public: this, {}, "trusted-substituters", R"( A list of [Nix store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace. - These are not used by default, but can be enabled by users of the Nix daemon by specifying [`substituters`](#conf-substituters). + These are not used by default, but users of the Nix daemon can enable them by specifying [`substituters`](#conf-substituters). Unprivileged users (those set in only [`allowed-users`](#conf-allowed-users) but not [`trusted-users`](#conf-trusted-users)) can pass as `substituters` only those URLs listed in `trusted-substituters`. )", From c8825e9d8c3fa802811f3829d055e3ef9aae90e2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 16 Jun 2023 15:19:14 -0400 Subject: [PATCH 024/909] Create nlohmann serializers for `std::optional` and use This is somewhat tricky. --- src/libutil/args.cc | 15 ++---- src/libutil/experimental-features.hh | 8 ++- src/libutil/json-utils.cc | 19 +++++++ src/libutil/json-utils.hh | 80 +++++++++++++++++++++++----- 4 files changed, 97 insertions(+), 25 deletions(-) create mode 100644 src/libutil/json-utils.cc diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 081dbeb28..3cf3ed9ca 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -1,10 +1,9 @@ #include "args.hh" #include "hash.hh" +#include "json-utils.hh" #include -#include - namespace nix { void Args::addFlag(Flag && flag_) @@ -247,11 +246,7 @@ nlohmann::json Args::toJSON() j["arity"] = flag->handler.arity; if (!flag->labels.empty()) j["labels"] = flag->labels; - // TODO With C++23 use `std::optional::tranform` - if (auto & xp = flag->experimentalFeature) - j["experimental-feature"] = showExperimentalFeature(*xp); - else - j["experimental-feature"] = nullptr; + j["experimental-feature"] = flag->experimentalFeature; flags[name] = std::move(j); } @@ -416,11 +411,7 @@ nlohmann::json MultiCommand::toJSON() cat["id"] = command->category(); cat["description"] = trim(categories[command->category()]); j["category"] = std::move(cat); - // TODO With C++23 use `std::optional::tranform` - if (auto xp = command->experimentalFeature()) - cat["experimental-feature"] = showExperimentalFeature(*xp); - else - cat["experimental-feature"] = nullptr; + cat["experimental-feature"] = command->experimentalFeature(); cmds[name] = std::move(j); } diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 892c6c371..15ff5e0cd 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -3,7 +3,7 @@ #include "comparator.hh" #include "error.hh" -#include "nlohmann/json_fwd.hpp" +#include "json-utils.hh" #include "types.hh" namespace nix { @@ -93,4 +93,10 @@ public: void to_json(nlohmann::json &, const ExperimentalFeature &); void from_json(const nlohmann::json &, ExperimentalFeature &); +/** + * It is always rendered as a string + */ +template<> +struct json_avoids_null : std::true_type {}; + } diff --git a/src/libutil/json-utils.cc b/src/libutil/json-utils.cc new file mode 100644 index 000000000..d7220e71d --- /dev/null +++ b/src/libutil/json-utils.cc @@ -0,0 +1,19 @@ +#include "json-utils.hh" + +namespace nix { + +const nlohmann::json * get(const nlohmann::json & map, const std::string & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &*i; +} + +nlohmann::json * get(nlohmann::json & map, const std::string & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &*i; +} + +} diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh index eb00e954f..5e63c1af4 100644 --- a/src/libutil/json-utils.hh +++ b/src/libutil/json-utils.hh @@ -2,21 +2,77 @@ ///@file #include +#include namespace nix { -const nlohmann::json * get(const nlohmann::json & map, const std::string & key) -{ - auto i = map.find(key); - if (i == map.end()) return nullptr; - return &*i; -} +const nlohmann::json * get(const nlohmann::json & map, const std::string & key); -nlohmann::json * get(nlohmann::json & map, const std::string & key) -{ - auto i = map.find(key); - if (i == map.end()) return nullptr; - return &*i; -} +nlohmann::json * get(nlohmann::json & map, const std::string & key); + +/** + * For `adl_serializer>` below, we need to track what + * types are not already using `null`. Only for them can we use `null` + * to represent `std::nullopt`. + */ +template +struct json_avoids_null; + +/** + * Handle numbers in default impl + */ +template +struct json_avoids_null : std::bool_constant::value> {}; + +template<> +struct json_avoids_null : std::false_type {}; + +template<> +struct json_avoids_null : std::true_type {}; + +template<> +struct json_avoids_null : std::true_type {}; + +template +struct json_avoids_null> : std::true_type {}; + +template +struct json_avoids_null> : std::true_type {}; + +template +struct json_avoids_null> : std::true_type {}; + +} + +namespace nlohmann { + +/** + * This "instance" is widely requested, see + * https://github.com/nlohmann/json/issues/1749, but momentum has stalled + * out. Writing there here in Nix as a stop-gap. + * + * We need to make sure the underlying type does not use `null` for this to + * round trip. We do that with a static assert. + */ +template +struct adl_serializer> { + static std::optional from_json(const json & json) { + static_assert( + nix::json_avoids_null::value, + "null is already in use for underlying type's JSON"); + return json.is_null() + ? std::nullopt + : std::optional { adl_serializer::from_json(json) }; + } + static void to_json(json & json, std::optional t) { + static_assert( + nix::json_avoids_null::value, + "null is already in use for underlying type's JSON"); + if (t) + adl_serializer::to_json(json, *t); + else + json = nullptr; + } +}; } From d2ce2e89b178e7e7467cf4c1e572704a4c2ca75e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 19 May 2023 10:56:59 -0400 Subject: [PATCH 025/909] Split `OptionalPathSetting` from `PathSetting` Rather than doing `allowEmpty` as boolean, have separate types and use `std::optional`. This makes it harder to forget the possibility of an empty path. The `build-hook` setting was categorized as a `PathSetting`, but actually it was split into arguments. No good! Now, it is `Setting` which actually reflects what it means and how it is used. Because of the subtyping, we now also have support for `Setting>` in general. I imagine this can be used to clean up many more settings also. --- src/libstore/build/hook-instance.cc | 6 +- src/libstore/build/local-derivation-goal.cc | 8 ++- src/libstore/globals.cc | 5 +- src/libstore/globals.hh | 6 +- src/libstore/local-fs-store.hh | 14 ++--- src/libstore/store-api.hh | 2 +- src/libutil/abstract-setting-to-json.hh | 1 + src/libutil/config-impl.hh | 61 +++++++++++++++++++- src/libutil/config.cc | 63 ++++++++------------- src/libutil/config.hh | 31 ++++++++-- 10 files changed, 134 insertions(+), 63 deletions(-) diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index 075ad554f..337c60bd4 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -5,14 +5,14 @@ namespace nix { HookInstance::HookInstance() { - debug("starting build hook '%s'", settings.buildHook); + debug("starting build hook '%s'", concatStringsSep(" ", settings.buildHook.get())); - auto buildHookArgs = tokenizeString>(settings.buildHook.get()); + auto buildHookArgs = settings.buildHook.get(); if (buildHookArgs.empty()) throw Error("'build-hook' setting is empty"); - auto buildHook = buildHookArgs.front(); + auto buildHook = canonPath(buildHookArgs.front()); buildHookArgs.pop_front(); Strings args; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 7f87cdf55..cf44779b7 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -65,8 +65,9 @@ void handleDiffHook( const Path & tryA, const Path & tryB, const Path & drvPath, const Path & tmpDir) { - auto diffHook = settings.diffHook; - if (diffHook != "" && settings.runDiffHook) { + auto & diffHookOpt = settings.diffHook.get(); + if (diffHookOpt && settings.runDiffHook) { + auto & diffHook = *diffHookOpt; try { auto diffRes = runProgram(RunOptions { .program = diffHook, @@ -1428,7 +1429,8 @@ void LocalDerivationGoal::startDaemon() Store::Params params; params["path-info-cache-size"] = "0"; params["store"] = worker.store.storeDir; - params["root"] = getLocalStore().rootDir; + if (auto & optRoot = getLocalStore().rootDir.get()) + params["root"] = *optRoot; params["state"] = "/no-such-path"; params["log"] = "/no-such-path"; auto store = make_ref(params, diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index d53377239..5a4cb1824 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -100,7 +100,10 @@ Settings::Settings() if (!pathExists(nixExePath)) { nixExePath = getSelfExe().value_or("nix"); } - buildHook = nixExePath + " __build-remote"; + buildHook = { + nixExePath, + "__build-remote", + }; } void loadConfFile() diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 820898350..b8dcf1f76 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -236,7 +236,7 @@ public: )", {"build-timeout"}}; - PathSetting buildHook{this, true, "", "build-hook", + Setting buildHook{this, {}, "build-hook", R"( The path to the helper program that executes remote builds. @@ -556,8 +556,8 @@ public: line. )"}; - PathSetting diffHook{ - this, true, "", "diff-hook", + OptionalPathSetting diffHook{ + this, std::nullopt, "diff-hook", R"( Absolute path to an executable capable of diffing build results. The hook is executed if `run-diff-hook` is true, and the diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index a03bb88f5..2ee2ef0c8 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -15,22 +15,22 @@ struct LocalFSStoreConfig : virtual StoreConfig // it to omit the call to the Setting constructor. Clang works fine // either way. - const PathSetting rootDir{(StoreConfig*) this, true, "", + const OptionalPathSetting rootDir{(StoreConfig*) this, std::nullopt, "root", "Directory prefixed to all other paths."}; - const PathSetting stateDir{(StoreConfig*) this, false, - rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, + const PathSetting stateDir{(StoreConfig*) this, + rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir, "state", "Directory where Nix will store state."}; - const PathSetting logDir{(StoreConfig*) this, false, - rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, + const PathSetting logDir{(StoreConfig*) this, + rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir, "log", "directory where Nix will store log files."}; - const PathSetting realStoreDir{(StoreConfig*) this, false, - rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", + const PathSetting realStoreDir{(StoreConfig*) this, + rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real", "Physical path of the Nix store."}; }; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 2ecbe2708..14a862eef 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -114,7 +114,7 @@ struct StoreConfig : public Config return ""; } - const PathSetting storeDir_{this, false, settings.nixStore, + const PathSetting storeDir_{this, settings.nixStore, "store", R"( Logical location of the Nix store, usually diff --git a/src/libutil/abstract-setting-to-json.hh b/src/libutil/abstract-setting-to-json.hh index 7b6c3fcb5..d506dfb74 100644 --- a/src/libutil/abstract-setting-to-json.hh +++ b/src/libutil/abstract-setting-to-json.hh @@ -3,6 +3,7 @@ #include #include "config.hh" +#include "json-utils.hh" namespace nix { template diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index b6cae5ec3..14822050e 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -50,8 +50,11 @@ template<> void BaseSetting>::appendOrSet(std::set template void BaseSetting::appendOrSet(T && newValue, bool append) { - static_assert(!trait::appendable, "using default `appendOrSet` implementation with an appendable type"); + static_assert( + !trait::appendable, + "using default `appendOrSet` implementation with an appendable type"); assert(!append); + value = std::move(newValue); } @@ -68,4 +71,60 @@ void BaseSetting::set(const std::string & str, bool append) } } +template<> void BaseSetting::convertToArg(Args & args, const std::string & category); + +template +void BaseSetting::convertToArg(Args & args, const std::string & category) +{ + args.addFlag({ + .longName = name, + .description = fmt("Set the `%s` setting.", name), + .category = category, + .labels = {"value"}, + .handler = {[this](std::string s) { overridden = true; set(s); }}, + .experimentalFeature = experimentalFeature, + }); + + if (isAppendable()) + args.addFlag({ + .longName = "extra-" + name, + .description = fmt("Append to the `%s` setting.", name), + .category = category, + .labels = {"value"}, + .handler = {[this](std::string s) { overridden = true; set(s, true); }}, + .experimentalFeature = experimentalFeature, + }); +} + +#define DECLARE_CONFIG_SERIALISER(TY) \ + template<> TY BaseSetting< TY >::parse(const std::string & str) const; \ + template<> std::string BaseSetting< TY >::to_string() const; + +DECLARE_CONFIG_SERIALISER(std::string) +DECLARE_CONFIG_SERIALISER(std::optional) +DECLARE_CONFIG_SERIALISER(bool) +DECLARE_CONFIG_SERIALISER(Strings) +DECLARE_CONFIG_SERIALISER(StringSet) +DECLARE_CONFIG_SERIALISER(StringMap) +DECLARE_CONFIG_SERIALISER(std::set) + +template +T BaseSetting::parse(const std::string & str) const +{ + static_assert(std::is_integral::value, "Integer required."); + + if (auto n = string2Int(str)) + return *n; + else + throw UsageError("setting '%s' has invalid value '%s'", name, str); +} + +template +std::string BaseSetting::to_string() const +{ + static_assert(std::is_integral::value, "Integer required."); + + return std::to_string(value); +} + } diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 085a884dc..38d406e8a 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -219,29 +219,6 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category) { } -template -void BaseSetting::convertToArg(Args & args, const std::string & category) -{ - args.addFlag({ - .longName = name, - .description = fmt("Set the `%s` setting.", name), - .category = category, - .labels = {"value"}, - .handler = {[this](std::string s) { overridden = true; set(s); }}, - .experimentalFeature = experimentalFeature, - }); - - if (isAppendable()) - args.addFlag({ - .longName = "extra-" + name, - .description = fmt("Append to the `%s` setting.", name), - .category = category, - .labels = {"value"}, - .handler = {[this](std::string s) { overridden = true; set(s, true); }}, - .experimentalFeature = experimentalFeature, - }); -} - template<> std::string BaseSetting::parse(const std::string & str) const { return str; @@ -252,21 +229,17 @@ template<> std::string BaseSetting::to_string() const return value; } -template -T BaseSetting::parse(const std::string & str) const +template<> std::optional BaseSetting>::parse(const std::string & str) const { - static_assert(std::is_integral::value, "Integer required."); - if (auto n = string2Int(str)) - return *n; + if (str == "") + return std::nullopt; else - throw UsageError("setting '%s' has invalid value '%s'", name, str); + return { str }; } -template -std::string BaseSetting::to_string() const +template<> std::string BaseSetting>::to_string() const { - static_assert(std::is_integral::value, "Integer required."); - return std::to_string(value); + return value ? *value : ""; } template<> bool BaseSetting::parse(const std::string & str) const @@ -403,15 +376,25 @@ template class BaseSetting; template class BaseSetting; template class BaseSetting>; +static Path parsePath(const AbstractSetting & s, const std::string & str) +{ + if (str == "") + throw UsageError("setting '%s' is a path and paths cannot be empty", s.name); + else + return canonPath(str); +} + Path PathSetting::parse(const std::string & str) const { - if (str == "") { - if (allowEmpty) - return ""; - else - throw UsageError("setting '%s' cannot be empty", name); - } else - return canonPath(str); + return parsePath(*this, str); +} + +std::optional OptionalPathSetting::parse(const std::string & str) const +{ + if (str == "") + return std::nullopt; + else + return parsePath(*this, str); } bool GlobalConfig::set(const std::string & name, const std::string & value) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 2675baed7..cc8532587 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -353,21 +353,20 @@ public: /** * A special setting for Paths. These are automatically canonicalised * (e.g. "/foo//bar/" becomes "/foo/bar"). + * + * It is mandatory to specify a path; i.e. the empty string is not + * permitted. */ class PathSetting : public BaseSetting { - bool allowEmpty; - public: PathSetting(Config * options, - bool allowEmpty, const Path & def, const std::string & name, const std::string & description, const std::set & aliases = {}) : BaseSetting(def, true, name, description, aliases) - , allowEmpty(allowEmpty) { options->addSetting(this); } @@ -379,6 +378,30 @@ public: void operator =(const Path & v) { this->assign(v); } }; +/** + * Like `PathSetting`, but the absence of a path is also allowed. + * + * `std::optional` is used instead of the empty string for clarity. + */ +class OptionalPathSetting : public BaseSetting> +{ +public: + + OptionalPathSetting(Config * options, + const std::optional & def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : BaseSetting>(def, true, name, description, aliases) + { + options->addSetting(this); + } + + std::optional parse(const std::string & str) const override; + + void operator =(const std::optional & v) { this->assign(v); } +}; + struct GlobalConfig : public AbstractConfig { typedef std::vector ConfigRegistrations; From 469d06f9bc9b2543f48d8e633e47f9344fba35ab Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 8 Mar 2022 21:53:26 +0000 Subject: [PATCH 026/909] Split out worker protocol template definitions from declarations This is generally a fine practice: Putting implementations in headers makes them harder to read and slows compilation. Unfortunately it is necessary for templates, but we can ameliorate that by putting them in a separate header. Only files which need to instantiate those templates will need to include the header with the implementation; the rest can just include the declaration. This is now documenting in the contributing guide. Also, it just happens that these polymorphic serializers are the protocol agnostic ones. (Worker and serve protocol have the same logic for these container types.) This means by doing this general template cleanup, we are also getting a head start on better indicating which code is protocol-specific and which code is shared between protocols. --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/contributing/cxx.md | 28 ++++++++ src/libstore/build/derivation-goal.cc | 1 + src/libstore/build/local-derivation-goal.cc | 1 + src/libstore/daemon.cc | 1 + src/libstore/derivations.cc | 1 + src/libstore/export-import.cc | 1 + src/libstore/legacy-ssh-store.cc | 1 + src/libstore/path-info.cc | 1 + src/libstore/remote-store.cc | 1 + src/libstore/worker-protocol-impl.hh | 78 +++++++++++++++++++++ src/libstore/worker-protocol.cc | 1 + src/libstore/worker-protocol.hh | 63 ----------------- src/libutil/config-impl.hh | 3 + src/nix-store/nix-store.cc | 1 + 15 files changed, 120 insertions(+), 63 deletions(-) create mode 100644 doc/manual/src/contributing/cxx.md create mode 100644 src/libstore/worker-protocol-impl.hh diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 13d2e4d15..bbb603713 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -106,6 +106,7 @@ - [Hacking](contributing/hacking.md) - [Experimental Features](contributing/experimental-features.md) - [CLI guideline](contributing/cli-guideline.md) + - [C++ style guide](contributing/cxx.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md) diff --git a/doc/manual/src/contributing/cxx.md b/doc/manual/src/contributing/cxx.md new file mode 100644 index 000000000..ff9297158 --- /dev/null +++ b/doc/manual/src/contributing/cxx.md @@ -0,0 +1,28 @@ +# C++ style guide + +Some miscellaneous notes on how we write C++. +Formatting we hope to eventually normalize automatically, so this section is free to just discuss higher-level concerns. + +## The `*-impl.hh` pattern + +Let's start with some background info first. +Headers, are supposed to contain declarations, not definitions. +This allows us to change a definition without changing the declaration, and have a very small rebuild during development. +Templates, however, need to be specialized to use-sites. +Absent fancier techniques, templates require that the definition, not just mere declaration, must be available at use-sites in order to make that specialization on the fly as part of compiling those use-sites. +Making definitions available like that means putting them in headers, but that is unfortunately means we get all the extra rebuilds we want to avoid by just putting declarations there as described above. + +The `*-impl.hh` pattern is a ham-fisted partial solution to this problem. +It constitutes: + +- Declaring items only in the main `foo.hh`, including templates. + +- Putting template definitions in a companion `foo-impl.hh` header. + +Most C++ developers would accompany this by having `foo.hh` include `foo-impl.hh`, to ensure any file getting the template declarations also got the template definitions. +But we've found not doing this has some benefits and fewer than imagined downsides. +The fact remains that headers are rarely as minimal as they could be; +there is often code that needs declarations from the headers but not the templates within them. +With our pattern where `foo.hh` doesn't include `foo-impl.hh`, that means they can just include `foo.hh` +Code that needs both just includes `foo.hh` and `foo-impl.hh`. +This does make linking error possible where something forgets to include `foo-impl.hh` that needs it, but those are build-time only as easy to fix. diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index df7d21e54..e02000c8b 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -9,6 +9,7 @@ #include "archive.hh" #include "compression.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "topo-sort.hh" #include "callback.hh" #include "local-store.hh" // TODO remove, along with remaining downcasts diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 7f87cdf55..6e06f6061 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -11,6 +11,7 @@ #include "compression.hh" #include "daemon.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "topo-sort.hh" #include "callback.hh" #include "json-utils.hh" diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index b6dd83684..54b089b30 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1,6 +1,7 @@ #include "daemon.hh" #include "monitor-fd.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "build-result.hh" #include "store-api.hh" #include "store-cast.hh" diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index ccb165d68..4a1f1e99f 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -5,6 +5,7 @@ #include "util.hh" #include "split.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "fs-accessor.hh" #include #include diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 5ea263a86..72ad77124 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -2,6 +2,7 @@ #include "store-api.hh" #include "archive.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 2b7bebe9d..bff024f96 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -7,6 +7,7 @@ #include "store-api.hh" #include "path-with-outputs.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "ssh.hh" #include "derivations.hh" #include "callback.hh" diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 97b72faa3..6b93976fa 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -1,5 +1,6 @@ #include "path-info.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "store-api.hh" namespace nix { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index c3dfb5979..f02342d32 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -6,6 +6,7 @@ #include "build-result.hh" #include "remote-store.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "archive.hh" #include "globals.hh" #include "derivations.hh" diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh new file mode 100644 index 000000000..509a8a5d0 --- /dev/null +++ b/src/libstore/worker-protocol-impl.hh @@ -0,0 +1,78 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * This file is an exmample of the "impl.hh" pattern. See the + * contributing guide. + */ + +#include "worker-protocol.hh" + +namespace nix { + +template +std::vector WorkerProto>::read(const Store & store, Source & from) +{ + std::vector resSet; + auto size = readNum(from); + while (size--) { + resSet.push_back(WorkerProto::read(store, from)); + } + return resSet; +} + +template +void WorkerProto>::write(const Store & store, Sink & out, const std::vector & resSet) +{ + out << resSet.size(); + for (auto & key : resSet) { + WorkerProto::write(store, out, key); + } +} + +template +std::set WorkerProto>::read(const Store & store, Source & from) +{ + std::set resSet; + auto size = readNum(from); + while (size--) { + resSet.insert(WorkerProto::read(store, from)); + } + return resSet; +} + +template +void WorkerProto>::write(const Store & store, Sink & out, const std::set & resSet) +{ + out << resSet.size(); + for (auto & key : resSet) { + WorkerProto::write(store, out, key); + } +} + +template +std::map WorkerProto>::read(const Store & store, Source & from) +{ + std::map resMap; + auto size = readNum(from); + while (size--) { + auto k = WorkerProto::read(store, from); + auto v = WorkerProto::read(store, from); + resMap.insert_or_assign(std::move(k), std::move(v)); + } + return resMap; +} + +template +void WorkerProto>::write(const Store & store, Sink & out, const std::map & resMap) +{ + out << resMap.size(); + for (auto & i : resMap) { + WorkerProto::write(store, out, i.first); + WorkerProto::write(store, out, i.second); + } +} + +} diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 51bb12026..49708b642 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -4,6 +4,7 @@ #include "store-api.hh" #include "build-result.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "archive.hh" #include "derivations.hh" diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index f06332d17..18a4e11b2 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -173,67 +173,4 @@ MAKE_WORKER_PROTO(std::optional); template<> MAKE_WORKER_PROTO(std::optional); -template -std::vector WorkerProto>::read(const Store & store, Source & from) -{ - std::vector resSet; - auto size = readNum(from); - while (size--) { - resSet.push_back(WorkerProto::read(store, from)); - } - return resSet; -} - -template -void WorkerProto>::write(const Store & store, Sink & out, const std::vector & resSet) -{ - out << resSet.size(); - for (auto & key : resSet) { - WorkerProto::write(store, out, key); - } -} - -template -std::set WorkerProto>::read(const Store & store, Source & from) -{ - std::set resSet; - auto size = readNum(from); - while (size--) { - resSet.insert(WorkerProto::read(store, from)); - } - return resSet; -} - -template -void WorkerProto>::write(const Store & store, Sink & out, const std::set & resSet) -{ - out << resSet.size(); - for (auto & key : resSet) { - WorkerProto::write(store, out, key); - } -} - -template -std::map WorkerProto>::read(const Store & store, Source & from) -{ - std::map resMap; - auto size = readNum(from); - while (size--) { - auto k = WorkerProto::read(store, from); - auto v = WorkerProto::read(store, from); - resMap.insert_or_assign(std::move(k), std::move(v)); - } - return resMap; -} - -template -void WorkerProto>::write(const Store & store, Sink & out, const std::map & resMap) -{ - out << resMap.size(); - for (auto & i : resMap) { - WorkerProto::write(store, out, i.first); - WorkerProto::write(store, out, i.second); - } -} - } diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index b6cae5ec3..d9726a907 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -4,6 +4,9 @@ * * Template implementations (as opposed to mere declarations). * + * This file is an exmample of the "impl.hh" pattern. See the + * contributing guide. + * * One only needs to include this when one is declaring a * `BaseClass` setting, or as derived class of such an * instantiation. diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 61c189efb..94368e415 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -12,6 +12,7 @@ #include "shared.hh" #include "util.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "graphml.hh" #include "legacy.hh" #include "path-with-outputs.hh" From 95eae0c002da5ef35e583b46a67818ebb95d3a1e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 26 May 2023 11:07:25 -0400 Subject: [PATCH 027/909] Put worker protocol items inside a `WorkerProto` struct See API docs on that struct for why. The pasing as as template argument doesn't yet happen in that commit, but will instead happen in later commit. Also make `WorkerOp` (now `Op`) and enum struct. This led us to catch that two operations were not handled! Co-authored-by: Robert Hensing --- src/build-remote/build-remote.cc | 2 +- src/libstore/build/derivation-goal.cc | 4 +- src/libstore/daemon.cc | 148 ++++++++++---------- src/libstore/derivations.cc | 4 +- src/libstore/export-import.cc | 4 +- src/libstore/legacy-ssh-store.cc | 16 +-- src/libstore/path-info.cc | 4 +- src/libstore/remote-store.cc | 130 +++++++++--------- src/libstore/worker-protocol-impl.hh | 28 ++-- src/libstore/worker-protocol.cc | 56 ++++---- src/libstore/worker-protocol.hh | 186 ++++++++++++++++---------- src/nix-store/nix-store.cc | 18 +-- 12 files changed, 327 insertions(+), 273 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 323e04fdb..2fb17d06f 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -299,7 +299,7 @@ connected: !trusted || *trusted; }); - // See the very large comment in `case wopBuildDerivation:` in + // See the very large comment in `case WorkerProto::Op::BuildDerivation:` in // `src/libstore/daemon.cc` that explains the trust model here. // // This condition mirrors that: that code enforces the "rules" outlined there; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index e02000c8b..1a946e3d2 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1153,7 +1153,7 @@ HookReply DerivationGoal::tryBuildHook() /* Tell the hook all the inputs that have to be copied to the remote system. */ - workerProtoWrite(worker.store, hook->sink, inputPaths); + WorkerProto::write(worker.store, hook->sink, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ @@ -1164,7 +1164,7 @@ HookReply DerivationGoal::tryBuildHook() if (buildMode != bmCheck && status.known && status.known->isValid()) continue; missingOutputs.insert(outputName); } - workerProtoWrite(worker.store, hook->sink, missingOutputs); + WorkerProto::write(worker.store, hook->sink, missingOutputs); } hook->sink = FdSink(); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 54b089b30..7eba1a79d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -264,7 +264,7 @@ static std::vector readDerivedPaths(Store & store, unsigned int cli { std::vector reqs; if (GET_PROTOCOL_MINOR(clientVersion) >= 30) { - reqs = WorkerProto>::read(store, from); + reqs = WorkerProto::Serialise>::read(store, from); } else { for (auto & s : readStrings(from)) reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath()); @@ -274,11 +274,11 @@ static std::vector readDerivedPaths(Store & store, unsigned int cli static void performOp(TunnelLogger * logger, ref store, TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, - Source & from, BufferedSink & to, unsigned int op) + Source & from, BufferedSink & to, WorkerProto::Op op) { switch (op) { - case wopIsValidPath: { + case WorkerProto::Op::IsValidPath: { auto path = store->parseStorePath(readString(from)); logger->startWork(); bool result = store->isValidPath(path); @@ -287,8 +287,8 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopQueryValidPaths: { - auto paths = WorkerProto::read(*store, from); + case WorkerProto::Op::QueryValidPaths: { + auto paths = WorkerProto::Serialise::read(*store, from); SubstituteFlag substitute = NoSubstitute; if (GET_PROTOCOL_MINOR(clientVersion) >= 27) { @@ -301,11 +301,11 @@ static void performOp(TunnelLogger * logger, ref store, } auto res = store->queryValidPaths(paths, substitute); logger->stopWork(); - workerProtoWrite(*store, to, res); + WorkerProto::write(*store, to, res); break; } - case wopHasSubstitutes: { + case WorkerProto::Op::HasSubstitutes: { auto path = store->parseStorePath(readString(from)); logger->startWork(); StorePathSet paths; // FIXME @@ -316,16 +316,16 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopQuerySubstitutablePaths: { - auto paths = WorkerProto::read(*store, from); + case WorkerProto::Op::QuerySubstitutablePaths: { + auto paths = WorkerProto::Serialise::read(*store, from); logger->startWork(); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - workerProtoWrite(*store, to, res); + WorkerProto::write(*store, to, res); break; } - case wopQueryPathHash: { + case WorkerProto::Op::QueryPathHash: { auto path = store->parseStorePath(readString(from)); logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; @@ -334,27 +334,27 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopQueryReferences: - case wopQueryReferrers: - case wopQueryValidDerivers: - case wopQueryDerivationOutputs: { + case WorkerProto::Op::QueryReferences: + case WorkerProto::Op::QueryReferrers: + case WorkerProto::Op::QueryValidDerivers: + case WorkerProto::Op::QueryDerivationOutputs: { auto path = store->parseStorePath(readString(from)); logger->startWork(); StorePathSet paths; - if (op == wopQueryReferences) + if (op == WorkerProto::Op::QueryReferences) for (auto & i : store->queryPathInfo(path)->references) paths.insert(i); - else if (op == wopQueryReferrers) + else if (op == WorkerProto::Op::QueryReferrers) store->queryReferrers(path, paths); - else if (op == wopQueryValidDerivers) + else if (op == WorkerProto::Op::QueryValidDerivers) paths = store->queryValidDerivers(path); else paths = store->queryDerivationOutputs(path); logger->stopWork(); - workerProtoWrite(*store, to, paths); + WorkerProto::write(*store, to, paths); break; } - case wopQueryDerivationOutputNames: { + case WorkerProto::Op::QueryDerivationOutputNames: { auto path = store->parseStorePath(readString(from)); logger->startWork(); auto names = store->readDerivation(path).outputNames(); @@ -363,16 +363,16 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopQueryDerivationOutputMap: { + case WorkerProto::Op::QueryDerivationOutputMap: { auto path = store->parseStorePath(readString(from)); logger->startWork(); auto outputs = store->queryPartialDerivationOutputMap(path); logger->stopWork(); - workerProtoWrite(*store, to, outputs); + WorkerProto::write(*store, to, outputs); break; } - case wopQueryDeriver: { + case WorkerProto::Op::QueryDeriver: { auto path = store->parseStorePath(readString(from)); logger->startWork(); auto info = store->queryPathInfo(path); @@ -381,7 +381,7 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopQueryPathFromHashPart: { + case WorkerProto::Op::QueryPathFromHashPart: { auto hashPart = readString(from); logger->startWork(); auto path = store->queryPathFromHashPart(hashPart); @@ -390,11 +390,11 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopAddToStore: { + case WorkerProto::Op::AddToStore: { if (GET_PROTOCOL_MINOR(clientVersion) >= 25) { auto name = readString(from); auto camStr = readString(from); - auto refs = WorkerProto::read(*store, from); + auto refs = WorkerProto::Serialise::read(*store, from); bool repairBool; from >> repairBool; auto repair = RepairFlag{repairBool}; @@ -476,7 +476,7 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopAddMultipleToStore: { + case WorkerProto::Op::AddMultipleToStore: { bool repair, dontCheckSigs; from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) @@ -493,10 +493,10 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopAddTextToStore: { + case WorkerProto::Op::AddTextToStore: { std::string suffix = readString(from); std::string s = readString(from); - auto refs = WorkerProto::read(*store, from); + auto refs = WorkerProto::Serialise::read(*store, from); logger->startWork(); auto path = store->addTextToStore(suffix, s, refs, NoRepair); logger->stopWork(); @@ -504,7 +504,7 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopExportPath: { + case WorkerProto::Op::ExportPath: { auto path = store->parseStorePath(readString(from)); readInt(from); // obsolete logger->startWork(); @@ -515,7 +515,7 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopImportPaths: { + case WorkerProto::Op::ImportPaths: { logger->startWork(); TunnelSource source(from, to); auto paths = store->importPaths(source, @@ -527,7 +527,7 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopBuildPaths: { + case WorkerProto::Op::BuildPaths: { auto drvs = readDerivedPaths(*store, clientVersion, from); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { @@ -552,7 +552,7 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopBuildPathsWithResults: { + case WorkerProto::Op::BuildPathsWithResults: { auto drvs = readDerivedPaths(*store, clientVersion, from); BuildMode mode = bmNormal; mode = (BuildMode) readInt(from); @@ -568,12 +568,12 @@ static void performOp(TunnelLogger * logger, ref store, auto results = store->buildPathsWithResults(drvs, mode); logger->stopWork(); - workerProtoWrite(*store, to, results); + WorkerProto::write(*store, to, results); break; } - case wopBuildDerivation: { + case WorkerProto::Op::BuildDerivation: { auto drvPath = store->parseStorePath(readString(from)); BasicDerivation drv; readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath)); @@ -645,12 +645,12 @@ static void performOp(TunnelLogger * logger, ref store, DrvOutputs builtOutputs; for (auto & [output, realisation] : res.builtOutputs) builtOutputs.insert_or_assign(realisation.id, realisation); - workerProtoWrite(*store, to, builtOutputs); + WorkerProto::write(*store, to, builtOutputs); } break; } - case wopEnsurePath: { + case WorkerProto::Op::EnsurePath: { auto path = store->parseStorePath(readString(from)); logger->startWork(); store->ensurePath(path); @@ -659,7 +659,7 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopAddTempRoot: { + case WorkerProto::Op::AddTempRoot: { auto path = store->parseStorePath(readString(from)); logger->startWork(); store->addTempRoot(path); @@ -668,7 +668,7 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopAddIndirectRoot: { + case WorkerProto::Op::AddIndirectRoot: { Path path = absPath(readString(from)); logger->startWork(); @@ -681,14 +681,14 @@ static void performOp(TunnelLogger * logger, ref store, } // Obsolete. - case wopSyncWithGC: { + case WorkerProto::Op::SyncWithGC: { logger->startWork(); logger->stopWork(); to << 1; break; } - case wopFindRoots: { + case WorkerProto::Op::FindRoots: { logger->startWork(); auto & gcStore = require(*store); Roots roots = gcStore.findRoots(!trusted); @@ -707,10 +707,10 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopCollectGarbage: { + case WorkerProto::Op::CollectGarbage: { GCOptions options; options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = WorkerProto::read(*store, from); + options.pathsToDelete = WorkerProto::Serialise::read(*store, from); from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields readInt(from); @@ -731,7 +731,7 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopSetOptions: { + case WorkerProto::Op::SetOptions: { ClientSettings clientSettings; @@ -768,7 +768,7 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopQuerySubstitutablePathInfo: { + case WorkerProto::Op::QuerySubstitutablePathInfo: { auto path = store->parseStorePath(readString(from)); logger->startWork(); SubstitutablePathInfos infos; @@ -780,22 +780,22 @@ static void performOp(TunnelLogger * logger, ref store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - workerProtoWrite(*store, to, i->second.references); + WorkerProto::write(*store, to, i->second.references); to << i->second.downloadSize << i->second.narSize; } break; } - case wopQuerySubstitutablePathInfos: { + case WorkerProto::Op::QuerySubstitutablePathInfos: { SubstitutablePathInfos infos; StorePathCAMap pathsMap = {}; if (GET_PROTOCOL_MINOR(clientVersion) < 22) { - auto paths = WorkerProto::read(*store, from); + auto paths = WorkerProto::Serialise::read(*store, from); for (auto & path : paths) pathsMap.emplace(path, std::nullopt); } else - pathsMap = WorkerProto::read(*store, from); + pathsMap = WorkerProto::Serialise::read(*store, from); logger->startWork(); store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); @@ -803,21 +803,21 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - workerProtoWrite(*store, to, i.second.references); + WorkerProto::write(*store, to, i.second.references); to << i.second.downloadSize << i.second.narSize; } break; } - case wopQueryAllValidPaths: { + case WorkerProto::Op::QueryAllValidPaths: { logger->startWork(); auto paths = store->queryAllValidPaths(); logger->stopWork(); - workerProtoWrite(*store, to, paths); + WorkerProto::write(*store, to, paths); break; } - case wopQueryPathInfo: { + case WorkerProto::Op::QueryPathInfo: { auto path = store->parseStorePath(readString(from)); std::shared_ptr info; logger->startWork(); @@ -838,14 +838,14 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopOptimiseStore: + case WorkerProto::Op::OptimiseStore: logger->startWork(); store->optimiseStore(); logger->stopWork(); to << 1; break; - case wopVerifyStore: { + case WorkerProto::Op::VerifyStore: { bool checkContents, repair; from >> checkContents >> repair; logger->startWork(); @@ -857,7 +857,7 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopAddSignatures: { + case WorkerProto::Op::AddSignatures: { auto path = store->parseStorePath(readString(from)); StringSet sigs = readStrings(from); logger->startWork(); @@ -869,7 +869,7 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopNarFromPath: { + case WorkerProto::Op::NarFromPath: { auto path = store->parseStorePath(readString(from)); logger->startWork(); logger->stopWork(); @@ -877,7 +877,7 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopAddToStoreNar: { + case WorkerProto::Op::AddToStoreNar: { bool repair, dontCheckSigs; auto path = store->parseStorePath(readString(from)); auto deriver = readString(from); @@ -885,7 +885,7 @@ static void performOp(TunnelLogger * logger, ref store, ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = WorkerProto::read(*store, from); + info.references = WorkerProto::Serialise::read(*store, from); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); info.ca = ContentAddress::parseOpt(readString(from)); @@ -929,21 +929,21 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopQueryMissing: { + case WorkerProto::Op::QueryMissing: { auto targets = readDerivedPaths(*store, clientVersion, from); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); logger->stopWork(); - workerProtoWrite(*store, to, willBuild); - workerProtoWrite(*store, to, willSubstitute); - workerProtoWrite(*store, to, unknown); + WorkerProto::write(*store, to, willBuild); + WorkerProto::write(*store, to, willSubstitute); + WorkerProto::write(*store, to, unknown); to << downloadSize << narSize; break; } - case wopRegisterDrvOutput: { + case WorkerProto::Op::RegisterDrvOutput: { logger->startWork(); if (GET_PROTOCOL_MINOR(clientVersion) < 31) { auto outputId = DrvOutput::parse(readString(from)); @@ -951,14 +951,14 @@ static void performOp(TunnelLogger * logger, ref store, store->registerDrvOutput(Realisation{ .id = outputId, .outPath = outputPath}); } else { - auto realisation = WorkerProto::read(*store, from); + auto realisation = WorkerProto::Serialise::read(*store, from); store->registerDrvOutput(realisation); } logger->stopWork(); break; } - case wopQueryRealisation: { + case WorkerProto::Op::QueryRealisation: { logger->startWork(); auto outputId = DrvOutput::parse(readString(from)); auto info = store->queryRealisation(outputId); @@ -966,16 +966,16 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) < 31) { std::set outPaths; if (info) outPaths.insert(info->outPath); - workerProtoWrite(*store, to, outPaths); + WorkerProto::write(*store, to, outPaths); } else { std::set realisations; if (info) realisations.insert(*info); - workerProtoWrite(*store, to, realisations); + WorkerProto::write(*store, to, realisations); } break; } - case wopAddBuildLog: { + case WorkerProto::Op::AddBuildLog: { StorePath path{readString(from)}; logger->startWork(); if (!trusted) @@ -992,6 +992,10 @@ static void performOp(TunnelLogger * logger, ref store, break; } + case WorkerProto::Op::QueryFailedPaths: + case WorkerProto::Op::ClearFailedPaths: + throw Error("Removed operation %1%", op); + default: throw Error("invalid operation %1%", op); } @@ -1046,7 +1050,7 @@ void processConnection( auto temp = trusted ? store->isTrustedClient() : std::optional { NotTrusted }; - workerProtoWrite(*store, to, temp); + WorkerProto::write(*store, to, temp); } /* Send startup error messages to the client. */ @@ -1059,9 +1063,9 @@ void processConnection( /* Process client requests. */ while (true) { - WorkerOp op; + WorkerProto::Op op; try { - op = (WorkerOp) readInt(from); + op = (enum WorkerProto::Op) readInt(from); } catch (Interrupted & e) { break; } catch (EndOfFile & e) { diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 4a1f1e99f..2295ff9e8 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -750,7 +750,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = WorkerProto::read(store, in); + drv.inputSrcs = WorkerProto::Serialise::read(store, in); in >> drv.platform >> drv.builder; drv.args = readStrings(in); @@ -798,7 +798,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, }, i.second.raw()); } - workerProtoWrite(store, out, drv.inputSrcs); + WorkerProto::write(store, out, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); for (auto & i : drv.env) diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 72ad77124..b4100bace 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -46,7 +46,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - workerProtoWrite(*this, teeSink, info->references); + WorkerProto::write(*this, teeSink, info->references); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -74,7 +74,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) //Activity act(*logger, lvlInfo, "importing path '%s'", info.path); - auto references = WorkerProto::read(*this, source); + auto references = WorkerProto::Serialise::read(*this, source); auto deriver = readString(source); auto narHash = hashString(htSHA256, saved.s); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index bff024f96..7d006f451 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -147,7 +147,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = WorkerProto::read(*this, conn->from); + info->references = WorkerProto::Serialise::read(*this, conn->from); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -181,7 +181,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - workerProtoWrite(*this, conn->to, info.references); + WorkerProto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize @@ -210,7 +210,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - workerProtoWrite(*this, conn->to, info.references); + WorkerProto::write(*this, conn->to, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -295,7 +295,7 @@ public: if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - auto builtOutputs = WorkerProto::read(*this, conn->from); + auto builtOutputs = WorkerProto::Serialise::read(*this, conn->from); for (auto && [output, realisation] : builtOutputs) status.builtOutputs.insert_or_assign( std::move(output.outputName), @@ -370,10 +370,10 @@ public: conn->to << cmdQueryClosure << includeOutputs; - workerProtoWrite(*this, conn->to, paths); + WorkerProto::write(*this, conn->to, paths); conn->to.flush(); - for (auto & i : WorkerProto::read(*this, conn->from)) + for (auto & i : WorkerProto::Serialise::read(*this, conn->from)) out.insert(i); } @@ -386,10 +386,10 @@ public: << cmdQueryValidPaths << false // lock << maybeSubstitute; - workerProtoWrite(*this, conn->to, paths); + WorkerProto::write(*this, conn->to, paths); conn->to.flush(); - return WorkerProto::read(*this, conn->from); + return WorkerProto::Serialise::read(*this, conn->from); } void connect() override diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 6b93976fa..9971de1dd 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -133,7 +133,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned auto narHash = Hash::parseAny(readString(source), htSHA256); ValidPathInfo info(path, narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.references = WorkerProto::read(store, source); + info.references = WorkerProto::Serialise::read(store, source); source >> info.registrationTime >> info.narSize; if (format >= 16) { source >> info.ultimate; @@ -154,7 +154,7 @@ void ValidPathInfo::write( sink << store.printStorePath(path); sink << (deriver ? store.printStorePath(*deriver) : "") << narHash.to_string(Base16, false); - workerProtoWrite(store, sink, references); + WorkerProto::write(store, sink, references); sink << registrationTime << narSize; if (format >= 16) { sink << ultimate diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f02342d32..9253ce09e 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -101,7 +101,7 @@ void RemoteStore::initConnection(Connection & conn) } if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) { - conn.remoteTrustsUs = WorkerProto>::read(*this, conn.from); + conn.remoteTrustsUs = WorkerProto::Serialise>::read(*this, conn.from); } else { // We don't know the answer; protocol to old. conn.remoteTrustsUs = std::nullopt; @@ -120,7 +120,7 @@ void RemoteStore::initConnection(Connection & conn) void RemoteStore::setOptions(Connection & conn) { - conn.to << wopSetOptions + conn.to << WorkerProto::Op::SetOptions << settings.keepFailed << settings.keepGoing << settings.tryFallback @@ -212,7 +212,7 @@ void RemoteStore::setOptions() bool RemoteStore::isValidPathUncached(const StorePath & path) { auto conn(getConnection()); - conn->to << wopIsValidPath << printStorePath(path); + conn->to << WorkerProto::Op::IsValidPath << printStorePath(path); conn.processStderr(); return readInt(conn->from); } @@ -227,13 +227,13 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute if (isValidPath(i)) res.insert(i); return res; } else { - conn->to << wopQueryValidPaths; - workerProtoWrite(*this, conn->to, paths); + conn->to << WorkerProto::Op::QueryValidPaths; + WorkerProto::write(*this, conn->to, paths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) { conn->to << (settings.buildersUseSubstitutes ? 1 : 0); } conn.processStderr(); - return WorkerProto::read(*this, conn->from); + return WorkerProto::Serialise::read(*this, conn->from); } } @@ -241,9 +241,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute StorePathSet RemoteStore::queryAllValidPaths() { auto conn(getConnection()); - conn->to << wopQueryAllValidPaths; + conn->to << WorkerProto::Op::QueryAllValidPaths; conn.processStderr(); - return WorkerProto::read(*this, conn->from); + return WorkerProto::Serialise::read(*this, conn->from); } @@ -253,16 +253,16 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { StorePathSet res; for (auto & i : paths) { - conn->to << wopHasSubstitutes << printStorePath(i); + conn->to << WorkerProto::Op::HasSubstitutes << printStorePath(i); conn.processStderr(); if (readInt(conn->from)) res.insert(i); } return res; } else { - conn->to << wopQuerySubstitutablePaths; - workerProtoWrite(*this, conn->to, paths); + conn->to << WorkerProto::Op::QuerySubstitutablePaths; + WorkerProto::write(*this, conn->to, paths); conn.processStderr(); - return WorkerProto::read(*this, conn->from); + return WorkerProto::Serialise::read(*this, conn->from); } } @@ -277,14 +277,14 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S for (auto & i : pathsMap) { SubstitutablePathInfo info; - conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first); + conn->to << WorkerProto::Op::QuerySubstitutablePathInfo << printStorePath(i.first); conn.processStderr(); unsigned int reply = readInt(conn->from); if (reply == 0) continue; auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = WorkerProto::read(*this, conn->from); + info.references = WorkerProto::Serialise::read(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -292,14 +292,14 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S } else { - conn->to << wopQuerySubstitutablePathInfos; + conn->to << WorkerProto::Op::QuerySubstitutablePathInfos; if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { StorePathSet paths; for (auto & path : pathsMap) paths.insert(path.first); - workerProtoWrite(*this, conn->to, paths); + WorkerProto::write(*this, conn->to, paths); } else - workerProtoWrite(*this, conn->to, pathsMap); + WorkerProto::write(*this, conn->to, pathsMap); conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { @@ -307,7 +307,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = WorkerProto::read(*this, conn->from); + info.references = WorkerProto::Serialise::read(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -323,7 +323,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, std::shared_ptr info; { auto conn(getConnection()); - conn->to << wopQueryPathInfo << printStorePath(path); + conn->to << WorkerProto::Op::QueryPathInfo << printStorePath(path); try { conn.processStderr(); } catch (Error & e) { @@ -348,9 +348,9 @@ void RemoteStore::queryReferrers(const StorePath & path, StorePathSet & referrers) { auto conn(getConnection()); - conn->to << wopQueryReferrers << printStorePath(path); + conn->to << WorkerProto::Op::QueryReferrers << printStorePath(path); conn.processStderr(); - for (auto & i : WorkerProto::read(*this, conn->from)) + for (auto & i : WorkerProto::Serialise::read(*this, conn->from)) referrers.insert(i); } @@ -358,9 +358,9 @@ void RemoteStore::queryReferrers(const StorePath & path, StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) { auto conn(getConnection()); - conn->to << wopQueryValidDerivers << printStorePath(path); + conn->to << WorkerProto::Op::QueryValidDerivers << printStorePath(path); conn.processStderr(); - return WorkerProto::read(*this, conn->from); + return WorkerProto::Serialise::read(*this, conn->from); } @@ -370,9 +370,9 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) return Store::queryDerivationOutputs(path); } auto conn(getConnection()); - conn->to << wopQueryDerivationOutputs << printStorePath(path); + conn->to << WorkerProto::Op::QueryDerivationOutputs << printStorePath(path); conn.processStderr(); - return WorkerProto::read(*this, conn->from); + return WorkerProto::Serialise::read(*this, conn->from); } @@ -380,9 +380,9 @@ std::map> RemoteStore::queryPartialDerivat { if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { auto conn(getConnection()); - conn->to << wopQueryDerivationOutputMap << printStorePath(path); + conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path); conn.processStderr(); - return WorkerProto>>::read(*this, conn->from); + return WorkerProto::Serialise>>::read(*this, conn->from); } else { // Fallback for old daemon versions. // For floating-CA derivations (and their co-dependencies) this is an @@ -403,7 +403,7 @@ std::map> RemoteStore::queryPartialDerivat std::optional RemoteStore::queryPathFromHashPart(const std::string & hashPart) { auto conn(getConnection()); - conn->to << wopQueryPathFromHashPart << hashPart; + conn->to << WorkerProto::Op::QueryPathFromHashPart << hashPart; conn.processStderr(); Path path = readString(conn->from); if (path.empty()) return {}; @@ -425,10 +425,10 @@ ref RemoteStore::addCAToStore( if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) { conn->to - << wopAddToStore + << WorkerProto::Op::AddToStore << name << caMethod.render(hashType); - workerProtoWrite(*this, conn->to, references); + WorkerProto::write(*this, conn->to, references); conn->to << repair; // The dump source may invoke the store, so we need to make some room. @@ -452,13 +452,13 @@ ref RemoteStore::addCAToStore( throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", name, printHashType(hashType)); std::string s = dump.drain(); - conn->to << wopAddTextToStore << name << s; - workerProtoWrite(*this, conn->to, references); + conn->to << WorkerProto::Op::AddTextToStore << name << s; + WorkerProto::write(*this, conn->to, references); conn.processStderr(); }, [&](const FileIngestionMethod & fim) -> void { conn->to - << wopAddToStore + << WorkerProto::Op::AddToStore << name << ((hashType == htSHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ << (fim == FileIngestionMethod::Recursive ? 1 : 0) @@ -510,7 +510,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { - conn->to << wopImportPaths; + conn->to << WorkerProto::Op::ImportPaths; auto source2 = sinkToSource([&](Sink & sink) { sink << 1 // == path follows @@ -519,7 +519,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - workerProtoWrite(*this, sink, info.references); + WorkerProto::write(*this, sink, info.references); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -529,16 +529,16 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); - auto importedPaths = WorkerProto::read(*this, conn->from); + auto importedPaths = WorkerProto::Serialise::read(*this, conn->from); assert(importedPaths.size() <= 1); } else { - conn->to << wopAddToStoreNar + conn->to << WorkerProto::Op::AddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - workerProtoWrite(*this, conn->to, info.references); + WorkerProto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; @@ -582,7 +582,7 @@ void RemoteStore::addMultipleToStore( if (GET_PROTOCOL_MINOR(getConnection()->daemonVersion) >= 32) { auto conn(getConnection()); conn->to - << wopAddMultipleToStore + << WorkerProto::Op::AddMultipleToStore << repair << !checkSigs; conn.withFramedSink([&](Sink & sink) { @@ -606,12 +606,12 @@ StorePath RemoteStore::addTextToStore( void RemoteStore::registerDrvOutput(const Realisation & info) { auto conn(getConnection()); - conn->to << wopRegisterDrvOutput; + conn->to << WorkerProto::Op::RegisterDrvOutput; if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { conn->to << info.id.to_string(); conn->to << std::string(info.outPath.to_string()); } else { - workerProtoWrite(*this, conn->to, info); + WorkerProto::write(*this, conn->to, info); } conn.processStderr(); } @@ -627,19 +627,19 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, return callback(nullptr); } - conn->to << wopQueryRealisation; + conn->to << WorkerProto::Op::QueryRealisation; conn->to << id.to_string(); conn.processStderr(); auto real = [&]() -> std::shared_ptr { if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { - auto outPaths = WorkerProto>::read( + auto outPaths = WorkerProto::Serialise>::read( *this, conn->from); if (outPaths.empty()) return nullptr; return std::make_shared(Realisation { .id = id, .outPath = *outPaths.begin() }); } else { - auto realisations = WorkerProto>::read( + auto realisations = WorkerProto::Serialise>::read( *this, conn->from); if (realisations.empty()) return nullptr; @@ -654,7 +654,7 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector & reqs) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 30) { - workerProtoWrite(store, conn->to, reqs); + WorkerProto::write(store, conn->to, reqs); } else { Strings ss; for (auto & p : reqs) { @@ -695,7 +695,7 @@ void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMod copyDrvsFromEvalStore(drvPaths, evalStore); auto conn(getConnection()); - conn->to << wopBuildPaths; + conn->to << WorkerProto::Op::BuildPaths; assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); writeDerivedPaths(*this, conn, drvPaths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) @@ -720,11 +720,11 @@ std::vector RemoteStore::buildPathsWithResults( auto & conn = *conn_; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) { - conn->to << wopBuildPathsWithResults; + conn->to << WorkerProto::Op::BuildPathsWithResults; writeDerivedPaths(*this, conn, paths); conn->to << buildMode; conn.processStderr(); - return WorkerProto>::read(*this, conn->from); + return WorkerProto::Serialise>::read(*this, conn->from); } else { // Avoid deadlock. conn_.reset(); @@ -796,7 +796,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD BuildMode buildMode) { auto conn(getConnection()); - conn->to << wopBuildDerivation << printStorePath(drvPath); + conn->to << WorkerProto::Op::BuildDerivation << printStorePath(drvPath); writeDerivation(conn->to, *this, drv); conn->to << buildMode; conn.processStderr(); @@ -807,7 +807,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime; } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) { - auto builtOutputs = WorkerProto::read(*this, conn->from); + auto builtOutputs = WorkerProto::Serialise::read(*this, conn->from); for (auto && [output, realisation] : builtOutputs) res.builtOutputs.insert_or_assign( std::move(output.outputName), @@ -820,7 +820,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD void RemoteStore::ensurePath(const StorePath & path) { auto conn(getConnection()); - conn->to << wopEnsurePath << printStorePath(path); + conn->to << WorkerProto::Op::EnsurePath << printStorePath(path); conn.processStderr(); readInt(conn->from); } @@ -829,7 +829,7 @@ void RemoteStore::ensurePath(const StorePath & path) void RemoteStore::addTempRoot(const StorePath & path) { auto conn(getConnection()); - conn->to << wopAddTempRoot << printStorePath(path); + conn->to << WorkerProto::Op::AddTempRoot << printStorePath(path); conn.processStderr(); readInt(conn->from); } @@ -838,7 +838,7 @@ void RemoteStore::addTempRoot(const StorePath & path) void RemoteStore::addIndirectRoot(const Path & path) { auto conn(getConnection()); - conn->to << wopAddIndirectRoot << path; + conn->to << WorkerProto::Op::AddIndirectRoot << path; conn.processStderr(); readInt(conn->from); } @@ -847,7 +847,7 @@ void RemoteStore::addIndirectRoot(const Path & path) Roots RemoteStore::findRoots(bool censor) { auto conn(getConnection()); - conn->to << wopFindRoots; + conn->to << WorkerProto::Op::FindRoots; conn.processStderr(); size_t count = readNum(conn->from); Roots result; @@ -865,8 +865,8 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) auto conn(getConnection()); conn->to - << wopCollectGarbage << options.action; - workerProtoWrite(*this, conn->to, options.pathsToDelete); + << WorkerProto::Op::CollectGarbage << options.action; + WorkerProto::write(*this, conn->to, options.pathsToDelete); conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ @@ -888,7 +888,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) void RemoteStore::optimiseStore() { auto conn(getConnection()); - conn->to << wopOptimiseStore; + conn->to << WorkerProto::Op::OptimiseStore; conn.processStderr(); readInt(conn->from); } @@ -897,7 +897,7 @@ void RemoteStore::optimiseStore() bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) { auto conn(getConnection()); - conn->to << wopVerifyStore << checkContents << repair; + conn->to << WorkerProto::Op::VerifyStore << checkContents << repair; conn.processStderr(); return readInt(conn->from); } @@ -906,7 +906,7 @@ bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & sigs) { auto conn(getConnection()); - conn->to << wopAddSignatures << printStorePath(storePath) << sigs; + conn->to << WorkerProto::Op::AddSignatures << printStorePath(storePath) << sigs; conn.processStderr(); readInt(conn->from); } @@ -922,12 +922,12 @@ void RemoteStore::queryMissing(const std::vector & targets, // Don't hold the connection handle in the fallback case // to prevent a deadlock. goto fallback; - conn->to << wopQueryMissing; + conn->to << WorkerProto::Op::QueryMissing; writeDerivedPaths(*this, conn, targets); conn.processStderr(); - willBuild = WorkerProto::read(*this, conn->from); - willSubstitute = WorkerProto::read(*this, conn->from); - unknown = WorkerProto::read(*this, conn->from); + willBuild = WorkerProto::Serialise::read(*this, conn->from); + willSubstitute = WorkerProto::Serialise::read(*this, conn->from); + unknown = WorkerProto::Serialise::read(*this, conn->from); conn->from >> downloadSize >> narSize; return; } @@ -941,7 +941,7 @@ void RemoteStore::queryMissing(const std::vector & targets, void RemoteStore::addBuildLog(const StorePath & drvPath, std::string_view log) { auto conn(getConnection()); - conn->to << wopAddBuildLog << drvPath.to_string(); + conn->to << WorkerProto::Op::AddBuildLog << drvPath.to_string(); StringSource source(log); conn.withFramedSink([&](Sink & sink) { source.drainInto(sink); @@ -993,7 +993,7 @@ RemoteStore::Connection::~Connection() void RemoteStore::narFromPath(const StorePath & path, Sink & sink) { auto conn(connections->get()); - conn->to << wopNarFromPath << printStorePath(path); + conn->to << WorkerProto::Op::NarFromPath << printStorePath(path); conn->processStderr(); copyNAR(conn->from, sink); } diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh index 509a8a5d0..f34e55c54 100644 --- a/src/libstore/worker-protocol-impl.hh +++ b/src/libstore/worker-protocol-impl.hh @@ -13,65 +13,65 @@ namespace nix { template -std::vector WorkerProto>::read(const Store & store, Source & from) +std::vector WorkerProto::Serialise>::read(const Store & store, Source & from) { std::vector resSet; auto size = readNum(from); while (size--) { - resSet.push_back(WorkerProto::read(store, from)); + resSet.push_back(WorkerProto::Serialise::read(store, from)); } return resSet; } template -void WorkerProto>::write(const Store & store, Sink & out, const std::vector & resSet) +void WorkerProto::Serialise>::write(const Store & store, Sink & out, const std::vector & resSet) { out << resSet.size(); for (auto & key : resSet) { - WorkerProto::write(store, out, key); + WorkerProto::Serialise::write(store, out, key); } } template -std::set WorkerProto>::read(const Store & store, Source & from) +std::set WorkerProto::Serialise>::read(const Store & store, Source & from) { std::set resSet; auto size = readNum(from); while (size--) { - resSet.insert(WorkerProto::read(store, from)); + resSet.insert(WorkerProto::Serialise::read(store, from)); } return resSet; } template -void WorkerProto>::write(const Store & store, Sink & out, const std::set & resSet) +void WorkerProto::Serialise>::write(const Store & store, Sink & out, const std::set & resSet) { out << resSet.size(); for (auto & key : resSet) { - WorkerProto::write(store, out, key); + WorkerProto::Serialise::write(store, out, key); } } template -std::map WorkerProto>::read(const Store & store, Source & from) +std::map WorkerProto::Serialise>::read(const Store & store, Source & from) { std::map resMap; auto size = readNum(from); while (size--) { - auto k = WorkerProto::read(store, from); - auto v = WorkerProto::read(store, from); + auto k = WorkerProto::Serialise::read(store, from); + auto v = WorkerProto::Serialise::read(store, from); resMap.insert_or_assign(std::move(k), std::move(v)); } return resMap; } template -void WorkerProto>::write(const Store & store, Sink & out, const std::map & resMap) +void WorkerProto::Serialise>::write(const Store & store, Sink & out, const std::map & resMap) { out << resMap.size(); for (auto & i : resMap) { - WorkerProto::write(store, out, i.first); - WorkerProto::write(store, out, i.second); + WorkerProto::Serialise::write(store, out, i.first); + WorkerProto::Serialise::write(store, out, i.second); } } diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 49708b642..0bf9e4d68 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -12,29 +12,29 @@ namespace nix { -std::string WorkerProto::read(const Store & store, Source & from) +std::string WorkerProto::Serialise::read(const Store & store, Source & from) { return readString(from); } -void WorkerProto::write(const Store & store, Sink & out, const std::string & str) +void WorkerProto::Serialise::write(const Store & store, Sink & out, const std::string & str) { out << str; } -StorePath WorkerProto::read(const Store & store, Source & from) +StorePath WorkerProto::Serialise::read(const Store & store, Source & from) { return store.parseStorePath(readString(from)); } -void WorkerProto::write(const Store & store, Sink & out, const StorePath & storePath) +void WorkerProto::Serialise::write(const Store & store, Sink & out, const StorePath & storePath) { out << store.printStorePath(storePath); } -std::optional WorkerProto>::read(const Store & store, Source & from) +std::optional WorkerProto::Serialise>::read(const Store & store, Source & from) { auto temp = readNum(from); switch (temp) { @@ -49,7 +49,7 @@ std::optional WorkerProto>::read(const S } } -void WorkerProto>::write(const Store & store, Sink & out, const std::optional & optTrusted) +void WorkerProto::Serialise>::write(const Store & store, Sink & out, const std::optional & optTrusted) { if (!optTrusted) out << (uint8_t)0; @@ -68,30 +68,30 @@ void WorkerProto>::write(const Store & store, Sink & } -ContentAddress WorkerProto::read(const Store & store, Source & from) +ContentAddress WorkerProto::Serialise::read(const Store & store, Source & from) { return ContentAddress::parse(readString(from)); } -void WorkerProto::write(const Store & store, Sink & out, const ContentAddress & ca) +void WorkerProto::Serialise::write(const Store & store, Sink & out, const ContentAddress & ca) { out << renderContentAddress(ca); } -DerivedPath WorkerProto::read(const Store & store, Source & from) +DerivedPath WorkerProto::Serialise::read(const Store & store, Source & from) { auto s = readString(from); return DerivedPath::parseLegacy(store, s); } -void WorkerProto::write(const Store & store, Sink & out, const DerivedPath & req) +void WorkerProto::Serialise::write(const Store & store, Sink & out, const DerivedPath & req) { out << req.to_string_legacy(store); } -Realisation WorkerProto::read(const Store & store, Source & from) +Realisation WorkerProto::Serialise::read(const Store & store, Source & from) { std::string rawInput = readString(from); return Realisation::fromJSON( @@ -100,41 +100,41 @@ Realisation WorkerProto::read(const Store & store, Source & from) ); } -void WorkerProto::write(const Store & store, Sink & out, const Realisation & realisation) +void WorkerProto::Serialise::write(const Store & store, Sink & out, const Realisation & realisation) { out << realisation.toJSON().dump(); } -DrvOutput WorkerProto::read(const Store & store, Source & from) +DrvOutput WorkerProto::Serialise::read(const Store & store, Source & from) { return DrvOutput::parse(readString(from)); } -void WorkerProto::write(const Store & store, Sink & out, const DrvOutput & drvOutput) +void WorkerProto::Serialise::write(const Store & store, Sink & out, const DrvOutput & drvOutput) { out << drvOutput.to_string(); } -KeyedBuildResult WorkerProto::read(const Store & store, Source & from) +KeyedBuildResult WorkerProto::Serialise::read(const Store & store, Source & from) { - auto path = WorkerProto::read(store, from); - auto br = WorkerProto::read(store, from); + auto path = WorkerProto::Serialise::read(store, from); + auto br = WorkerProto::Serialise::read(store, from); return KeyedBuildResult { std::move(br), /* .path = */ std::move(path), }; } -void WorkerProto::write(const Store & store, Sink & to, const KeyedBuildResult & res) +void WorkerProto::Serialise::write(const Store & store, Sink & to, const KeyedBuildResult & res) { - workerProtoWrite(store, to, res.path); - workerProtoWrite(store, to, static_cast(res)); + WorkerProto::write(store, to, res.path); + WorkerProto::write(store, to, static_cast(res)); } -BuildResult WorkerProto::read(const Store & store, Source & from) +BuildResult WorkerProto::Serialise::read(const Store & store, Source & from) { BuildResult res; res.status = (BuildResult::Status) readInt(from); @@ -144,7 +144,7 @@ BuildResult WorkerProto::read(const Store & store, Source & from) >> res.isNonDeterministic >> res.startTime >> res.stopTime; - auto builtOutputs = WorkerProto::read(store, from); + auto builtOutputs = WorkerProto::Serialise::read(store, from); for (auto && [output, realisation] : builtOutputs) res.builtOutputs.insert_or_assign( std::move(output.outputName), @@ -152,7 +152,7 @@ BuildResult WorkerProto::read(const Store & store, Source & from) return res; } -void WorkerProto::write(const Store & store, Sink & to, const BuildResult & res) +void WorkerProto::Serialise::write(const Store & store, Sink & to, const BuildResult & res) { to << res.status @@ -164,28 +164,28 @@ void WorkerProto::write(const Store & store, Sink & to, const Build DrvOutputs builtOutputs; for (auto & [output, realisation] : res.builtOutputs) builtOutputs.insert_or_assign(realisation.id, realisation); - workerProtoWrite(store, to, builtOutputs); + WorkerProto::write(store, to, builtOutputs); } -std::optional WorkerProto>::read(const Store & store, Source & from) +std::optional WorkerProto::Serialise>::read(const Store & store, Source & from) { auto s = readString(from); return s == "" ? std::optional {} : store.parseStorePath(s); } -void WorkerProto>::write(const Store & store, Sink & out, const std::optional & storePathOpt) +void WorkerProto::Serialise>::write(const Store & store, Sink & out, const std::optional & storePathOpt) { out << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); } -std::optional WorkerProto>::read(const Store & store, Source & from) +std::optional WorkerProto::Serialise>::read(const Store & store, Source & from) { return ContentAddress::parseOpt(readString(from)); } -void WorkerProto>::write(const Store & store, Sink & out, const std::optional & caOpt) +void WorkerProto::Serialise>::write(const Store & store, Sink & out, const std::optional & caOpt) { out << (caOpt ? renderContentAddress(*caOpt) : ""); } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 18a4e11b2..cd6801290 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -14,57 +14,6 @@ namespace nix { #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) -/** - * Enumeration of all the request types for the "worker protocol", used - * by unix:// and ssh-ng:// stores. - */ -typedef enum { - wopIsValidPath = 1, - wopHasSubstitutes = 3, - wopQueryPathHash = 4, // obsolete - wopQueryReferences = 5, // obsolete - wopQueryReferrers = 6, - wopAddToStore = 7, - wopAddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use wopAddToStore - wopBuildPaths = 9, - wopEnsurePath = 10, - wopAddTempRoot = 11, - wopAddIndirectRoot = 12, - wopSyncWithGC = 13, - wopFindRoots = 14, - wopExportPath = 16, // obsolete - wopQueryDeriver = 18, // obsolete - wopSetOptions = 19, - wopCollectGarbage = 20, - wopQuerySubstitutablePathInfo = 21, - wopQueryDerivationOutputs = 22, // obsolete - wopQueryAllValidPaths = 23, - wopQueryFailedPaths = 24, - wopClearFailedPaths = 25, - wopQueryPathInfo = 26, - wopImportPaths = 27, // obsolete - wopQueryDerivationOutputNames = 28, // obsolete - wopQueryPathFromHashPart = 29, - wopQuerySubstitutablePathInfos = 30, - wopQueryValidPaths = 31, - wopQuerySubstitutablePaths = 32, - wopQueryValidDerivers = 33, - wopOptimiseStore = 34, - wopVerifyStore = 35, - wopBuildDerivation = 36, - wopAddSignatures = 37, - wopNarFromPath = 38, - wopAddToStoreNar = 39, - wopQueryMissing = 40, - wopQueryDerivationOutputMap = 41, - wopRegisterDrvOutput = 42, - wopQueryRealisation = 43, - wopAddMultipleToStore = 44, - wopAddBuildLog = 45, - wopBuildPathsWithResults = 46, -} WorkerOp; - - #define STDERR_NEXT 0x6f6c6d67 #define STDERR_READ 0x64617461 // data needed from source #define STDERR_WRITE 0x64617416 // data for sink @@ -78,7 +27,7 @@ typedef enum { class Store; struct Source; -// items being serialized +// items being serialised struct DerivedPath; struct DrvOutput; struct Realisation; @@ -88,31 +37,132 @@ enum TrustedFlag : bool; /** - * Data type for canonical pairs of serializers for the worker protocol. + * The "worker protocol", used by unix:// and ssh-ng:// stores. * - * See https://en.cppreference.com/w/cpp/language/adl for the broader - * concept of what is going on here. + * This `struct` is basically just a `namespace`; We use a type rather + * than a namespace just so we can use it as a template argument. */ -template -struct WorkerProto { - static T read(const Store & store, Source & from); - static void write(const Store & store, Sink & out, const T & t); +struct WorkerProto +{ + /** + * Enumeration of all the request types for the protocol. + */ + enum struct Op : uint64_t; + + /** + * Data type for canonical pairs of serialisers for the worker protocol. + * + * See https://en.cppreference.com/w/cpp/language/adl for the broader + * concept of what is going on here. + */ + template + struct Serialise; + // This is the definition of `Serialise` we *want* to put here, but + // do not do so. + // + // The problem is that if we do so, C++ will think we have + // seralisers for *all* types. We don't, of course, but that won't + // cause an error until link time. That makes for long debug cycles + // when there is a missing serialiser. + // + // By not defining it globally, and instead letting individual + // serialisers specialise the type, we get back the compile-time + // errors we would like. When no serialiser exists, C++ sees an + // abstract "incomplete" type with no definition, and any attempt to + // use `to` or `from` static methods is a compile-time error because + // they don't exist on an incomplete type. + // + // This makes for a quicker debug cycle, as desired. +#if 0 + { + static T read(const Store & store, Source & from); + static void write(const Store & store, Sink & out, const T & t); + }; +#endif + + /** + * Wrapper function around `WorkerProto::Serialise::write` that allows us to + * infer the type instead of having to write it down explicitly. + */ + template + static void write(const Store & store, Sink & out, const T & t) + { + WorkerProto::Serialise::write(store, out, t); + } +}; + +enum struct WorkerProto::Op : uint64_t +{ + IsValidPath = 1, + HasSubstitutes = 3, + QueryPathHash = 4, // obsolete + QueryReferences = 5, // obsolete + QueryReferrers = 6, + AddToStore = 7, + AddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use WorkerProto::Op::AddToStore + BuildPaths = 9, + EnsurePath = 10, + AddTempRoot = 11, + AddIndirectRoot = 12, + SyncWithGC = 13, + FindRoots = 14, + ExportPath = 16, // obsolete + QueryDeriver = 18, // obsolete + SetOptions = 19, + CollectGarbage = 20, + QuerySubstitutablePathInfo = 21, + QueryDerivationOutputs = 22, // obsolete + QueryAllValidPaths = 23, + QueryFailedPaths = 24, + ClearFailedPaths = 25, + QueryPathInfo = 26, + ImportPaths = 27, // obsolete + QueryDerivationOutputNames = 28, // obsolete + QueryPathFromHashPart = 29, + QuerySubstitutablePathInfos = 30, + QueryValidPaths = 31, + QuerySubstitutablePaths = 32, + QueryValidDerivers = 33, + OptimiseStore = 34, + VerifyStore = 35, + BuildDerivation = 36, + AddSignatures = 37, + NarFromPath = 38, + AddToStoreNar = 39, + QueryMissing = 40, + QueryDerivationOutputMap = 41, + RegisterDrvOutput = 42, + QueryRealisation = 43, + AddMultipleToStore = 44, + AddBuildLog = 45, + BuildPathsWithResults = 46, }; /** - * Wrapper function around `WorkerProto::write` that allows us to - * infer the type instead of having to write it down explicitly. + * Convenience for sending operation codes. + * + * @todo Switch to using `WorkerProto::Serialise` instead probably. But + * this was not done at this time so there would be less churn. */ -template -void workerProtoWrite(const Store & store, Sink & out, const T & t) +inline Sink & operator << (Sink & sink, WorkerProto::Op op) { - WorkerProto::write(store, out, t); + return sink << (uint64_t) op; } /** - * Declare a canonical serializer pair for the worker protocol. + * Convenience for debugging. * - * We specialize the struct merely to indicate that we are implementing + * @todo Perhaps render known opcodes more nicely. + */ +inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op) +{ + return s << (uint64_t) op; +} + +/** + * Declare a canonical serialiser pair for the worker protocol. + * + * We specialise the struct merely to indicate that we are implementing * the function for the given type. * * Some sort of `template<...>` must be used with the caller for this to @@ -120,7 +170,7 @@ void workerProtoWrite(const Store & store, Sink & out, const T & t) * practice. */ #define MAKE_WORKER_PROTO(T) \ - struct WorkerProto< T > { \ + struct WorkerProto::Serialise< T > { \ static T read(const Store & store, Source & from); \ static void write(const Store & store, Sink & out, const T & t); \ }; @@ -156,7 +206,7 @@ MAKE_WORKER_PROTO(X_); /** * These use the empty string for the null case, relying on the fact - * that the underlying types never serialize to the empty string. + * that the underlying types never serialise to the empty string. * * We do this instead of a generic std::optional instance because * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 94368e415..1558ac11e 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -850,7 +850,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdQueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - auto paths = WorkerProto::read(*store, in); + auto paths = WorkerProto::Serialise::read(*store, in); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -859,19 +859,19 @@ static void opServe(Strings opFlags, Strings opArgs) store->substitutePaths(paths); } - workerProtoWrite(*store, out, store->queryValidPaths(paths)); + WorkerProto::write(*store, out, store->queryValidPaths(paths)); break; } case cmdQueryPathInfos: { - auto paths = WorkerProto::read(*store, in); + auto paths = WorkerProto::Serialise::read(*store, in); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - workerProtoWrite(*store, out, info->references); + WorkerProto::write(*store, out, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -899,7 +899,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdExportPaths: { readInt(in); // obsolete - store->exportPaths(WorkerProto::read(*store, in), out); + store->exportPaths(WorkerProto::Serialise::read(*store, in), out); break; } @@ -945,7 +945,7 @@ static void opServe(Strings opFlags, Strings opArgs) DrvOutputs builtOutputs; for (auto & [output, realisation] : status.builtOutputs) builtOutputs.insert_or_assign(realisation.id, realisation); - workerProtoWrite(*store, out, builtOutputs); + WorkerProto::write(*store, out, builtOutputs); } break; @@ -954,9 +954,9 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdQueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; - store->computeFSClosure(WorkerProto::read(*store, in), + store->computeFSClosure(WorkerProto::Serialise::read(*store, in), closure, false, includeOutputs); - workerProtoWrite(*store, out, closure); + WorkerProto::write(*store, out, closure); break; } @@ -971,7 +971,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = WorkerProto::read(*store, in); + info.references = WorkerProto::Serialise::read(*store, in); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = ContentAddress::parseOpt(readString(in)); From 4e8b495ad7dddabc35bf9d6afe3573426ffed15d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 26 May 2023 11:22:24 -0400 Subject: [PATCH 028/909] Likewise namespace and `enum struct`-ify `ServeCommand` The motivation is exactly the same as for the last commit. In addition, this anticipates us formally defining separate serialisers for the serve protocol. --- src/libstore/legacy-ssh-store.cc | 16 ++++----- src/libstore/serve-protocol.hh | 58 ++++++++++++++++++++++++++------ src/nix-store/nix-store.cc | 22 ++++++------ 3 files changed, 66 insertions(+), 30 deletions(-) diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 7d006f451..55ecab0ff 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -134,7 +134,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); - conn->to << cmdQueryPathInfos << PathSet{printStorePath(path)}; + conn->to << ServeProto::Command::QueryPathInfos << PathSet{printStorePath(path)}; conn->to.flush(); auto p = readString(conn->from); @@ -177,7 +177,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) { conn->to - << cmdAddToStoreNar + << ServeProto::Command::AddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); @@ -199,7 +199,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor } else { conn->to - << cmdImportPaths + << ServeProto::Command::ImportPaths << 1; try { copyNAR(source, conn->to); @@ -227,7 +227,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor { auto conn(connections->get()); - conn->to << cmdDumpStorePath << printStorePath(path); + conn->to << ServeProto::Command::DumpStorePath << printStorePath(path); conn->to.flush(); copyNAR(conn->from, sink); } @@ -280,7 +280,7 @@ public: auto conn(connections->get()); conn->to - << cmdBuildDerivation + << ServeProto::Command::BuildDerivation << printStorePath(drvPath); writeDerivation(conn->to, *this, drv); @@ -311,7 +311,7 @@ public: auto conn(connections->get()); - conn->to << cmdBuildPaths; + conn->to << ServeProto::Command::BuildPaths; Strings ss; for (auto & p : drvPaths) { auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); @@ -368,7 +368,7 @@ public: auto conn(connections->get()); conn->to - << cmdQueryClosure + << ServeProto::Command::QueryClosure << includeOutputs; WorkerProto::write(*this, conn->to, paths); conn->to.flush(); @@ -383,7 +383,7 @@ public: auto conn(connections->get()); conn->to - << cmdQueryValidPaths + << ServeProto::Command::QueryValidPaths << false // lock << maybeSubstitute; WorkerProto::write(*this, conn->to, paths); diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 553fd3a09..7e43b3969 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -10,16 +10,52 @@ namespace nix { #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) -typedef enum { - cmdQueryValidPaths = 1, - cmdQueryPathInfos = 2, - cmdDumpStorePath = 3, - cmdImportPaths = 4, - cmdExportPaths = 5, - cmdBuildPaths = 6, - cmdQueryClosure = 7, - cmdBuildDerivation = 8, - cmdAddToStoreNar = 9, -} ServeCommand; +/** + * The "serve protocol", used by ssh:// stores. + * + * This `struct` is basically just a `namespace`; We use a type rather + * than a namespace just so we can use it as a template argument. + */ +struct ServeProto +{ + /** + * Enumeration of all the request types for the protocol. + */ + enum struct Command : uint64_t; +}; + +enum struct ServeProto::Command : uint64_t +{ + QueryValidPaths = 1, + QueryPathInfos = 2, + DumpStorePath = 3, + ImportPaths = 4, + ExportPaths = 5, + BuildPaths = 6, + QueryClosure = 7, + BuildDerivation = 8, + AddToStoreNar = 9, +}; + +/** + * Convenience for sending operation codes. + * + * @todo Switch to using `ServeProto::Serialize` instead probably. But + * this was not done at this time so there would be less churn. + */ +inline Sink & operator << (Sink & sink, ServeProto::Command op) +{ + return sink << (uint64_t) op; +} + +/** + * Convenience for debugging. + * + * @todo Perhaps render known opcodes more nicely. + */ +inline std::ostream & operator << (std::ostream & s, ServeProto::Command op) +{ + return s << (uint64_t) op; +} } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 1558ac11e..062c68ff8 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -838,16 +838,16 @@ static void opServe(Strings opFlags, Strings opArgs) }; while (true) { - ServeCommand cmd; + ServeProto::Command cmd; try { - cmd = (ServeCommand) readInt(in); + cmd = (ServeProto::Command) readInt(in); } catch (EndOfFile & e) { break; } switch (cmd) { - case cmdQueryValidPaths: { + case ServeProto::Command::QueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); auto paths = WorkerProto::Serialise::read(*store, in); @@ -863,7 +863,7 @@ static void opServe(Strings opFlags, Strings opArgs) break; } - case cmdQueryPathInfos: { + case ServeProto::Command::QueryPathInfos: { auto paths = WorkerProto::Serialise::read(*store, in); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { @@ -886,24 +886,24 @@ static void opServe(Strings opFlags, Strings opArgs) break; } - case cmdDumpStorePath: + case ServeProto::Command::DumpStorePath: store->narFromPath(store->parseStorePath(readString(in)), out); break; - case cmdImportPaths: { + case ServeProto::Command::ImportPaths: { if (!writeAllowed) throw Error("importing paths is not allowed"); store->importPaths(in, NoCheckSigs); // FIXME: should we skip sig checking? out << 1; // indicate success break; } - case cmdExportPaths: { + case ServeProto::Command::ExportPaths: { readInt(in); // obsolete store->exportPaths(WorkerProto::Serialise::read(*store, in), out); break; } - case cmdBuildPaths: { + case ServeProto::Command::BuildPaths: { if (!writeAllowed) throw Error("building paths is not allowed"); @@ -924,7 +924,7 @@ static void opServe(Strings opFlags, Strings opArgs) break; } - case cmdBuildDerivation: { /* Used by hydra-queue-runner. */ + case ServeProto::Command::BuildDerivation: { /* Used by hydra-queue-runner. */ if (!writeAllowed) throw Error("building paths is not allowed"); @@ -951,7 +951,7 @@ static void opServe(Strings opFlags, Strings opArgs) break; } - case cmdQueryClosure: { + case ServeProto::Command::QueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; store->computeFSClosure(WorkerProto::Serialise::read(*store, in), @@ -960,7 +960,7 @@ static void opServe(Strings opFlags, Strings opArgs) break; } - case cmdAddToStoreNar: { + case ServeProto::Command::AddToStoreNar: { if (!writeAllowed) throw Error("importing paths is not allowed"); auto path = readString(in); From 9f69b7dee9fc6035b8aa0cc718f5e74af460d9aa Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Apr 2023 13:40:46 -0400 Subject: [PATCH 029/909] Create `worker_proto::{Read,Write}Conn` Pass this around instead of `Source &` and `Sink &` directly. This will give us something to put the protocol version on once the time comes. To do this ergonomically, we need to expose `RemoteStore::Connection`, so do that too. Give it some more API docs while we are at it. --- src/libstore/build/derivation-goal.cc | 6 +- src/libstore/daemon.cc | 64 ++++++++------- src/libstore/derivations.cc | 7 +- src/libstore/export-import.cc | 7 +- src/libstore/legacy-ssh-store.cc | 52 ++++++++++-- src/libstore/path-info.cc | 7 +- src/libstore/remote-store-connection.hh | 97 ++++++++++++++++++++++ src/libstore/remote-store.cc | 76 +++++++++--------- src/libstore/remote-store.hh | 16 +--- src/libstore/ssh-store.cc | 1 + src/libstore/uds-remote-store.hh | 1 + src/libstore/worker-protocol-impl.hh | 40 +++++----- src/libstore/worker-protocol.cc | 102 ++++++++++++------------ src/libstore/worker-protocol.hh | 34 ++++++-- src/nix-store/nix-store.cc | 21 ++--- src/nix/daemon.cc | 1 + 16 files changed, 348 insertions(+), 184 deletions(-) create mode 100644 src/libstore/remote-store-connection.hh diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 1a946e3d2..5e37f7ecb 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1151,9 +1151,11 @@ HookReply DerivationGoal::tryBuildHook() throw; } + WorkerProto::WriteConn conn { hook->sink }; + /* Tell the hook all the inputs that have to be copied to the remote system. */ - WorkerProto::write(worker.store, hook->sink, inputPaths); + WorkerProto::write(worker.store, conn, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ @@ -1164,7 +1166,7 @@ HookReply DerivationGoal::tryBuildHook() if (buildMode != bmCheck && status.known && status.known->isValid()) continue; missingOutputs.insert(outputName); } - WorkerProto::write(worker.store, hook->sink, missingOutputs); + WorkerProto::write(worker.store, conn, missingOutputs); } hook->sink = FdSink(); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 7eba1a79d..75c3d2aca 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -260,13 +260,13 @@ struct ClientSettings } }; -static std::vector readDerivedPaths(Store & store, unsigned int clientVersion, Source & from) +static std::vector readDerivedPaths(Store & store, unsigned int clientVersion, WorkerProto::ReadConn conn) { std::vector reqs; if (GET_PROTOCOL_MINOR(clientVersion) >= 30) { - reqs = WorkerProto::Serialise>::read(store, from); + reqs = WorkerProto::Serialise>::read(store, conn); } else { - for (auto & s : readStrings(from)) + for (auto & s : readStrings(conn.from)) reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath()); } return reqs; @@ -276,6 +276,9 @@ static void performOp(TunnelLogger * logger, ref store, TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, Source & from, BufferedSink & to, WorkerProto::Op op) { + WorkerProto::ReadConn rconn { .from = from }; + WorkerProto::WriteConn wconn { .to = to }; + switch (op) { case WorkerProto::Op::IsValidPath: { @@ -288,7 +291,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryValidPaths: { - auto paths = WorkerProto::Serialise::read(*store, from); + auto paths = WorkerProto::Serialise::read(*store, rconn); SubstituteFlag substitute = NoSubstitute; if (GET_PROTOCOL_MINOR(clientVersion) >= 27) { @@ -301,7 +304,7 @@ static void performOp(TunnelLogger * logger, ref store, } auto res = store->queryValidPaths(paths, substitute); logger->stopWork(); - WorkerProto::write(*store, to, res); + WorkerProto::write(*store, wconn, res); break; } @@ -317,11 +320,11 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QuerySubstitutablePaths: { - auto paths = WorkerProto::Serialise::read(*store, from); + auto paths = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - WorkerProto::write(*store, to, res); + WorkerProto::write(*store, wconn, res); break; } @@ -350,7 +353,7 @@ static void performOp(TunnelLogger * logger, ref store, paths = store->queryValidDerivers(path); else paths = store->queryDerivationOutputs(path); logger->stopWork(); - WorkerProto::write(*store, to, paths); + WorkerProto::write(*store, wconn, paths); break; } @@ -368,7 +371,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto outputs = store->queryPartialDerivationOutputMap(path); logger->stopWork(); - WorkerProto::write(*store, to, outputs); + WorkerProto::write(*store, wconn, outputs); break; } @@ -394,7 +397,7 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 25) { auto name = readString(from); auto camStr = readString(from); - auto refs = WorkerProto::Serialise::read(*store, from); + auto refs = WorkerProto::Serialise::read(*store, rconn); bool repairBool; from >> repairBool; auto repair = RepairFlag{repairBool}; @@ -496,7 +499,7 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::AddTextToStore: { std::string suffix = readString(from); std::string s = readString(from); - auto refs = WorkerProto::Serialise::read(*store, from); + auto refs = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); auto path = store->addTextToStore(suffix, s, refs, NoRepair); logger->stopWork(); @@ -528,7 +531,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::BuildPaths: { - auto drvs = readDerivedPaths(*store, clientVersion, from); + auto drvs = readDerivedPaths(*store, clientVersion, rconn); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode) readInt(from); @@ -553,7 +556,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::BuildPathsWithResults: { - auto drvs = readDerivedPaths(*store, clientVersion, from); + auto drvs = readDerivedPaths(*store, clientVersion, rconn); BuildMode mode = bmNormal; mode = (BuildMode) readInt(from); @@ -568,7 +571,7 @@ static void performOp(TunnelLogger * logger, ref store, auto results = store->buildPathsWithResults(drvs, mode); logger->stopWork(); - WorkerProto::write(*store, to, results); + WorkerProto::write(*store, wconn, results); break; } @@ -645,7 +648,7 @@ static void performOp(TunnelLogger * logger, ref store, DrvOutputs builtOutputs; for (auto & [output, realisation] : res.builtOutputs) builtOutputs.insert_or_assign(realisation.id, realisation); - WorkerProto::write(*store, to, builtOutputs); + WorkerProto::write(*store, wconn, builtOutputs); } break; } @@ -710,7 +713,7 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::CollectGarbage: { GCOptions options; options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = WorkerProto::Serialise::read(*store, from); + options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields readInt(from); @@ -780,7 +783,7 @@ static void performOp(TunnelLogger * logger, ref store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - WorkerProto::write(*store, to, i->second.references); + WorkerProto::write(*store, wconn, i->second.references); to << i->second.downloadSize << i->second.narSize; } @@ -791,11 +794,11 @@ static void performOp(TunnelLogger * logger, ref store, SubstitutablePathInfos infos; StorePathCAMap pathsMap = {}; if (GET_PROTOCOL_MINOR(clientVersion) < 22) { - auto paths = WorkerProto::Serialise::read(*store, from); + auto paths = WorkerProto::Serialise::read(*store, rconn); for (auto & path : paths) pathsMap.emplace(path, std::nullopt); } else - pathsMap = WorkerProto::Serialise::read(*store, from); + pathsMap = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); @@ -803,7 +806,7 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - WorkerProto::write(*store, to, i.second.references); + WorkerProto::write(*store, wconn, i.second.references); to << i.second.downloadSize << i.second.narSize; } break; @@ -813,7 +816,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto paths = store->queryAllValidPaths(); logger->stopWork(); - WorkerProto::write(*store, to, paths); + WorkerProto::write(*store, wconn, paths); break; } @@ -885,7 +888,7 @@ static void performOp(TunnelLogger * logger, ref store, ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(*store, from); + info.references = WorkerProto::Serialise::read(*store, rconn); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); info.ca = ContentAddress::parseOpt(readString(from)); @@ -930,15 +933,15 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryMissing: { - auto targets = readDerivedPaths(*store, clientVersion, from); + auto targets = readDerivedPaths(*store, clientVersion, rconn); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); logger->stopWork(); - WorkerProto::write(*store, to, willBuild); - WorkerProto::write(*store, to, willSubstitute); - WorkerProto::write(*store, to, unknown); + WorkerProto::write(*store, wconn, willBuild); + WorkerProto::write(*store, wconn, willSubstitute); + WorkerProto::write(*store, wconn, unknown); to << downloadSize << narSize; break; } @@ -951,7 +954,7 @@ static void performOp(TunnelLogger * logger, ref store, store->registerDrvOutput(Realisation{ .id = outputId, .outPath = outputPath}); } else { - auto realisation = WorkerProto::Serialise::read(*store, from); + auto realisation = WorkerProto::Serialise::read(*store, rconn); store->registerDrvOutput(realisation); } logger->stopWork(); @@ -966,11 +969,11 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) < 31) { std::set outPaths; if (info) outPaths.insert(info->outPath); - WorkerProto::write(*store, to, outPaths); + WorkerProto::write(*store, wconn, outPaths); } else { std::set realisations; if (info) realisations.insert(*info); - WorkerProto::write(*store, to, realisations); + WorkerProto::write(*store, wconn, realisations); } break; } @@ -1050,7 +1053,8 @@ void processConnection( auto temp = trusted ? store->isTrustedClient() : std::optional { NotTrusted }; - WorkerProto::write(*store, to, temp); + WorkerProto::WriteConn wconn { .to = to }; + WorkerProto::write(*store, wconn, temp); } /* Send startup error messages to the client. */ diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 2295ff9e8..6f63685d4 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -750,7 +750,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = WorkerProto::Serialise::read(store, in); + drv.inputSrcs = WorkerProto::Serialise::read(store, + WorkerProto::ReadConn { .from = in }); in >> drv.platform >> drv.builder; drv.args = readStrings(in); @@ -798,7 +799,9 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, }, i.second.raw()); } - WorkerProto::write(store, out, drv.inputSrcs); + WorkerProto::write(store, + WorkerProto::WriteConn { .to = out }, + drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); for (auto & i : drv.env) diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index b4100bace..e866aeb42 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -46,7 +46,9 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - WorkerProto::write(*this, teeSink, info->references); + WorkerProto::write(*this, + WorkerProto::WriteConn { .to = teeSink }, + info->references); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -74,7 +76,8 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) //Activity act(*logger, lvlInfo, "importing path '%s'", info.path); - auto references = WorkerProto::Serialise::read(*this, source); + auto references = WorkerProto::Serialise::read(*this, + WorkerProto::ReadConn { .from = source }); auto deriver = readString(source); auto narHash = hashString(htSHA256, saved.s); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 55ecab0ff..fa17d606d 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -48,6 +48,42 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor FdSource from; int remoteVersion; bool good = true; + + /** + * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the + * factored out worker protocol searlizers with a + * `LegacySSHStore::Connection`. + * + * The worker protocol connection types are unidirectional, unlike + * this type. + * + * @todo Use server protocol serializers, not worker protocol + * serializers, once we have made that distiction. + */ + operator WorkerProto::ReadConn () + { + return WorkerProto::ReadConn { + .from = from, + }; + } + + /* + * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the + * factored out worker protocol searlizers with a + * `LegacySSHStore::Connection`. + * + * The worker protocol connection types are unidirectional, unlike + * this type. + * + * @todo Use server protocol serializers, not worker protocol + * serializers, once we have made that distiction. + */ + operator WorkerProto::WriteConn () + { + return WorkerProto::WriteConn { + .to = to, + }; + } }; std::string host; @@ -147,7 +183,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = WorkerProto::Serialise::read(*this, conn->from); + info->references = WorkerProto::Serialise::read(*this, *conn); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -181,7 +217,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - WorkerProto::write(*this, conn->to, info.references); + WorkerProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize @@ -210,7 +246,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - WorkerProto::write(*this, conn->to, info.references); + WorkerProto::write(*this, *conn, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -295,7 +331,7 @@ public: if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - auto builtOutputs = WorkerProto::Serialise::read(*this, conn->from); + auto builtOutputs = WorkerProto::Serialise::read(*this, *conn); for (auto && [output, realisation] : builtOutputs) status.builtOutputs.insert_or_assign( std::move(output.outputName), @@ -370,10 +406,10 @@ public: conn->to << ServeProto::Command::QueryClosure << includeOutputs; - WorkerProto::write(*this, conn->to, paths); + WorkerProto::write(*this, *conn, paths); conn->to.flush(); - for (auto & i : WorkerProto::Serialise::read(*this, conn->from)) + for (auto & i : WorkerProto::Serialise::read(*this, *conn)) out.insert(i); } @@ -386,10 +422,10 @@ public: << ServeProto::Command::QueryValidPaths << false // lock << maybeSubstitute; - WorkerProto::write(*this, conn->to, paths); + WorkerProto::write(*this, *conn, paths); conn->to.flush(); - return WorkerProto::Serialise::read(*this, conn->from); + return WorkerProto::Serialise::read(*this, *conn); } void connect() override diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 9971de1dd..981bbfb14 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -133,7 +133,8 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned auto narHash = Hash::parseAny(readString(source), htSHA256); ValidPathInfo info(path, narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(store, source); + info.references = WorkerProto::Serialise::read(store, + WorkerProto::ReadConn { .from = source }); source >> info.registrationTime >> info.narSize; if (format >= 16) { source >> info.ultimate; @@ -154,7 +155,9 @@ void ValidPathInfo::write( sink << store.printStorePath(path); sink << (deriver ? store.printStorePath(*deriver) : "") << narHash.to_string(Base16, false); - WorkerProto::write(store, sink, references); + WorkerProto::write(store, + WorkerProto::WriteConn { .to = sink }, + references); sink << registrationTime << narSize; if (format >= 16) { sink << ultimate diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh new file mode 100644 index 000000000..d32d91a60 --- /dev/null +++ b/src/libstore/remote-store-connection.hh @@ -0,0 +1,97 @@ +#include "remote-store.hh" +#include "worker-protocol.hh" + +namespace nix { + +/** + * Bidirectional connection (send and receive) used by the Remote Store + * implementation. + * + * Contains `Source` and `Sink` for actual communication, along with + * other information learned when negotiating the connection. + */ +struct RemoteStore::Connection +{ + /** + * Send with this. + */ + FdSink to; + + /** + * Receive with this. + */ + FdSource from; + + /** + * Worker protocol version used for the connection. + * + * Despite its name, I think it is actually the maximum version both + * sides support. (If the maximum doesn't exist, we would fail to + * establish a connection and produce a value of this type.) + */ + unsigned int daemonVersion; + + /** + * Whether the remote side trusts us or not. + * + * 3 values: "yes", "no", or `std::nullopt` for "unknown". + * + * Note that the "remote side" might not be just the end daemon, but + * also an intermediary forwarder that can make its own trusting + * decisions. This would be the intersection of all their trust + * decisions, since it takes only one link in the chain to start + * denying operations. + */ + std::optional remoteTrustsUs; + + /** + * The version of the Nix daemon that is processing our requests. + * + * Do note, it may or may not communicating with another daemon, + * rather than being an "end" `LocalStore` or similar. + */ + std::optional daemonNixVersion; + + /** + * Time this connection was established. + */ + std::chrono::time_point startTime; + + /** + * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the + * factored out worker protocol searlizers with a + * `RemoteStore::Connection`. + * + * The worker protocol connection types are unidirectional, unlike + * this type. + */ + operator WorkerProto::ReadConn () + { + return WorkerProto::ReadConn { + .from = from, + }; + } + + /** + * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the + * factored out worker protocol searlizers with a + * `RemoteStore::Connection`. + * + * The worker protocol connection types are unidirectional, unlike + * this type. + */ + operator WorkerProto::WriteConn () + { + return WorkerProto::WriteConn { + .to = to, + }; + } + + virtual ~Connection(); + + virtual void closeWrite() = 0; + + std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); +}; + +} diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 9253ce09e..1e2104e1f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -5,6 +5,7 @@ #include "remote-fs-accessor.hh" #include "build-result.hh" #include "remote-store.hh" +#include "remote-store-connection.hh" #include "worker-protocol.hh" #include "worker-protocol-impl.hh" #include "archive.hh" @@ -101,7 +102,7 @@ void RemoteStore::initConnection(Connection & conn) } if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) { - conn.remoteTrustsUs = WorkerProto::Serialise>::read(*this, conn.from); + conn.remoteTrustsUs = WorkerProto::Serialise>::read(*this, conn); } else { // We don't know the answer; protocol to old. conn.remoteTrustsUs = std::nullopt; @@ -185,6 +186,7 @@ struct ConnectionHandle } RemoteStore::Connection * operator -> () { return &*handle; } + RemoteStore::Connection & operator * () { return *handle; } void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true) { @@ -228,12 +230,12 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute return res; } else { conn->to << WorkerProto::Op::QueryValidPaths; - WorkerProto::write(*this, conn->to, paths); + WorkerProto::write(*this, *conn, paths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) { conn->to << (settings.buildersUseSubstitutes ? 1 : 0); } conn.processStderr(); - return WorkerProto::Serialise::read(*this, conn->from); + return WorkerProto::Serialise::read(*this, *conn); } } @@ -243,7 +245,7 @@ StorePathSet RemoteStore::queryAllValidPaths() auto conn(getConnection()); conn->to << WorkerProto::Op::QueryAllValidPaths; conn.processStderr(); - return WorkerProto::Serialise::read(*this, conn->from); + return WorkerProto::Serialise::read(*this, *conn); } @@ -260,9 +262,9 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) return res; } else { conn->to << WorkerProto::Op::QuerySubstitutablePaths; - WorkerProto::write(*this, conn->to, paths); + WorkerProto::write(*this, *conn, paths); conn.processStderr(); - return WorkerProto::Serialise::read(*this, conn->from); + return WorkerProto::Serialise::read(*this, *conn); } } @@ -284,7 +286,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(*this, conn->from); + info.references = WorkerProto::Serialise::read(*this, *conn); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -297,9 +299,9 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S StorePathSet paths; for (auto & path : pathsMap) paths.insert(path.first); - WorkerProto::write(*this, conn->to, paths); + WorkerProto::write(*this, *conn, paths); } else - WorkerProto::write(*this, conn->to, pathsMap); + WorkerProto::write(*this, *conn, pathsMap); conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { @@ -307,7 +309,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(*this, conn->from); + info.references = WorkerProto::Serialise::read(*this, *conn); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -350,7 +352,7 @@ void RemoteStore::queryReferrers(const StorePath & path, auto conn(getConnection()); conn->to << WorkerProto::Op::QueryReferrers << printStorePath(path); conn.processStderr(); - for (auto & i : WorkerProto::Serialise::read(*this, conn->from)) + for (auto & i : WorkerProto::Serialise::read(*this, *conn)) referrers.insert(i); } @@ -360,7 +362,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) auto conn(getConnection()); conn->to << WorkerProto::Op::QueryValidDerivers << printStorePath(path); conn.processStderr(); - return WorkerProto::Serialise::read(*this, conn->from); + return WorkerProto::Serialise::read(*this, *conn); } @@ -372,7 +374,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) auto conn(getConnection()); conn->to << WorkerProto::Op::QueryDerivationOutputs << printStorePath(path); conn.processStderr(); - return WorkerProto::Serialise::read(*this, conn->from); + return WorkerProto::Serialise::read(*this, *conn); } @@ -382,7 +384,7 @@ std::map> RemoteStore::queryPartialDerivat auto conn(getConnection()); conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path); conn.processStderr(); - return WorkerProto::Serialise>>::read(*this, conn->from); + return WorkerProto::Serialise>>::read(*this, *conn); } else { // Fallback for old daemon versions. // For floating-CA derivations (and their co-dependencies) this is an @@ -428,7 +430,7 @@ ref RemoteStore::addCAToStore( << WorkerProto::Op::AddToStore << name << caMethod.render(hashType); - WorkerProto::write(*this, conn->to, references); + WorkerProto::write(*this, *conn, references); conn->to << repair; // The dump source may invoke the store, so we need to make some room. @@ -453,7 +455,7 @@ ref RemoteStore::addCAToStore( name, printHashType(hashType)); std::string s = dump.drain(); conn->to << WorkerProto::Op::AddTextToStore << name << s; - WorkerProto::write(*this, conn->to, references); + WorkerProto::write(*this, *conn, references); conn.processStderr(); }, [&](const FileIngestionMethod & fim) -> void { @@ -519,7 +521,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - WorkerProto::write(*this, sink, info.references); + WorkerProto::write(*this, *conn, info.references); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -529,7 +531,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); - auto importedPaths = WorkerProto::Serialise::read(*this, conn->from); + auto importedPaths = WorkerProto::Serialise::read(*this, *conn); assert(importedPaths.size() <= 1); } @@ -538,7 +540,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - WorkerProto::write(*this, conn->to, info.references); + WorkerProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; @@ -611,7 +613,7 @@ void RemoteStore::registerDrvOutput(const Realisation & info) conn->to << info.id.to_string(); conn->to << std::string(info.outPath.to_string()); } else { - WorkerProto::write(*this, conn->to, info); + WorkerProto::write(*this, *conn, info); } conn.processStderr(); } @@ -634,13 +636,13 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, auto real = [&]() -> std::shared_ptr { if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { auto outPaths = WorkerProto::Serialise>::read( - *this, conn->from); + *this, *conn); if (outPaths.empty()) return nullptr; return std::make_shared(Realisation { .id = id, .outPath = *outPaths.begin() }); } else { auto realisations = WorkerProto::Serialise>::read( - *this, conn->from); + *this, *conn); if (realisations.empty()) return nullptr; return std::make_shared(*realisations.begin()); @@ -651,10 +653,10 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, } catch (...) { return callback.rethrow(); } } -static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector & reqs) +static void writeDerivedPaths(RemoteStore & store, RemoteStore::Connection & conn, const std::vector & reqs) { - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 30) { - WorkerProto::write(store, conn->to, reqs); + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 30) { + WorkerProto::write(store, conn, reqs); } else { Strings ss; for (auto & p : reqs) { @@ -666,12 +668,12 @@ static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, cons [&](const StorePath & drvPath) { throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", store.printStorePath(drvPath), - GET_PROTOCOL_MAJOR(conn->daemonVersion), - GET_PROTOCOL_MINOR(conn->daemonVersion)); + GET_PROTOCOL_MAJOR(conn.daemonVersion), + GET_PROTOCOL_MINOR(conn.daemonVersion)); }, }, sOrDrvPath); } - conn->to << ss; + conn.to << ss; } } @@ -697,7 +699,7 @@ void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMod auto conn(getConnection()); conn->to << WorkerProto::Op::BuildPaths; assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); - writeDerivedPaths(*this, conn, drvPaths); + writeDerivedPaths(*this, *conn, drvPaths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) conn->to << buildMode; else @@ -721,10 +723,10 @@ std::vector RemoteStore::buildPathsWithResults( if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) { conn->to << WorkerProto::Op::BuildPathsWithResults; - writeDerivedPaths(*this, conn, paths); + writeDerivedPaths(*this, *conn, paths); conn->to << buildMode; conn.processStderr(); - return WorkerProto::Serialise>::read(*this, conn->from); + return WorkerProto::Serialise>::read(*this, *conn); } else { // Avoid deadlock. conn_.reset(); @@ -807,7 +809,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime; } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) { - auto builtOutputs = WorkerProto::Serialise::read(*this, conn->from); + auto builtOutputs = WorkerProto::Serialise::read(*this, *conn); for (auto && [output, realisation] : builtOutputs) res.builtOutputs.insert_or_assign( std::move(output.outputName), @@ -866,7 +868,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) conn->to << WorkerProto::Op::CollectGarbage << options.action; - WorkerProto::write(*this, conn->to, options.pathsToDelete); + WorkerProto::write(*this, *conn, options.pathsToDelete); conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ @@ -923,11 +925,11 @@ void RemoteStore::queryMissing(const std::vector & targets, // to prevent a deadlock. goto fallback; conn->to << WorkerProto::Op::QueryMissing; - writeDerivedPaths(*this, conn, targets); + writeDerivedPaths(*this, *conn, targets); conn.processStderr(); - willBuild = WorkerProto::Serialise::read(*this, conn->from); - willSubstitute = WorkerProto::Serialise::read(*this, conn->from); - unknown = WorkerProto::Serialise::read(*this, conn->from); + willBuild = WorkerProto::Serialise::read(*this, *conn); + willSubstitute = WorkerProto::Serialise::read(*this, *conn); + unknown = WorkerProto::Serialise::read(*this, *conn); conn->from >> downloadSize >> narSize; return; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 4f3971bfd..cb7a71acf 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -166,21 +166,7 @@ public: void flushBadConnections(); - struct Connection - { - FdSink to; - FdSource from; - unsigned int daemonVersion; - std::optional remoteTrustsUs; - std::optional daemonNixVersion; - std::chrono::time_point startTime; - - virtual ~Connection(); - - virtual void closeWrite() = 0; - - std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); - }; + struct Connection; ref openConnectionWrapper(); diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 962221ad2..0200076c0 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,6 +1,7 @@ #include "ssh-store-config.hh" #include "store-api.hh" #include "remote-store.hh" +#include "remote-store-connection.hh" #include "remote-fs-accessor.hh" #include "archive.hh" #include "worker-protocol.hh" diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 3bbab371c..2bd6517fa 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -2,6 +2,7 @@ ///@file #include "remote-store.hh" +#include "remote-store-connection.hh" #include "local-fs-store.hh" namespace nix { diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh index f34e55c54..d3d2792ff 100644 --- a/src/libstore/worker-protocol-impl.hh +++ b/src/libstore/worker-protocol-impl.hh @@ -13,65 +13,65 @@ namespace nix { template -std::vector WorkerProto::Serialise>::read(const Store & store, Source & from) +std::vector WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) { std::vector resSet; - auto size = readNum(from); + auto size = readNum(conn.from); while (size--) { - resSet.push_back(WorkerProto::Serialise::read(store, from)); + resSet.push_back(WorkerProto::Serialise::read(store, conn)); } return resSet; } template -void WorkerProto::Serialise>::write(const Store & store, Sink & out, const std::vector & resSet) +void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::vector & resSet) { - out << resSet.size(); + conn.to << resSet.size(); for (auto & key : resSet) { - WorkerProto::Serialise::write(store, out, key); + WorkerProto::Serialise::write(store, conn, key); } } template -std::set WorkerProto::Serialise>::read(const Store & store, Source & from) +std::set WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) { std::set resSet; - auto size = readNum(from); + auto size = readNum(conn.from); while (size--) { - resSet.insert(WorkerProto::Serialise::read(store, from)); + resSet.insert(WorkerProto::Serialise::read(store, conn)); } return resSet; } template -void WorkerProto::Serialise>::write(const Store & store, Sink & out, const std::set & resSet) +void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::set & resSet) { - out << resSet.size(); + conn.to << resSet.size(); for (auto & key : resSet) { - WorkerProto::Serialise::write(store, out, key); + WorkerProto::Serialise::write(store, conn, key); } } template -std::map WorkerProto::Serialise>::read(const Store & store, Source & from) +std::map WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) { std::map resMap; - auto size = readNum(from); + auto size = readNum(conn.from); while (size--) { - auto k = WorkerProto::Serialise::read(store, from); - auto v = WorkerProto::Serialise::read(store, from); + auto k = WorkerProto::Serialise::read(store, conn); + auto v = WorkerProto::Serialise::read(store, conn); resMap.insert_or_assign(std::move(k), std::move(v)); } return resMap; } template -void WorkerProto::Serialise>::write(const Store & store, Sink & out, const std::map & resMap) +void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::map & resMap) { - out << resMap.size(); + conn.to << resMap.size(); for (auto & i : resMap) { - WorkerProto::Serialise::write(store, out, i.first); - WorkerProto::Serialise::write(store, out, i.second); + WorkerProto::Serialise::write(store, conn, i.first); + WorkerProto::Serialise::write(store, conn, i.second); } } diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 0bf9e4d68..a23130743 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -12,31 +12,31 @@ namespace nix { -std::string WorkerProto::Serialise::read(const Store & store, Source & from) +std::string WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { - return readString(from); + return readString(conn.from); } -void WorkerProto::Serialise::write(const Store & store, Sink & out, const std::string & str) +void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const std::string & str) { - out << str; + conn.to << str; } -StorePath WorkerProto::Serialise::read(const Store & store, Source & from) +StorePath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { - return store.parseStorePath(readString(from)); + return store.parseStorePath(readString(conn.from)); } -void WorkerProto::Serialise::write(const Store & store, Sink & out, const StorePath & storePath) +void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const StorePath & storePath) { - out << store.printStorePath(storePath); + conn.to << store.printStorePath(storePath); } -std::optional WorkerProto::Serialise>::read(const Store & store, Source & from) +std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) { - auto temp = readNum(from); + auto temp = readNum(conn.from); switch (temp) { case 0: return std::nullopt; @@ -49,17 +49,17 @@ std::optional WorkerProto::Serialise>::r } } -void WorkerProto::Serialise>::write(const Store & store, Sink & out, const std::optional & optTrusted) +void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & optTrusted) { if (!optTrusted) - out << (uint8_t)0; + conn.to << (uint8_t)0; else { switch (*optTrusted) { case Trusted: - out << (uint8_t)1; + conn.to << (uint8_t)1; break; case NotTrusted: - out << (uint8_t)2; + conn.to << (uint8_t)2; break; default: assert(false); @@ -68,83 +68,83 @@ void WorkerProto::Serialise>::write(const Store & sto } -ContentAddress WorkerProto::Serialise::read(const Store & store, Source & from) +ContentAddress WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { - return ContentAddress::parse(readString(from)); + return ContentAddress::parse(readString(conn.from)); } -void WorkerProto::Serialise::write(const Store & store, Sink & out, const ContentAddress & ca) +void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const ContentAddress & ca) { - out << renderContentAddress(ca); + conn.to << renderContentAddress(ca); } -DerivedPath WorkerProto::Serialise::read(const Store & store, Source & from) +DerivedPath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { - auto s = readString(from); + auto s = readString(conn.from); return DerivedPath::parseLegacy(store, s); } -void WorkerProto::Serialise::write(const Store & store, Sink & out, const DerivedPath & req) +void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const DerivedPath & req) { - out << req.to_string_legacy(store); + conn.to << req.to_string_legacy(store); } -Realisation WorkerProto::Serialise::read(const Store & store, Source & from) +Realisation WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { - std::string rawInput = readString(from); + std::string rawInput = readString(conn.from); return Realisation::fromJSON( nlohmann::json::parse(rawInput), "remote-protocol" ); } -void WorkerProto::Serialise::write(const Store & store, Sink & out, const Realisation & realisation) +void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const Realisation & realisation) { - out << realisation.toJSON().dump(); + conn.to << realisation.toJSON().dump(); } -DrvOutput WorkerProto::Serialise::read(const Store & store, Source & from) +DrvOutput WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { - return DrvOutput::parse(readString(from)); + return DrvOutput::parse(readString(conn.from)); } -void WorkerProto::Serialise::write(const Store & store, Sink & out, const DrvOutput & drvOutput) +void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const DrvOutput & drvOutput) { - out << drvOutput.to_string(); + conn.to << drvOutput.to_string(); } -KeyedBuildResult WorkerProto::Serialise::read(const Store & store, Source & from) +KeyedBuildResult WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { - auto path = WorkerProto::Serialise::read(store, from); - auto br = WorkerProto::Serialise::read(store, from); + auto path = WorkerProto::Serialise::read(store, conn); + auto br = WorkerProto::Serialise::read(store, conn); return KeyedBuildResult { std::move(br), /* .path = */ std::move(path), }; } -void WorkerProto::Serialise::write(const Store & store, Sink & to, const KeyedBuildResult & res) +void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const KeyedBuildResult & res) { - WorkerProto::write(store, to, res.path); - WorkerProto::write(store, to, static_cast(res)); + WorkerProto::write(store, conn, res.path); + WorkerProto::write(store, conn, static_cast(res)); } -BuildResult WorkerProto::Serialise::read(const Store & store, Source & from) +BuildResult WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { BuildResult res; - res.status = (BuildResult::Status) readInt(from); - from + res.status = (BuildResult::Status) readInt(conn.from); + conn.from >> res.errorMsg >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime; - auto builtOutputs = WorkerProto::Serialise::read(store, from); + auto builtOutputs = WorkerProto::Serialise::read(store, conn); for (auto && [output, realisation] : builtOutputs) res.builtOutputs.insert_or_assign( std::move(output.outputName), @@ -152,9 +152,9 @@ BuildResult WorkerProto::Serialise::read(const Store & store, Sourc return res; } -void WorkerProto::Serialise::write(const Store & store, Sink & to, const BuildResult & res) +void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const BuildResult & res) { - to + conn.to << res.status << res.errorMsg << res.timesBuilt @@ -164,30 +164,30 @@ void WorkerProto::Serialise::write(const Store & store, Sink & to, DrvOutputs builtOutputs; for (auto & [output, realisation] : res.builtOutputs) builtOutputs.insert_or_assign(realisation.id, realisation); - WorkerProto::write(store, to, builtOutputs); + WorkerProto::write(store, conn, builtOutputs); } -std::optional WorkerProto::Serialise>::read(const Store & store, Source & from) +std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) { - auto s = readString(from); + auto s = readString(conn.from); return s == "" ? std::optional {} : store.parseStorePath(s); } -void WorkerProto::Serialise>::write(const Store & store, Sink & out, const std::optional & storePathOpt) +void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & storePathOpt) { - out << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); + conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); } -std::optional WorkerProto::Serialise>::read(const Store & store, Source & from) +std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) { - return ContentAddress::parseOpt(readString(from)); + return ContentAddress::parseOpt(readString(conn.from)); } -void WorkerProto::Serialise>::write(const Store & store, Sink & out, const std::optional & caOpt) +void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & caOpt) { - out << (caOpt ? renderContentAddress(*caOpt) : ""); + conn.to << (caOpt ? renderContentAddress(*caOpt) : ""); } } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index cd6801290..ff762c924 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -49,6 +49,28 @@ struct WorkerProto */ enum struct Op : uint64_t; + /** + * A unidirectional read connection, to be used by the read half of the + * canonical serializers below. + * + * This currently is just a `Source &`, but more fields will be added + * later. + */ + struct ReadConn { + Source & from; + }; + + /** + * A unidirectional write connection, to be used by the write half of the + * canonical serializers below. + * + * This currently is just a `Sink &`, but more fields will be added + * later. + */ + struct WriteConn { + Sink & to; + }; + /** * Data type for canonical pairs of serialisers for the worker protocol. * @@ -75,8 +97,8 @@ struct WorkerProto // This makes for a quicker debug cycle, as desired. #if 0 { - static T read(const Store & store, Source & from); - static void write(const Store & store, Sink & out, const T & t); + static T read(const Store & store, ReadConn conn); + static void write(const Store & store, WriteConn conn, const T & t); }; #endif @@ -85,9 +107,9 @@ struct WorkerProto * infer the type instead of having to write it down explicitly. */ template - static void write(const Store & store, Sink & out, const T & t) + static void write(const Store & store, WriteConn conn, const T & t) { - WorkerProto::Serialise::write(store, out, t); + WorkerProto::Serialise::write(store, conn, t); } }; @@ -171,8 +193,8 @@ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op) */ #define MAKE_WORKER_PROTO(T) \ struct WorkerProto::Serialise< T > { \ - static T read(const Store & store, Source & from); \ - static void write(const Store & store, Sink & out, const T & t); \ + static T read(const Store & store, WorkerProto::ReadConn conn); \ + static void write(const Store & store, WorkerProto::WriteConn conn, const T & t); \ }; template<> diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 062c68ff8..caa0248f1 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -807,6 +807,9 @@ static void opServe(Strings opFlags, Strings opArgs) out.flush(); unsigned int clientVersion = readInt(in); + WorkerProto::ReadConn rconn { .from = in }; + WorkerProto::WriteConn wconn { .to = out }; + auto getBuildSettings = [&]() { // FIXME: changing options here doesn't work if we're // building through the daemon. @@ -850,7 +853,7 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::QueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - auto paths = WorkerProto::Serialise::read(*store, in); + auto paths = WorkerProto::Serialise::read(*store, rconn); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -859,19 +862,19 @@ static void opServe(Strings opFlags, Strings opArgs) store->substitutePaths(paths); } - WorkerProto::write(*store, out, store->queryValidPaths(paths)); + WorkerProto::write(*store, wconn, store->queryValidPaths(paths)); break; } case ServeProto::Command::QueryPathInfos: { - auto paths = WorkerProto::Serialise::read(*store, in); + auto paths = WorkerProto::Serialise::read(*store, rconn); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - WorkerProto::write(*store, out, info->references); + WorkerProto::write(*store, wconn, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -899,7 +902,7 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::ExportPaths: { readInt(in); // obsolete - store->exportPaths(WorkerProto::Serialise::read(*store, in), out); + store->exportPaths(WorkerProto::Serialise::read(*store, rconn), out); break; } @@ -945,7 +948,7 @@ static void opServe(Strings opFlags, Strings opArgs) DrvOutputs builtOutputs; for (auto & [output, realisation] : status.builtOutputs) builtOutputs.insert_or_assign(realisation.id, realisation); - WorkerProto::write(*store, out, builtOutputs); + WorkerProto::write(*store, wconn, builtOutputs); } break; @@ -954,9 +957,9 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::QueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; - store->computeFSClosure(WorkerProto::Serialise::read(*store, in), + store->computeFSClosure(WorkerProto::Serialise::read(*store, rconn), closure, false, includeOutputs); - WorkerProto::write(*store, out, closure); + WorkerProto::write(*store, wconn, closure); break; } @@ -971,7 +974,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(*store, in); + info.references = WorkerProto::Serialise::read(*store, rconn); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = ContentAddress::parseOpt(readString(in)); diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 9fe9b3b1e..8e2bcf7e1 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -4,6 +4,7 @@ #include "shared.hh" #include "local-store.hh" #include "remote-store.hh" +#include "remote-store-connection.hh" #include "util.hh" #include "serialise.hh" #include "archive.hh" From 3859cf6b21dddd7e9de729dc42b2f52d8523c239 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 19 Jun 2023 12:18:04 -0400 Subject: [PATCH 030/909] Remove unused `#include` from `local-derivation-goal.cc` These were never needed for this file, and date back to before this was split from `derivation-goal.cc`. --- src/libstore/build/local-derivation-goal.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6e06f6061..09c1fcf08 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -10,8 +10,6 @@ #include "archive.hh" #include "compression.hh" #include "daemon.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" #include "topo-sort.hh" #include "callback.hh" #include "json-utils.hh" From 3910430b9d034c277650f4ff05a27008ede73c18 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Mon, 19 Jun 2023 21:00:49 +0200 Subject: [PATCH 031/909] Add more links in nix-build documentation (#8545) * Add more links in nix-build documentation Co-authored-by: John Ericson --- doc/manual/src/command-ref/nix-build.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md index f70bbd7f2..7369ea2eb 100644 --- a/doc/manual/src/command-ref/nix-build.md +++ b/doc/manual/src/command-ref/nix-build.md @@ -51,8 +51,9 @@ derivation). # Options -All options not listed here are passed to `nix-store --realise`, -except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`. +All options not listed here are passed to +[`nix-store --realise`](nix-store/realise.md), +except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](nix-instantiate.md). - [`--no-out-link`](#opt-no-out-link) From 33d388983156e3cdd360a9bb87662da1b8a35f7a Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Mon, 19 Jun 2023 15:17:50 -0500 Subject: [PATCH 032/909] redirect old platform uninstall instruction links Uninstall instructions were moved to their own page in #8267. The overall section link was redirected in #8286, but platform-specific links (which I give out frequently when I triage installer trouble) weren't included. --- doc/manual/redirects.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 5cd6fdea2..bddd56eb8 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -340,6 +340,8 @@ const redirects = { "attribute-sets": "#attribute-set" }, "installation/installing-binary.html": { + "linux": "uninstall.html#linux", + "macos": "uninstall.html#macos", "uninstalling": "uninstall.html" } }; From a6a75ecad8c1a8bf264eb003b8c07c0fd66f80fb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 20 Jun 2023 11:19:14 +0200 Subject: [PATCH 033/909] GC server: Clear O_NONBLOCK on the right file descriptor The bug fix in 6d30f9e6fea7d451033653f8f167aef58f7f5347 erroneously cleared O_NONBLOCK on the server rather than client FD (leaving both in an incorrect state). Fixes #8551. --- src/libstore/gc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 0038ec802..d023bec71 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -563,7 +563,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* On macOS, accepted sockets inherit the non-blocking flag from the server socket, so explicitly make it blocking. */ - if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) & ~O_NONBLOCK) == -1) + if (fcntl(fdClient.get(), F_SETFL, fcntl(fdClient.get(), F_GETFL) & ~O_NONBLOCK) == -1) abort(); while (true) { From 6ae35534b7b6e10a26a0f2b2a0e37d7f7cfe47dd Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Tue, 20 Jun 2023 10:34:09 +0100 Subject: [PATCH 034/909] Support opening local store with database on read-only filesystem (#8356) Previously it was not possible to open a local store when its database is on a read-only filesystem. Obviously a store on a read-only filesystem cannot be modified, but it would still be useful to be able to query it. This change adds a new read-only setting to LocalStore. When set to true, Nix will skip operations that fail when the database is on a read-only filesystem (acquiring big-lock, schema migration, etc), and the store database will be opened in immutable mode. Co-authored-by: Ben Radford Co-authored-by: cidkidnix Co-authored-by: Dylan Green <67574902+cidkidnix@users.noreply.github.com> Co-authored-by: John Ericson Co-authored-by: Valentin Gagarin --- src/libstore/gc.cc | 5 ++++ src/libstore/local-store.cc | 45 ++++++++++++++++++++++------ src/libstore/local-store.hh | 21 +++++++++++++ src/libstore/sqlite.cc | 11 ++++--- src/libstore/sqlite.hh | 23 +++++++++++++- src/libutil/experimental-features.cc | 9 +++++- src/libutil/experimental-features.hh | 1 + tests/local.mk | 3 +- tests/read-only-store.sh | 42 ++++++++++++++++++++++++++ 9 files changed, 144 insertions(+), 16 deletions(-) create mode 100644 tests/read-only-store.sh diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 0038ec802..3c9544017 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -110,6 +110,11 @@ void LocalStore::createTempRootsFile() void LocalStore::addTempRoot(const StorePath & path) { + if (readOnly) { + debug("Read-only store doesn't support creating lock files for temp roots, but nothing can be deleted anyways."); + return; + } + createTempRootsFile(); /* Open/create the global GC lock file. */ diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7fb312c37..e69460e6c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -190,7 +190,11 @@ LocalStore::LocalStore(const Params & params) /* Create missing state directories if they don't already exist. */ createDirs(realStoreDir); - makeStoreWritable(); + if (readOnly) { + experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore); + } else { + makeStoreWritable(); + } createDirs(linksDir); Path profilesDir = stateDir + "/profiles"; createDirs(profilesDir); @@ -204,8 +208,10 @@ LocalStore::LocalStore(const Params & params) for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { createDirs(perUserDir); - if (chmod(perUserDir.c_str(), 0755) == -1) - throw SysError("could not set permissions on '%s' to 755", perUserDir); + if (!readOnly) { + if (chmod(perUserDir.c_str(), 0755) == -1) + throw SysError("could not set permissions on '%s' to 755", perUserDir); + } } /* Optionally, create directories and set permissions for a @@ -269,10 +275,12 @@ LocalStore::LocalStore(const Params & params) /* Acquire the big fat lock in shared mode to make sure that no schema upgrade is in progress. */ - Path globalLockPath = dbDir + "/big-lock"; - globalLock = openLockFile(globalLockPath.c_str(), true); + if (!readOnly) { + Path globalLockPath = dbDir + "/big-lock"; + globalLock = openLockFile(globalLockPath.c_str(), true); + } - if (!lockFile(globalLock.get(), ltRead, false)) { + if (!readOnly && !lockFile(globalLock.get(), ltRead, false)) { printInfo("waiting for the big Nix store lock..."); lockFile(globalLock.get(), ltRead, true); } @@ -280,6 +288,14 @@ LocalStore::LocalStore(const Params & params) /* Check the current database schema and if necessary do an upgrade. */ int curSchema = getSchema(); + if (readOnly && curSchema < nixSchemaVersion) { + debug("current schema version: %d", curSchema); + debug("supported schema version: %d", nixSchemaVersion); + throw Error(curSchema == 0 ? + "database does not exist, and cannot be created in read-only mode" : + "database schema needs migrating, but this cannot be done in read-only mode"); + } + if (curSchema > nixSchemaVersion) throw Error("current Nix store schema is version %1%, but I only support %2%", curSchema, nixSchemaVersion); @@ -344,7 +360,11 @@ LocalStore::LocalStore(const Params & params) else openDB(*state, false); if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); + if (!readOnly) { + migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); + } else { + throw Error("need to migrate to content-addressed schema, but this cannot be done in read-only mode"); + } } /* Prepare SQL statements. */ @@ -475,13 +495,20 @@ int LocalStore::getSchema() void LocalStore::openDB(State & state, bool create) { - if (access(dbDir.c_str(), R_OK | W_OK)) + if (create && readOnly) { + throw Error("cannot create database while in read-only mode"); + } + + if (access(dbDir.c_str(), R_OK | (readOnly ? 0 : W_OK))) throw SysError("Nix database directory '%1%' is not writable", dbDir); /* Open the Nix database. */ std::string dbPath = dbDir + "/db.sqlite"; auto & db(state.db); - state.db = SQLite(dbPath, create); + auto openMode = readOnly ? SQLiteOpenMode::Immutable + : create ? SQLiteOpenMode::Normal + : SQLiteOpenMode::NoCreate; + state.db = SQLite(dbPath, openMode); #ifdef __CYGWIN__ /* The cygwin version of sqlite3 has a patch which calls diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 70debad38..8a3b0b43f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -46,6 +46,23 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig "require-sigs", "Whether store paths copied into this store should have a trusted signature."}; + Setting readOnly{(StoreConfig*) this, + false, + "read-only", + R"( + Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. + + Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem. + + Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. + + > **Warning** + > Do not use this unless the filesystem is read-only. + > + > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. + > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. + )"}; + const std::string name() override { return "Local Store"; } std::string doc() override; @@ -269,6 +286,10 @@ public: private: + /** + * Retrieve the current version of the database schema. + * If the database does not exist yet, the version returned will be 0. + */ int getSchema(); void openDB(State & state, bool create); diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index df334c23c..7c8decb74 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -1,6 +1,7 @@ #include "sqlite.hh" #include "globals.hh" #include "util.hh" +#include "url.hh" #include @@ -50,15 +51,17 @@ static void traceSQL(void * x, const char * sql) notice("SQL<[%1%]>", sql); }; -SQLite::SQLite(const Path & path, bool create) +SQLite::SQLite(const Path & path, SQLiteOpenMode mode) { // useSQLiteWAL also indicates what virtual file system we need. Using // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem // for Linux (WSL) where useSQLiteWAL should be false by default. const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile"; - int flags = SQLITE_OPEN_READWRITE; - if (create) flags |= SQLITE_OPEN_CREATE; - int ret = sqlite3_open_v2(path.c_str(), &db, flags, vfs); + bool immutable = mode == SQLiteOpenMode::Immutable; + int flags = immutable ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE; + if (mode == SQLiteOpenMode::Normal) flags |= SQLITE_OPEN_CREATE; + auto uri = "file:" + percentEncode(path) + "?immutable=" + (immutable ? "1" : "0"); + int ret = sqlite3_open_v2(uri.c_str(), &db, SQLITE_OPEN_URI | flags, vfs); if (ret != SQLITE_OK) { const char * err = sqlite3_errstr(ret); throw Error("cannot open SQLite database '%s': %s", path, err); diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 6e14852cb..0c08267f7 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -11,6 +11,27 @@ struct sqlite3_stmt; namespace nix { +enum class SQLiteOpenMode { + /** + * Open the database in read-write mode. + * If the database does not exist, it will be created. + */ + Normal, + /** + * Open the database in read-write mode. + * Fails with an error if the database does not exist. + */ + NoCreate, + /** + * Open the database in immutable mode. + * In addition to the database being read-only, + * no wal or journal files will be created by sqlite. + * Use this mode if the database is on a read-only filesystem. + * Fails with an error if the database does not exist. + */ + Immutable, +}; + /** * RAII wrapper to close a SQLite database automatically. */ @@ -18,7 +39,7 @@ struct SQLite { sqlite3 * db = 0; SQLite() { } - SQLite(const Path & path, bool create = true); + SQLite(const Path & path, SQLiteOpenMode mode = SQLiteOpenMode::Normal); SQLite(const SQLite & from) = delete; SQLite& operator = (const SQLite & from) = delete; SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; } diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index c4642d333..7c4112d32 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -221,6 +221,13 @@ constexpr std::array xpFeatureDetails = {{ Allow parsing of timestamps in builtins.fromTOML. )", }, + { + .tag = Xp::ReadOnlyLocalStore, + .name = "read-only-local-store", + .description = R"( + Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs. + )", + }, }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 892c6c371..507b0cc06 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -31,6 +31,7 @@ enum struct ExperimentalFeature DaemonTrustOverride, DynamicDerivations, ParseTomlTimestamps, + ReadOnlyLocalStore, }; /** diff --git a/tests/local.mk b/tests/local.mk index 8e387fe45..88848926b 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -136,7 +136,8 @@ nix_tests = \ impure-derivations.sh \ path-from-hash-part.sh \ test-libstoreconsumer.sh \ - toString-path.sh + toString-path.sh \ + read-only-store.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh diff --git a/tests/read-only-store.sh b/tests/read-only-store.sh new file mode 100644 index 000000000..d63920c19 --- /dev/null +++ b/tests/read-only-store.sh @@ -0,0 +1,42 @@ +source common.sh + +enableFeatures "read-only-local-store" + +needLocalStore "cannot open store read-only when daemon has already opened it writeable" + +clearStore + +happy () { + # We can do a read-only query just fine with a read-only store + nix --store local?read-only=true path-info $dummyPath + + # We can "write" an already-present store-path a read-only store, because no IO is actually required + nix-store --store local?read-only=true --add dummy +} +## Testing read-only mode without forcing the underlying store to actually be read-only + +# Make sure the command fails when the store doesn't already have a database +expectStderr 1 nix-store --store local?read-only=true --add dummy | grepQuiet "database does not exist, and cannot be created in read-only mode" + +# Make sure the store actually has a current-database, with at least one store object +dummyPath=$(nix-store --add dummy) + +# Try again and make sure we fail when adding a item not already in the store +expectStderr 1 nix-store --store local?read-only=true --add eval.nix | grepQuiet "attempt to write a readonly database" + +# Test a few operations that should work with the read-only store in its current state +happy + +## Testing read-only mode with an underlying store that is actually read-only + +# Ensure store is actually read-only +chmod -R -w $TEST_ROOT/store +chmod -R -w $TEST_ROOT/var + +# Make sure we fail on add operations on the read-only store +# This is only for adding files that are not *already* in the store +expectStderr 1 nix-store --add eval.nix | grepQuiet "error: opening lock file '$(readlink -e $TEST_ROOT)/var/nix/db/big-lock'" +expectStderr 1 nix-store --store local?read-only=true --add eval.nix | grepQuiet "Permission denied" + +# Test the same operations from before should again succeed +happy From f2b54e3b71891c29f951e46b00c6edc4807b3b63 Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Sat, 17 Jun 2023 03:25:57 +0200 Subject: [PATCH 035/909] add links to environment variables documentation --- .../src/command-ref/conf-file-prefix.md | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md index 44b7ba86d..d2989ff20 100644 --- a/doc/manual/src/command-ref/conf-file-prefix.md +++ b/doc/manual/src/command-ref/conf-file-prefix.md @@ -6,28 +6,25 @@ By default Nix reads settings from the following places: - - The system-wide configuration file `sysconfdir/nix/nix.conf` (i.e. - `/etc/nix/nix.conf` on most systems), or `$NIX_CONF_DIR/nix.conf` if - `NIX_CONF_DIR` is set. Values loaded in this file are not forwarded - to the Nix daemon. The client assumes that the daemon has already - loaded them. + - The system-wide configuration file `sysconfdir/nix/nix.conf` (i.e. `/etc/nix/nix.conf` on most systems), or `$NIX_CONF_DIR/nix.conf` if [`NIX_CONF_DIR`](./env-common.md#env-NIX_CONF_DIR) is set. - - If `NIX_USER_CONF_FILES` is set, then each path separated by `:` - will be loaded in reverse order. + Values loaded in this file are not forwarded to the Nix daemon. + The client assumes that the daemon has already loaded them. - Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS` - and `XDG_CONFIG_HOME`. If unset, `XDG_CONFIG_DIRS` defaults to - `/etc/xdg`, and `XDG_CONFIG_HOME` defaults to `$HOME/.config` - as per [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). + - If [`NIX_USER_CONF_FILES`](./env-common.md#env-NIX_USER_CONF_FILES) is set, then each path separated by `:` will be loaded in reverse order. - - If `NIX_CONFIG` is set, its contents is treated as the contents of - a configuration file. + Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS` and [`XDG_CONFIG_HOME`](./env-common.md#env-XDG_CONFIG_HOME). + If unset, `XDG_CONFIG_DIRS` defaults to `/etc/xdg`, and `XDG_CONFIG_HOME` defaults to `$HOME/.config` as per [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). + + - If [`NIX_CONFIG`](./env-common.md#env-NIX_CONFIG) is set, its contents are treated as the contents of a configuration file. The configuration files consist of `name = value` pairs, one per line. Other files can be included with a line like `include path`, where *path* is interpreted relative to the current conf file and a missing file is an error unless `!include` is used instead. Comments -start with a `#` character. Here is an example configuration file: +start with a `#` character. + +Here is an example configuration file: keep-outputs = true # Nice for developers keep-derivations = true # Idem From 38bd1cc9bc80e6dec59e56a7d1ce19379915a910 Mon Sep 17 00:00:00 2001 From: fricklerhandwerk Date: Sat, 17 Jun 2023 03:53:37 +0200 Subject: [PATCH 036/909] split configuration file page into sections this makes it easier to scan for specific information, such as the format of command line flags --- .../src/command-ref/conf-file-prefix.md | 53 +++++++++++-------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md index d2989ff20..29325ee45 100644 --- a/doc/manual/src/command-ref/conf-file-prefix.md +++ b/doc/manual/src/command-ref/conf-file-prefix.md @@ -4,6 +4,10 @@ # Description +Nix supports a variety of configuration settings, which are read from configuration files or taken as command line flags. + +## Configuration file + By default Nix reads settings from the following places: - The system-wide configuration file `sysconfdir/nix/nix.conf` (i.e. `/etc/nix/nix.conf` on most systems), or `$NIX_CONF_DIR/nix.conf` if [`NIX_CONF_DIR`](./env-common.md#env-NIX_CONF_DIR) is set. @@ -18,32 +22,39 @@ By default Nix reads settings from the following places: - If [`NIX_CONFIG`](./env-common.md#env-NIX_CONFIG) is set, its contents are treated as the contents of a configuration file. -The configuration files consist of `name = value` pairs, one per -line. Other files can be included with a line like `include path`, -where *path* is interpreted relative to the current conf file and a -missing file is an error unless `!include` is used instead. Comments -start with a `#` character. +### File format -Here is an example configuration file: +Configuration files consist of `name = value` pairs, one per line. +Comments start with a `#` character. - keep-outputs = true # Nice for developers - keep-derivations = true # Idem +Example: -You can override settings on the command line using the `--option` -flag, e.g. `--option keep-outputs false`. Every configuration setting -also has a corresponding command line flag, e.g. `--max-jobs 16`; for -Boolean settings, there are two flags to enable or disable the setting -(e.g. `--keep-failed` and `--no-keep-failed`). +``` +keep-outputs = true # Nice for developers +keep-derivations = true # Idem +``` -A configuration setting usually overrides any previous value. However, -you can prefix the name of the setting by `extra-` to *append* to the -previous value. For instance, +Other files can be included with a line like `include `, where `` is interpreted relative to the current configuration file. +A missing file is an error unless `!include` is used instead. - substituters = a b - extra-substituters = c d +A configuration setting usually overrides any previous value. +However, you can prefix the name of the setting by `extra-` to *append* to the previous value. -defines the `substituters` setting to be `a b c d`. This is also -available as a command line flag (e.g. `--extra-substituters`). +For instance, -The following settings are currently available: +``` +substituters = a b +extra-substituters = c d +``` + +defines the `substituters` setting to be `a b c d`. + +## Command line flags + +Every configuration setting has a corresponding command line flag (e.g. `--max-jobs 16`). +Boolean settings do not need an argument, and can be explicitly disabled with the `no-` prefix (e.g. `--keep-failed` and `--no-keep-failed`). + +Existing settings can be appended to using the `extra-` prefix (e.g. `--extra-substituters`). + +# Available settings From bc7324e912578cdbeb1f10bf1c8d9b447a416dbd Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 20 Jun 2023 12:12:03 +0200 Subject: [PATCH 037/909] clarify read order for configuration settings --- doc/manual/src/command-ref/conf-file-prefix.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md index 29325ee45..3cd622247 100644 --- a/doc/manual/src/command-ref/conf-file-prefix.md +++ b/doc/manual/src/command-ref/conf-file-prefix.md @@ -8,19 +8,19 @@ Nix supports a variety of configuration settings, which are read from configurat ## Configuration file -By default Nix reads settings from the following places: +By default Nix reads settings from the following places, in that order: - - The system-wide configuration file `sysconfdir/nix/nix.conf` (i.e. `/etc/nix/nix.conf` on most systems), or `$NIX_CONF_DIR/nix.conf` if [`NIX_CONF_DIR`](./env-common.md#env-NIX_CONF_DIR) is set. +1. The system-wide configuration file `sysconfdir/nix/nix.conf` (i.e. `/etc/nix/nix.conf` on most systems), or `$NIX_CONF_DIR/nix.conf` if [`NIX_CONF_DIR`](./env-common.md#env-NIX_CONF_DIR) is set. - Values loaded in this file are not forwarded to the Nix daemon. - The client assumes that the daemon has already loaded them. + Values loaded in this file are not forwarded to the Nix daemon. + The client assumes that the daemon has already loaded them. - - If [`NIX_USER_CONF_FILES`](./env-common.md#env-NIX_USER_CONF_FILES) is set, then each path separated by `:` will be loaded in reverse order. +1. If [`NIX_USER_CONF_FILES`](./env-common.md#env-NIX_USER_CONF_FILES) is set, then each path separated by `:` will be loaded in reverse order. - Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS` and [`XDG_CONFIG_HOME`](./env-common.md#env-XDG_CONFIG_HOME). - If unset, `XDG_CONFIG_DIRS` defaults to `/etc/xdg`, and `XDG_CONFIG_HOME` defaults to `$HOME/.config` as per [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). + Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS` and [`XDG_CONFIG_HOME`](./env-common.md#env-XDG_CONFIG_HOME). + If unset, `XDG_CONFIG_DIRS` defaults to `/etc/xdg`, and `XDG_CONFIG_HOME` defaults to `$HOME/.config` as per [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). - - If [`NIX_CONFIG`](./env-common.md#env-NIX_CONFIG) is set, its contents are treated as the contents of a configuration file. +1. If [`NIX_CONFIG`](./env-common.md#env-NIX_CONFIG) is set, its contents are treated as the contents of a configuration file. ### File format @@ -38,7 +38,7 @@ Other files can be included with a line like `include `, where `` is A missing file is an error unless `!include` is used instead. A configuration setting usually overrides any previous value. -However, you can prefix the name of the setting by `extra-` to *append* to the previous value. +However, for settings that take a list of items, you can prefix the name of the setting by `extra-` to *append* to the previous value. For instance, From 68c62193439ec90d34a7988c6a1a73b3e553a435 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 20 Jun 2023 12:12:23 +0200 Subject: [PATCH 038/909] clarify setting options on the command line --- doc/manual/src/command-ref/conf-file-prefix.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md index 3cd622247..1e4085977 100644 --- a/doc/manual/src/command-ref/conf-file-prefix.md +++ b/doc/manual/src/command-ref/conf-file-prefix.md @@ -49,12 +49,22 @@ extra-substituters = c d defines the `substituters` setting to be `a b c d`. +Unknown option names are not an error, and are simply ignored with a warning. + ## Command line flags -Every configuration setting has a corresponding command line flag (e.g. `--max-jobs 16`). -Boolean settings do not need an argument, and can be explicitly disabled with the `no-` prefix (e.g. `--keep-failed` and `--no-keep-failed`). +Configuration options can be set on the command line, overriding the values set in the [configuration file](#configuration-file): -Existing settings can be appended to using the `extra-` prefix (e.g. `--extra-substituters`). +- Every configuration setting has corresponding command line flag (e.g. `--max-jobs 16`). + Boolean settings do not need an argument, and can be explicitly disabled with the `no-` prefix (e.g. `--keep-failed` and `--no-keep-failed`). + + Unknown option names are invalid flags (unless there is already a flag with that name), and are rejected with an error. + +- The flag `--option ` is interpreted exactly like a ` = ` in a setting file. + + Unknown option names are ignored with a warning. + +The `extra-` prefix is supported for settings that take a list of items (e.g. `--extra-trusted users alice` or `--option extra-trusted-users alice`). # Available settings From e91d19db5f827cec56e32fe9b7c07c8d2c546ce6 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 20 Jun 2023 12:23:53 +0200 Subject: [PATCH 039/909] be more serious about security risks with trusted users --- src/nix/daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 6dcd7f94d..41f826228 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -64,7 +64,7 @@ struct AuthorizationSettings : Config { > **Warning** > > Adding a user to `trusted-users` is essentially equivalent to giving that user root access to the system. - > For example, the user can set [`sandbox-paths`](#conf-sandbox-paths) and thereby obtain read access to directories that are otherwise inacessible to them. + > For example, the user can access or replace store path contents that are critical for system security. )"}; /** From 71317162c50df91d421eacdf87d50b4ba6d37635 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 20 Jun 2023 13:44:04 +0200 Subject: [PATCH 040/909] use more self-descriptive section headings --- doc/manual/src/contributing/hacking.md | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index c57d45138..12b91c79e 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -12,14 +12,15 @@ The following instructions assume you already have some version of Nix installed [installation instructions]: ../installation/installation.md -## Nix with flakes +## Building Nix with flakes -This section assumes you are using Nix with [flakes] enabled. See the [next section](#classic-nix) for equivalent instructions which don't require flakes. +This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. +See the [Building Nix](#building-nix) section for equivalent instructions using stable Nix interfaces. -[flakes]: ../command-ref/new-cli/nix3-flake.md#description +[`flakes`]: @docroot@/contributing/experimental-features.md#xp-feature-flakes +[`nix-command`]: @docroot@/contributing/experimental-features.md#xp-nix-command -To build all dependencies and start a shell in which all environment -variables are set up so that those dependencies can be found: +To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: ```console $ nix develop @@ -55,7 +56,7 @@ To install it in `$(pwd)/outputs` and test it: nix (Nix) 2.12 ``` -To build a release version of Nix: +To build a release version of Nix for the current operating system and CPU architecture: ```console $ nix build @@ -63,12 +64,9 @@ $ nix build You can also build Nix for one of the [supported target platforms](#target-platforms). -## Classic Nix +## Building Nix -This section is for Nix without [flakes]. - -To build all dependencies and start a shell in which all environment -variables are set up so that those dependencies can be found: +To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: ```console $ nix-shell @@ -102,7 +100,7 @@ To install it in `$(pwd)/outputs` and test it: nix (Nix) 2.12 ``` -To build Nix for the current operating system and CPU architecture use +To build a release version of Nix for the current operating system and CPU architecture: ```console $ nix-build From 3a20c7c46c64d2feb8caba767724a78d7efa3279 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Tue, 20 Jun 2023 22:51:29 +0200 Subject: [PATCH 041/909] Update tests/linux-sandbox.sh Co-authored-by: John Ericson --- tests/linux-sandbox.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh index 19a7527bf..f2a77174d 100644 --- a/tests/linux-sandbox.sh +++ b/tests/linux-sandbox.sh @@ -52,7 +52,6 @@ testCert () { certFile=$3 # a string that can be the path to a cert file [ "$mode" == fixed-output ] && ret=1 || ret=100 expectStderr $ret nix-sandbox-build linux-sandbox-cert-test.nix --argstr mode "$mode" --option ssl-cert-file "$certFile" | - # tee /dev/stderr | grepQuiet "CERT_${expectation}_IN_SANDBOX" } From a78f929065fd209e3d89853c6beb48942e3820e9 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 20 Jun 2023 12:39:07 +0200 Subject: [PATCH 042/909] fix anchor link --- doc/manual/src/contributing/hacking.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 12b91c79e..310d69310 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -62,7 +62,7 @@ To build a release version of Nix for the current operating system and CPU archi $ nix build ``` -You can also build Nix for one of the [supported target platforms](#target-platforms). +You can also build Nix for one of the [supported target platforms](#platforms). ## Building Nix @@ -106,7 +106,7 @@ To build a release version of Nix for the current operating system and CPU archi $ nix-build ``` -You can also build Nix for one of the [supported target platforms](#target-platforms). +You can also build Nix for one of the [supported target platforms](#platforms). ## Platforms From 085104944b2957f4269fcbfaf0d1874ab0c9ce84 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 21 Jun 2023 09:46:23 +0200 Subject: [PATCH 043/909] add redirects to changed anchors --- doc/manual/redirects.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 5cd6fdea2..6813dc6d6 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -342,6 +342,10 @@ const redirects = { "installation/installing-binary.html": { "uninstalling": "uninstall.html" } + "contributing/hacking.html": { + "nix-with-flakes": "#building-nix-with-flakes" + "classic-nix": "#building-nix" + } }; // the following code matches the current page's URL against the set of redirects. From 5f9a921bc1e6d3e6bf65601be565585cf0190e8f Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 21 Jun 2023 14:31:09 +0200 Subject: [PATCH 044/909] do not use "target", as it's a loaded term in the domain of compilers Co-authored-by: Robert Hensing --- doc/manual/src/contributing/hacking.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 310d69310..6c6f2eb52 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -62,7 +62,7 @@ To build a release version of Nix for the current operating system and CPU archi $ nix build ``` -You can also build Nix for one of the [supported target platforms](#platforms). +You can also build Nix for one of the [supported platforms](#platforms). ## Building Nix @@ -106,7 +106,7 @@ To build a release version of Nix for the current operating system and CPU archi $ nix-build ``` -You can also build Nix for one of the [supported target platforms](#platforms). +You can also build Nix for one of the [supported platforms](#platforms). ## Platforms From 5cc22e337023cee9cbffd3f9d2ff0eb3f18a4d12 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 15 Jun 2023 22:29:03 -0400 Subject: [PATCH 045/909] Clarify docs on deleting generations, including fixing a mistake Deleting store info corrected (there is a foot-gun in Nix with `--delete-generations old`!) Also a few things are cleaned up based on feedback. Co-authored-by: Valentin Gagarin Co-authored-by: Eelco Dolstra --- .../src/command-ref/nix-collect-garbage.md | 2 +- .../command-ref/nix-env/delete-generations.md | 28 ++++++++++++------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md index a679ceaf7..3cab79f0e 100644 --- a/doc/manual/src/command-ref/nix-collect-garbage.md +++ b/doc/manual/src/command-ref/nix-collect-garbage.md @@ -57,7 +57,7 @@ These options are for deleting old [profiles] prior to deleting unreachable [sto Delete all generations of profiles older than the specified amount (except for the generations that were active at that point in time). *period* is a value such as `30d`, which would mean 30 days. - This is the equivalent of invoking [`nix-env --delete-generations `](@docroot@/command-ref/nix-env/delete-generations.md#generations-days) on each found profile. + This is the equivalent of invoking [`nix-env --delete-generations `](@docroot@/command-ref/nix-env/delete-generations.md#generations-time) on each found profile. See the documentation of that command for additional information about the *period* argument. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-env/delete-generations.md b/doc/manual/src/command-ref/nix-env/delete-generations.md index d828a5b9e..adc6fc219 100644 --- a/doc/manual/src/command-ref/nix-env/delete-generations.md +++ b/doc/manual/src/command-ref/nix-env/delete-generations.md @@ -20,22 +20,30 @@ This operation deletes the specified generations of the current profile. - The special value `old` - Delete all generations older than the current one. + Delete all generations except the current one. -- `d`:\ - The last *days* days + > **WARNING** + > + > Older *and newer* generations will be deleted by this operation. + > + > One might expect this to just delete older generations than the curent one, but that is only true if the current generation is also the latest. + > Because one can roll back to a previous generation, it is possible to have generations newer than the current one. + > They will also be deleted. + +- `d`:\ + The last *number* days *Example*: `30d` - Delete all generations older than *days* days. - The generation that was active at that point in time is excluded, and will not be deleted. + Delete all generations created more than *number* days ago, except the most recent one of them. + This allows rolling back to generations that were available within the specified period. -- `+`:\ - The last *count* generations up to the present +- `+`:\ + The last *number* generations up to the present *Example*: `+5` - Keep the last *count* generations, along with any newer than current. + Keep the last *number* generations, along with any newer than current. Periodically deleting old generations is important to make garbage collection effective. @@ -61,7 +69,7 @@ $ nix-env --delete-generations 3 4 8 Delete the generations numbered 3, 4, and 8, so long as the current active generation is not any of those. -## Keep most-recent by count count +## Keep most-recent by count (number of generations) ```console $ nix-env --delete-generations +5 @@ -72,7 +80,7 @@ Suppose `30` is the current generation, and we currently have generations number Then this command will delete generations `20` through `25` (`<= 30 - 5`), and keep generations `26` through `31` (`> 30 - 5`). -## Keep most-recent in days +## Keep most-recent by time (number of days) ```console $ nix-env --delete-generations 30d From c48277c1c15f0b6be6a47271a6f8399fad96f7fd Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Thu, 22 Jun 2023 18:19:42 +0800 Subject: [PATCH 046/909] libexpr: add tests for `nix::Value::print` --- src/libexpr/tests/value/print.cc | 170 +++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 src/libexpr/tests/value/print.cc diff --git a/src/libexpr/tests/value/print.cc b/src/libexpr/tests/value/print.cc new file mode 100644 index 000000000..2268658b2 --- /dev/null +++ b/src/libexpr/tests/value/print.cc @@ -0,0 +1,170 @@ +#include "tests/libexpr.hh" + +#include "value.hh" + +namespace nix { + +using namespace testing; + +struct ValuePrintingTests : LibExprTest +{ + template + void test(Value v, std::string_view expected, A... args) + { + std::stringstream out; + v.print(state.symbols, out, args...); + ASSERT_EQ(out.str(), expected); + } +}; + +TEST_F(ValuePrintingTests, tInt) +{ + Value vInt; + vInt.mkInt(10); + test(vInt, "10"); +} + +TEST_F(ValuePrintingTests, tBool) +{ + Value vBool; + vBool.mkBool(true); + test(vBool, "true"); +} + +TEST_F(ValuePrintingTests, tString) +{ + Value vString; + vString.mkString("some-string"); + test(vString, "\"some-string\""); +} + +TEST_F(ValuePrintingTests, tPath) +{ + Value vPath; + vPath.mkString("/foo"); + test(vPath, "\"/foo\""); +} + +TEST_F(ValuePrintingTests, tNull) +{ + Value vNull; + vNull.mkNull(); + test(vNull, "null"); +} + +TEST_F(ValuePrintingTests, tAttrs) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.symbols.create("one"), &vOne); + builder.insert(state.symbols.create("two"), &vTwo); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + test(vAttrs, "{ one = 1; two = 2; }"); +} + +TEST_F(ValuePrintingTests, tList) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + Value vList; + state.mkList(vList, 5); + vList.bigList.elems[0] = &vOne; + vList.bigList.elems[1] = &vTwo; + vList.bigList.size = 3; + + test(vList, "[ 1 2 (nullptr) ]"); +} + +TEST_F(ValuePrintingTests, vThunk) +{ + Value vThunk; + vThunk.mkThunk(nullptr, nullptr); + + test(vThunk, ""); +} + +TEST_F(ValuePrintingTests, vApp) +{ + Value vApp; + vApp.mkApp(nullptr, nullptr); + + test(vApp, ""); +} + +TEST_F(ValuePrintingTests, vLambda) +{ + Value vLambda; + vLambda.mkLambda(nullptr, nullptr); + + test(vLambda, ""); +} + +TEST_F(ValuePrintingTests, vPrimOp) +{ + Value vPrimOp; + vPrimOp.mkPrimOp(nullptr); + + test(vPrimOp, ""); +} + +TEST_F(ValuePrintingTests, vPrimOpApp) +{ + Value vPrimOpApp; + vPrimOpApp.mkPrimOpApp(nullptr, nullptr); + + test(vPrimOpApp, ""); +} + +TEST_F(ValuePrintingTests, vExternal) +{ + struct MyExternal : ExternalValueBase + { + public: + std::string showType() const override + { + return ""; + } + std::string typeOf() const override + { + return ""; + } + virtual std::ostream & print(std::ostream & str) const override + { + str << "testing-external!"; + return str; + } + } myExternal; + Value vExternal; + vExternal.mkExternal(&myExternal); + + test(vExternal, "testing-external!"); +} + +TEST_F(ValuePrintingTests, vFloat) +{ + Value vFloat; + vFloat.mkFloat(2.0); + + test(vFloat, "2"); +} + +TEST_F(ValuePrintingTests, vBlackhole) +{ + Value vBlackhole; + vBlackhole.mkBlackhole(); + test(vBlackhole, "«potential infinite recursion»"); +} + +} // namespace nix From 1400fde144e1fdde2647fda5a7f9511f25572a35 Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Thu, 22 Jun 2023 12:27:19 +0800 Subject: [PATCH 047/909] libexpr: extend `Value::print` to allow limited depth --- src/libexpr/eval.cc | 20 ++++++---- src/libexpr/tests/value/print.cc | 66 ++++++++++++++++++++++++++++++++ src/libexpr/value.hh | 5 ++- 3 files changed, 81 insertions(+), 10 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 71fd6e6e4..da55d3f9e 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -95,11 +95,16 @@ RootValue allocRootValue(Value * v) #endif } -void Value::print(const SymbolTable & symbols, std::ostream & str, - std::set * seen) const +void Value::print(const SymbolTable &symbols, std::ostream &str, + std::set *seen, int depth) const + { checkInterrupt(); + if (depth <= 0) { + str << "«too deep»"; + return; + } switch (internalType) { case tInt: str << integer; @@ -123,7 +128,7 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, str << "{ "; for (auto & i : attrs->lexicographicOrder(symbols)) { str << symbols[i->name] << " = "; - i->value->print(symbols, str, seen); + i->value->print(symbols, str, seen, depth - 1); str << "; "; } str << "}"; @@ -139,7 +144,7 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, str << "[ "; for (auto v2 : listItems()) { if (v2) - v2->print(symbols, str, seen); + v2->print(symbols, str, seen, depth - 1); else str << "(nullptr)"; str << " "; @@ -181,11 +186,10 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, } } - -void Value::print(const SymbolTable & symbols, std::ostream & str, bool showRepeated) const -{ +void Value::print(const SymbolTable &symbols, std::ostream &str, + bool showRepeated, int depth) const { std::set seen; - print(symbols, str, showRepeated ? nullptr : &seen); + print(symbols, str, showRepeated ? nullptr : &seen, depth); } // Pretty print types for assertion errors diff --git a/src/libexpr/tests/value/print.cc b/src/libexpr/tests/value/print.cc index 2268658b2..5e96e12ec 100644 --- a/src/libexpr/tests/value/print.cc +++ b/src/libexpr/tests/value/print.cc @@ -167,4 +167,70 @@ TEST_F(ValuePrintingTests, vBlackhole) test(vBlackhole, "«potential infinite recursion»"); } +TEST_F(ValuePrintingTests, depthAttrs) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.symbols.create("one"), &vOne); + builder.insert(state.symbols.create("two"), &vTwo); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + BindingsBuilder builder2(state, state.allocBindings(10)); + builder2.insert(state.symbols.create("one"), &vOne); + builder2.insert(state.symbols.create("two"), &vTwo); + builder2.insert(state.symbols.create("nested"), &vAttrs); + + Value vNested; + vNested.mkAttrs(builder2.finish()); + + test(vNested, "{ nested = «too deep»; one = «too deep»; two = «too deep»; }", false, 1); + test(vNested, "{ nested = { one = «too deep»; two = «too deep»; }; one = 1; two = 2; }", false, 2); + test(vNested, "{ nested = { one = 1; two = 2; }; one = 1; two = 2; }", false, 3); + test(vNested, "{ nested = { one = 1; two = 2; }; one = 1; two = 2; }", false, 4); +} + +TEST_F(ValuePrintingTests, depthList) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.symbols.create("one"), &vOne); + builder.insert(state.symbols.create("two"), &vTwo); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + BindingsBuilder builder2(state, state.allocBindings(10)); + builder2.insert(state.symbols.create("one"), &vOne); + builder2.insert(state.symbols.create("two"), &vTwo); + builder2.insert(state.symbols.create("nested"), &vAttrs); + + Value vNested; + vNested.mkAttrs(builder2.finish()); + + Value vList; + state.mkList(vList, 5); + vList.bigList.elems[0] = &vOne; + vList.bigList.elems[1] = &vTwo; + vList.bigList.elems[2] = &vNested; + vList.bigList.size = 3; + + test(vList, "[ «too deep» «too deep» «too deep» ]", false, 1); + test(vList, "[ 1 2 { nested = «too deep»; one = «too deep»; two = «too deep»; } ]", false, 2); + test(vList, "[ 1 2 { nested = { one = «too deep»; two = «too deep»; }; one = 1; two = 2; } ]", false, 3); + test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", false, 4); + test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", false, 5); +} + } // namespace nix diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 89c0c36fd..6b32e2b04 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -2,6 +2,7 @@ ///@file #include +#include #include "symbol-table.hh" #include "value/context.hh" @@ -137,11 +138,11 @@ private: friend std::string showType(const Value & v); - void print(const SymbolTable & symbols, std::ostream & str, std::set * seen) const; + void print(const SymbolTable &symbols, std::ostream &str, std::set *seen, int depth) const; public: - void print(const SymbolTable & symbols, std::ostream & str, bool showRepeated = false) const; + void print(const SymbolTable &symbols, std::ostream &str, bool showRepeated = false, int depth = INT_MAX) const; // Functions needed to distinguish the type // These should be removed eventually, by putting the functionality that's From 97df060588e4d328a7e219107553087149a77bac Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 22 Jun 2023 14:23:25 -0400 Subject: [PATCH 048/909] Better document build failure exit codes - Improved API docs from comment - Exit codes are for `nix-build`, not just `nix-store --release` - Make note in tests so the magic numbers are not surprising Picking up where #8387 left off. --- doc/manual/src/command-ref/nix-build.md | 2 ++ .../src/command-ref/nix-store/realise.md | 31 +---------------- .../src/command-ref/status-build-failure.md | 34 +++++++++++++++++++ src/libstore/build/entry-points.cc | 10 +++--- src/libstore/build/worker.cc | 11 ++---- src/libstore/build/worker.hh | 23 ++++++++++++- tests/check.sh | 3 ++ tests/linux-sandbox.sh | 3 ++ 8 files changed, 72 insertions(+), 45 deletions(-) create mode 100644 doc/manual/src/command-ref/status-build-failure.md diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md index 7369ea2eb..b548edf82 100644 --- a/doc/manual/src/command-ref/nix-build.md +++ b/doc/manual/src/command-ref/nix-build.md @@ -70,6 +70,8 @@ except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](n Change the name of the symlink to the output path created from `result` to *outlink*. +{{#include ./status-build-failure.md}} + {{#include ./opt-common.md}} {{#include ./env-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 6b50d2145..c19aea75e 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -54,36 +54,7 @@ The following flags are available: previous build, the new output path is left in `/nix/store/name.check.` -Special exit codes: - - - `100`\ - Generic build failure, the builder process returned with a non-zero - exit code. - - - `101`\ - Build timeout, the build was aborted because it did not complete - within the specified `timeout`. - - - `102`\ - Hash mismatch, the build output was rejected because it does not - match the [`outputHash` attribute of the - derivation](@docroot@/language/advanced-attributes.md). - - - `104`\ - Not deterministic, the build succeeded in check mode but the - resulting output is not binary reproducible. - -With the `--keep-going` flag it's possible for multiple failures to -occur, in this case the 1xx status codes are or combined using binary -or. - - 1100100 - ^^^^ - |||`- timeout - ||`-- output hash mismatch - |`--- build failure - `---- not deterministic - +{{#include ../status-build-failure.md}} {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/status-build-failure.md b/doc/manual/src/command-ref/status-build-failure.md new file mode 100644 index 000000000..06114eb29 --- /dev/null +++ b/doc/manual/src/command-ref/status-build-failure.md @@ -0,0 +1,34 @@ +# Special exit codes for build failure + +1xx status codes are used when requested builds failed. +The following codes are in use: + +- `100` Generic build failure + + The builder process returned with a non-zero exit code. + +- `101` Build timeout + + The build was aborted because it did not complete within the specified `timeout`. + +- `102` Hash mismatch + + The build output was rejected because it does not match the + [`outputHash` attribute of the derivation](@docroot@/language/advanced-attributes.md). + +- `104` Not deterministic + + The build succeeded in check mode but the resulting output is not binary reproducible. + +With the `--keep-going` flag it's possible for multiple failures to occur. +In this case the 1xx status codes are or combined using +[bitwise OR](https://en.wikipedia.org/wiki/Bitwise_operation#OR). + +``` +0b1100100 + ^^^^ + |||`- timeout + ||`-- output hash mismatch + |`--- build failure + `---- not deterministic +``` diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index edd6cb6d2..4aa4d6dca 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -31,11 +31,11 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod } if (failed.size() == 1 && ex) { - ex->status = worker.exitStatus(); + ex->status = worker.failingExitStatus(); throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); - throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); + throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed)); } } @@ -102,10 +102,10 @@ void Store::ensurePath(const StorePath & path) if (goal->exitCode != Goal::ecSuccess) { if (goal->ex) { - goal->ex->status = worker.exitStatus(); + goal->ex->status = worker.failingExitStatus(); throw std::move(*goal->ex); } else - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); + throw Error(worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); } } @@ -128,7 +128,7 @@ void Store::repairPath(const StorePath & path) goals.insert(worker.makeDerivationGoal(*info->deriver, OutputsSpec::All { }, bmRepair)); worker.run(goals); } else - throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); + throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path)); } } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index ee334d54a..a9ca9cbbc 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -468,16 +468,9 @@ void Worker::waitForInput() } -unsigned int Worker::exitStatus() +unsigned int Worker::failingExitStatus() { - /* - * 1100100 - * ^^^^ - * |||`- timeout - * ||`-- output hash mismatch - * |`--- build failure - * `---- not deterministic - */ + // See API docs in header for explanation unsigned int mask = 0; bool buildFailure = permanentFailure || timedOut || hashMismatch; if (buildFailure) diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 63624d910..5abceca0d 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -280,7 +280,28 @@ public: */ void waitForInput(); - unsigned int exitStatus(); + /*** + * The exit status in case of failure. + * + * In the case of a build failure, returned value follows this + * bitmask: + * + * ``` + * 0b1100100 + * ^^^^ + * |||`- timeout + * ||`-- output hash mismatch + * |`--- build failure + * `---- not deterministic + * ``` + * + * In other words, the failure code is at least 100 (0b1100100), but + * might also be greater. + * + * Otherwise (no build failure, but some other sort of failure by + * assumption), this returned value is 1. + */ + unsigned int failingExitStatus(); /** * Check whether the given valid path exists and has the right diff --git a/tests/check.sh b/tests/check.sh index 645b90222..e13abf747 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -18,6 +18,9 @@ clearStore nix-build dependencies.nix --no-out-link nix-build dependencies.nix --no-out-link --check +# Build failure exit codes (100, 104, etc.) are from +# doc/manual/src/command-ref/status-build-failure.md + # check for dangling temporary build directories # only retain if build fails and --keep-failed is specified, or... # ...build is non-deterministic and --check and --keep-failed are both specified diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh index f2a77174d..ff7d257bd 100644 --- a/tests/linux-sandbox.sh +++ b/tests/linux-sandbox.sh @@ -36,11 +36,13 @@ nix-sandbox-build dependencies.nix --check # Test that sandboxed builds with --check and -K can move .check directory to store nix-sandbox-build check.nix -A nondeterministic +# `100 + 4` means non-determinstic, see doc/manual/src/command-ref/status-build-failure.md expectStderr 104 nix-sandbox-build check.nix -A nondeterministic --check -K > $TEST_ROOT/log grepQuietInverse 'error: renaming' $TEST_ROOT/log grepQuiet 'may not be deterministic' $TEST_ROOT/log # Test that sandboxed builds cannot write to /etc easily +# `100` means build failure without extra info, see doc/manual/src/command-ref/status-build-failure.md expectStderr 100 nix-sandbox-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' | grepQuiet "/etc/test: Permission denied" @@ -50,6 +52,7 @@ testCert () { expectation=$1 # "missing" | "present" mode=$2 # "normal" | "fixed-output" certFile=$3 # a string that can be the path to a cert file + # `100` means build failure without extra info, see doc/manual/src/command-ref/status-build-failure.md [ "$mode" == fixed-output ] && ret=1 || ret=100 expectStderr $ret nix-sandbox-build linux-sandbox-cert-test.nix --argstr mode "$mode" --option ssl-cert-file "$certFile" | grepQuiet "CERT_${expectation}_IN_SANDBOX" From 9d8c4ac446bed82eb22a989cd81982acd722c58a Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Fri, 23 Jun 2023 13:35:41 +0800 Subject: [PATCH 049/909] libexpr: remove unused token `ATTRPATH` in token declaration --- src/libexpr/parser.y | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 3b545fd84..5c9597f8a 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -330,7 +330,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err %type ind_string_parts %type path_start string_parts string_attr %type attr -%token ID ATTRPATH +%token ID %token STR IND_STR %token INT %token FLOAT From 484290a9e05ea840f9bc6ba3cc98d64ccffe202b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 23 Jun 2023 12:01:10 -0400 Subject: [PATCH 050/909] Use a struct not `std::pair` for `SearchPathElem` I got very confused trying to keep all the `first` and `second` straight reading the code, *especially* as there is also another `(boolean, string)` pair type also being used. Named fields is much better. There are other cleanups that we can do (for example, the existing TODO), but we can do them later. Doing them now would just make this harder to review. --- src/libexpr/eval.hh | 8 ++++++-- src/libexpr/parser.y | 33 ++++++++++++++++++--------------- src/libexpr/primops.cc | 9 ++++++--- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 8e41bdbd0..7b726a78f 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -65,8 +65,12 @@ std::string printValue(const EvalState & state, const Value & v); std::ostream & operator << (std::ostream & os, const ValueType t); -// FIXME: maybe change this to an std::variant. -typedef std::pair SearchPathElem; +struct SearchPathElem +{ + std::string prefix; + // FIXME: maybe change this to an std::variant. + std::string path; +}; typedef std::list SearchPath; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 5c9597f8a..03e6fb02c 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -741,7 +741,10 @@ void EvalState::addToSearchPath(const std::string & s) path = std::string(s, pos + 1); } - searchPath.emplace_back(prefix, path); + searchPath.emplace_back(SearchPathElem { + .prefix = prefix, + .path = path, + }); } @@ -755,11 +758,11 @@ SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view p { for (auto & i : searchPath) { std::string suffix; - if (i.first.empty()) + if (i.prefix.empty()) suffix = concatStrings("/", path); else { - auto s = i.first.size(); - if (path.compare(0, s, i.first) != 0 || + auto s = i.prefix.size(); + if (path.compare(0, s, i.prefix) != 0 || (path.size() > s && path[s] != '/')) continue; suffix = path.size() == s ? "" : concatStrings("/", path.substr(s)); @@ -785,47 +788,47 @@ SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view p std::pair EvalState::resolveSearchPathElem(const SearchPathElem & elem) { - auto i = searchPathResolved.find(elem.second); + auto i = searchPathResolved.find(elem.path); if (i != searchPathResolved.end()) return i->second; std::pair res; - if (EvalSettings::isPseudoUrl(elem.second)) { + if (EvalSettings::isPseudoUrl(elem.path)) { try { auto storePath = fetchers::downloadTarball( - store, EvalSettings::resolvePseudoUrl(elem.second), "source", false).tree.storePath; + store, EvalSettings::resolvePseudoUrl(elem.path), "source", false).tree.storePath; res = { true, store->toRealPath(storePath) }; } catch (FileTransferError & e) { logWarning({ - .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second) + .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.path) }); res = { false, "" }; } } - else if (hasPrefix(elem.second, "flake:")) { + else if (hasPrefix(elem.path, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(elem.second.substr(6), {}, true, false); - debug("fetching flake search path element '%s''", elem.second); + auto flakeRef = parseFlakeRef(elem.path.substr(6), {}, true, false); + debug("fetching flake search path element '%s''", elem.path); auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; res = { true, store->toRealPath(storePath) }; } else { - auto path = absPath(elem.second); + auto path = absPath(elem.path); if (pathExists(path)) res = { true, path }; else { logWarning({ - .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.second) + .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.path) }); res = { false, "" }; } } - debug("resolved search path element '%s' to '%s'", elem.second, res.second); + debug("resolved search path element '%s' to '%s'", elem.path, res.second); - searchPathResolved[elem.second] = res; + searchPathResolved[elem.path] = res; return res; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5b2f7e8b7..9dd7bae02 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1656,7 +1656,10 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V })); } - searchPath.emplace_back(prefix, path); + searchPath.emplace_back(SearchPathElem { + .prefix = prefix, + .path = path, + }); } auto path = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.findFile"); @@ -4129,8 +4132,8 @@ void EvalState::createBaseEnv() int n = 0; for (auto & i : searchPath) { auto attrs = buildBindings(2); - attrs.alloc("path").mkString(i.second); - attrs.alloc("prefix").mkString(i.first); + attrs.alloc("path").mkString(i.path); + attrs.alloc("prefix").mkString(i.prefix); (v.listElems()[n++] = allocValue())->mkAttrs(attrs); } addConstant("__nixPath", v); From a7b49086c76bc181ab1cf1d3c7fba44b06d2f1dd Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Thu, 14 Oct 2021 23:44:45 +1100 Subject: [PATCH 051/909] Add `dirtyRev` and `dirtyShortRev` to `fetchGit` Fixes #4682 --- src/libexpr/primops/fetchTree.cc | 7 ++++++- src/libfetchers/git.cc | 9 ++++++++- src/nix/flake.cc | 6 ++++++ tests/fetchGit.sh | 6 ++++++ tests/flakes/flakes.sh | 5 +++++ 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index cd7039025..bfae34a48 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -22,7 +22,7 @@ void emitTreeAttrs( { assert(input.isLocked()); - auto attrs = state.buildBindings(8); + auto attrs = state.buildBindings(10); state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath)); @@ -56,6 +56,11 @@ void emitTreeAttrs( } + if (auto dirtyRev = fetchers::maybeGetStrAttr(input.attrs, "dirtyRev")) { + attrs.alloc("dirtyRev").mkString(*dirtyRev); + attrs.alloc("dirtyShortRev").mkString(*fetchers::maybeGetStrAttr(input.attrs, "dirtyShortRev")); + } + if (auto lastModified = input.getLastModified()) { attrs.alloc("lastModified").mkInt(*lastModified); attrs.alloc("lastModifiedDate").mkString( diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 47282f6c4..be5842d53 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -243,6 +243,13 @@ std::pair fetchFromWorkdir(ref store, Input & input, co "lastModified", workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); + if (workdirInfo.hasHead) { + input.attrs.insert_or_assign("dirtyRev", chomp( + runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "HEAD" })) + "-dirty"); + input.attrs.insert_or_assign("dirtyShortRev", chomp( + runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "--short", "HEAD" })) + "-dirty"); + } + return {std::move(storePath), input}; } } // end namespace @@ -283,7 +290,7 @@ struct GitInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "git") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name") + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev") throw Error("unsupported Git input attribute '%s'", name); parseURL(getStrAttr(attrs, "url")); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 1eea52e15..8d39de389 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -179,6 +179,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs()); if (auto rev = flake.lockedRef.input.getRev()) j["revision"] = rev->to_string(Base16, false); + if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) + j["dirtyRevision"] = *dirtyRev; if (auto revCount = flake.lockedRef.input.getRevCount()) j["revCount"] = *revCount; if (auto lastModified = flake.lockedRef.input.getLastModified()) @@ -204,6 +206,10 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", rev->to_string(Base16, false)); + if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) + logger->cout( + ANSI_BOLD "Revision:" ANSI_NORMAL " %s", + *dirtyRev); if (auto revCount = flake.lockedRef.input.getRevCount()) logger->cout( ANSI_BOLD "Revisions:" ANSI_NORMAL " %s", diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index e2ccb0e97..418b4f63f 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -105,6 +105,8 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") [[ $(cat $path2/dir1/foo) = foo ]] [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).dirtyRev") = "${rev2}-dirty" ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).dirtyShortRev") = "${rev2:0:7}-dirty" ]] # ... unless we're using an explicit ref or rev. path3=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") @@ -119,6 +121,10 @@ git -C $repo commit -m 'Bla3' -a path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$repo).outPath") [[ $path2 = $path4 ]] +[[ $(nix eval --impure --expr "builtins.hasAttr \"rev\" (builtins.fetchGit $repo)") == "true" ]] +[[ $(nix eval --impure --expr "builtins.hasAttr \"dirtyRev\" (builtins.fetchGit $repo)") == "false" ]] +[[ $(nix eval --impure --expr "builtins.hasAttr \"dirtyShortRev\" (builtins.fetchGit $repo)") == "false" ]] + status=0 nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-B5yIPHhEm0eysJKEsO7nqxprh9vcblFxpJG11gXJus1=\"; }).outPath" || status=$? [[ "$status" = "102" ]] diff --git a/tests/flakes/flakes.sh b/tests/flakes/flakes.sh index f2e216435..128f759ea 100644 --- a/tests/flakes/flakes.sh +++ b/tests/flakes/flakes.sh @@ -95,11 +95,16 @@ json=$(nix flake metadata flake1 --json | jq .) [[ $(echo "$json" | jq -r .lastModified) = $(git -C $flake1Dir log -n1 --format=%ct) ]] hash1=$(echo "$json" | jq -r .revision) +echo foo > $flake1Dir/foo +git -C $flake1Dir add $flake1Dir/foo +[[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]] + echo -n '# foo' >> $flake1Dir/flake.nix flake1OriginalCommit=$(git -C $flake1Dir rev-parse HEAD) git -C $flake1Dir commit -a -m 'Foo' flake1NewCommit=$(git -C $flake1Dir rev-parse HEAD) hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision) +[[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "null" ]] [[ $hash1 != $hash2 ]] # Test 'nix build' on a flake. From f11445952fcf0f5fd3f827997591d43a41f2074a Mon Sep 17 00:00:00 2001 From: Michal Sojka Date: Sun, 11 Jun 2023 21:36:56 +0200 Subject: [PATCH 052/909] Document builtins.fetchTree Co-authored-by: Valentin Gagarin Supersedes #6740 --- src/libexpr/primops/fetchTree.cc | 43 ++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 579a45f92..3b048c446 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -198,10 +198,49 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false }); } -// FIXME: document static RegisterPrimOp primop_fetchTree({ .name = "fetchTree", - .arity = 1, + .args = {"input"}, + .doc = R"( + Fetch a source tree or a plain file using one of the supported backends. + *input* can be an attribute set representation of [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) or a URL. + The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is allowed. + + Here are some examples of how to use `fetchTree`: + + - Fetch a GitHub repository: + + ```nix + builtins.fetchTree { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; + } + ``` + + This evaluates to attribute set: + + ``` + { + lastModified = 1686503798; + lastModifiedDate = "20230611171638"; + narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc="; + outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source"; + rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; + shortRev = "ae2e6b3"; + } + ``` + - Fetch a single file from a URL: + + ```nix + builtins.fetchTree "https://example.com/" + ``` + + > **Note** + > + > This function requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. + )", .fun = prim_fetchTree }); From 559fd7ffe7a8cb61b11bec081f14ce3c3ffb210d Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 27 Jun 2023 14:58:29 +0200 Subject: [PATCH 053/909] nix flake check: improve error message if overlay is not a lambda (#8582) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * nix flake check: improve error message if overlay is not a lambda Suppose you have an overlay like this { inputs = { /* ... */ }; outputs = { flake-utils, ... }: flake-utils.lib.eachDefaultSystem (system: { overlays.default = final: prev: { }; }); } then `nix flake check` (correctly) fails because `overlays` are supposed to have the structure `overlays. = final: prev: exp`. However, the error-message is a little bit counter-intuitive: error: overlay does not take an argument named 'final' While one might guess where the error actually comes from because the trace above says `… while checking the overlay 'overlays.x86_64-linux'` this is still pretty confusing because it complains about an argument not being named `final` even though that's evidently the case. With this change, the error-message actually makes it clear what's wrong: [ma27@carsten:~/Projects/nix/tmp]$ nix flake check --extra-experimental-features 'nix-command flakes' path:$(pwd) error: … while checking flake output 'overlays' at /nix/store/clgblnxx003hyrq8qkz5ab6kgqkck6qc-source/flake.nix:4:5: 3| outputs = { ... }: { 4| overlays.x86_64-linux.snens = final: prev: { | ^ 5| kek = throw "snens"; … while checking the overlay 'overlays.x86_64-linux' at /nix/store/clgblnxx003hyrq8qkz5ab6kgqkck6qc-source/flake.nix:4:5: 3| outputs = { ... }: { 4| overlays.x86_64-linux.snens = final: prev: { | ^ 5| kek = throw "snens"; error: overlay is not a lambda, but a set instead --- src/nix/flake.cc | 6 ++++-- tests/flakes/check.sh | 12 ++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 8d39de389..b5f5d0cac 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -386,8 +386,10 @@ struct CmdFlakeCheck : FlakeCommand auto checkOverlay = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { state->forceValue(v, pos); - if (!v.isLambda() - || v.lambda.fun->hasFormals() + if (!v.isLambda()) { + throw Error("overlay is not a function, but %s instead", showType(v)); + } + if (v.lambda.fun->hasFormals() || !argHasName(v.lambda.fun->arg, "final")) throw Error("overlay does not take an argument named 'final'"); auto body = dynamic_cast(v.lambda.fun->body); diff --git a/tests/flakes/check.sh b/tests/flakes/check.sh index 34b82c61c..0433e5335 100644 --- a/tests/flakes/check.sh +++ b/tests/flakes/check.sh @@ -25,6 +25,18 @@ EOF (! nix flake check $flakeDir) +cat > $flakeDir/flake.nix <&1 && fail "nix flake check --all-systems should have failed" || true) +echo "$checkRes" | grepQuiet "error: overlay is not a function, but a set instead" + cat > $flakeDir/flake.nix < Date: Wed, 21 Jun 2023 16:06:16 -0400 Subject: [PATCH 054/909] Generialize `showType` --- src/libexpr/eval.cc | 23 ++++++++++++----------- src/libexpr/eval.hh | 5 ++++- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 71fd6e6e4..8a0ff4cce 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -211,20 +211,21 @@ const Value * getPrimOp(const Value &v) { return primOp; } -std::string_view showType(ValueType type) +std::string_view showType(ValueType type, bool withArticle) { + #define WA(a, w) withArticle ? a " " w : w switch (type) { - case nInt: return "an integer"; - case nBool: return "a Boolean"; - case nString: return "a string"; - case nPath: return "a path"; + case nInt: return WA("an", "integer"); + case nBool: return WA("a", "Boolean"); + case nString: return WA("a", "string"); + case nPath: return WA("a", "path"); case nNull: return "null"; - case nAttrs: return "a set"; - case nList: return "a list"; - case nFunction: return "a function"; - case nExternal: return "an external value"; - case nFloat: return "a float"; - case nThunk: return "a thunk"; + case nAttrs: return WA("a", "set"); + case nList: return WA("a", "list"); + case nFunction: return WA("a", "function"); + case nExternal: return WA("an", "external value"); + case nFloat: return WA("a", "float"); + case nThunk: return WA("a", "thunk"); } abort(); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 7b726a78f..0c07ae081 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -704,8 +704,11 @@ struct DebugTraceStacker { /** * @return A string representing the type of the value `v`. + * + * @param withArticle Whether to begin with an english article, e.g. "an + * integer" vs "integer". */ -std::string_view showType(ValueType type); +std::string_view showType(ValueType type, bool withArticle = true); std::string showType(const Value & v); /** From 4da7c866181c699b43d3cd2fff0afd2505774be7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 27 Jun 2023 09:32:40 -0400 Subject: [PATCH 055/909] Switch example to a primop this is less ill-advised Any primop will do for this, so might as well use one that isn't impure. Co-authored-by: Robert Hensing --- doc/manual/src/language/builtin-constants.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/language/builtin-constants.md b/doc/manual/src/language/builtin-constants.md index e6bc7e915..eb8b53bf0 100644 --- a/doc/manual/src/language/builtin-constants.md +++ b/doc/manual/src/language/builtin-constants.md @@ -9,7 +9,8 @@ These constants are built into the Nix language evaluator: Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations: ```nix - if builtins ? getEnv then builtins.getEnv "PATH" else "" + # if hasContext is not available, we assume `s` has a context + if builtins ? hasContext then builtins.hasContext s else true ``` - [`builtins.currentSystem`]{#builtins-currentSystem} (string) From d40f0e534d657e619634ba204fb8970f4a01fbc5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 27 Jun 2023 09:35:51 -0400 Subject: [PATCH 056/909] Don't say this when we still pollute the global scope Co-authored-by: Robert Hensing --- doc/manual/src/language/builtin-constants.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/language/builtin-constants.md b/doc/manual/src/language/builtin-constants.md index eb8b53bf0..99520ad3f 100644 --- a/doc/manual/src/language/builtin-constants.md +++ b/doc/manual/src/language/builtin-constants.md @@ -4,7 +4,7 @@ These constants are built into the Nix language evaluator: - [`builtins`]{#builtins-builtins} (attribute set) - Contains all the [built-in functions](./builtins.md) and values, in order to avoid polluting the global scope. + Contains all the [built-in functions](./builtins.md) and values. Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations: From 22b278e011ab9c1328749a126514c57b89a39173 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 13 May 2023 13:52:45 -0400 Subject: [PATCH 057/909] Automatically document builtin constants This is done in roughly the same way builtin functions are documented. Also auto-link experimental features for primops, subsuming PR #8371. Co-authored-by: Eelco Dolstra Co-authored-by: Robert Hensing Co-authored-by: Valentin Gagarin --- .gitignore | 3 +- doc/manual/generate-builtin-constants.nix | 29 ++ doc/manual/generate-builtins.nix | 18 +- doc/manual/local.mk | 16 +- .../src/language/builtin-constants-prefix.md | 5 + .../src/language/builtin-constants-suffix.md | 1 + doc/manual/src/language/builtin-constants.md | 44 --- src/libexpr/eval.cc | 37 ++- src/libexpr/eval.hh | 87 +++++- src/libexpr/flake/flake.cc | 3 - src/libexpr/primops.cc | 283 +++++++++++++++--- src/libexpr/primops.hh | 14 +- src/libexpr/primops/fetchClosure.cc | 3 - src/libexpr/value.hh | 10 +- src/nix/main.cc | 41 ++- 15 files changed, 447 insertions(+), 147 deletions(-) create mode 100644 doc/manual/generate-builtin-constants.nix create mode 100644 doc/manual/src/language/builtin-constants-prefix.md create mode 100644 doc/manual/src/language/builtin-constants-suffix.md delete mode 100644 doc/manual/src/language/builtin-constants.md diff --git a/.gitignore b/.gitignore index 29d9106ae..969194650 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,7 @@ perl/Makefile.config /doc/manual/generated/* /doc/manual/nix.json /doc/manual/conf-file.json -/doc/manual/builtins.json +/doc/manual/language.json /doc/manual/xp-features.json /doc/manual/src/SUMMARY.md /doc/manual/src/command-ref/new-cli @@ -26,6 +26,7 @@ perl/Makefile.config /doc/manual/src/command-ref/experimental-features-shortlist.md /doc/manual/src/contributing/experimental-feature-descriptions.md /doc/manual/src/language/builtins.md +/doc/manual/src/language/builtin-constants.md # /scripts/ /scripts/nix-profile.sh diff --git a/doc/manual/generate-builtin-constants.nix b/doc/manual/generate-builtin-constants.nix new file mode 100644 index 000000000..3fc1fae42 --- /dev/null +++ b/doc/manual/generate-builtin-constants.nix @@ -0,0 +1,29 @@ +let + inherit (builtins) concatStringsSep attrValues mapAttrs; + inherit (import ./utils.nix) optionalString squash; +in + +builtinsInfo: +let + showBuiltin = name: { doc, type, impure-only }: + let + type' = optionalString (type != null) " (${type})"; + + impureNotice = optionalString impure-only '' + Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). + ''; + in + squash '' +
+ ${name}${type'} +
+
+ + ${doc} + + ${impureNotice} + +
+ ''; +in +concatStringsSep "\n" (attrValues (mapAttrs showBuiltin builtinsInfo)) diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 71f96153f..813a287f5 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -1,24 +1,28 @@ let - inherit (builtins) concatStringsSep attrNames; + inherit (builtins) concatStringsSep attrValues mapAttrs; + inherit (import ./utils.nix) optionalString squash; in builtinsInfo: let - showBuiltin = name: + showBuiltin = name: { doc, args, arity, experimental-feature }: let - inherit (builtinsInfo.${name}) doc args; + experimentalNotice = optionalString (experimental-feature != null) '' + This function is only available if the [${experimental-feature}](@docroot@/contributing/experimental-features.md#xp-feature-${experimental-feature}) experimental feature is enabled. + ''; in - '' + squash ''
${name} ${listArgs args}
- ${doc} + ${doc} + + ${experimentalNotice}
''; listArgs = args: concatStringsSep " " (map (s: "${s}") args); in -concatStringsSep "\n" (map showBuiltin (attrNames builtinsInfo)) - +concatStringsSep "\n" (attrValues (mapAttrs showBuiltin builtinsInfo)) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index b4b7283ef..abdfd6a62 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -128,14 +128,20 @@ $(d)/xp-features.json: $(bindir)/nix $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp @mv $@.tmp $@ -$(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix +$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix @cat doc/manual/src/language/builtins-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<)).builtins' >> $@.tmp; @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ -$(d)/builtins.json: $(bindir)/nix - $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp +$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(bindir)/nix + @cat doc/manual/src/language/builtin-constants-prefix.md > $@.tmp + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtin-constants.nix (builtins.fromJSON (builtins.readFile $<)).constants' >> $@.tmp; + @cat doc/manual/src/language/builtin-constants-suffix.md >> $@.tmp + @mv $@.tmp $@ + +$(d)/language.json: $(bindir)/nix + $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-language > $@.tmp @mv $@.tmp $@ # Generate the HTML manual. @@ -167,7 +173,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/src/language/builtin-constants-prefix.md b/doc/manual/src/language/builtin-constants-prefix.md new file mode 100644 index 000000000..50f43006d --- /dev/null +++ b/doc/manual/src/language/builtin-constants-prefix.md @@ -0,0 +1,5 @@ +# Built-in Constants + +These constants are built into the Nix language evaluator: + +
diff --git a/doc/manual/src/language/builtin-constants-suffix.md b/doc/manual/src/language/builtin-constants-suffix.md new file mode 100644 index 000000000..a74db2857 --- /dev/null +++ b/doc/manual/src/language/builtin-constants-suffix.md @@ -0,0 +1 @@ +
diff --git a/doc/manual/src/language/builtin-constants.md b/doc/manual/src/language/builtin-constants.md deleted file mode 100644 index 99520ad3f..000000000 --- a/doc/manual/src/language/builtin-constants.md +++ /dev/null @@ -1,44 +0,0 @@ -# Built-in Constants - -These constants are built into the Nix language evaluator: - -- [`builtins`]{#builtins-builtins} (attribute set) - - Contains all the [built-in functions](./builtins.md) and values. - - Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations: - - ```nix - # if hasContext is not available, we assume `s` has a context - if builtins ? hasContext then builtins.hasContext s else true - ``` - -- [`builtins.currentSystem`]{#builtins-currentSystem} (string) - - The built-in value `currentSystem` evaluates to the Nix platform - identifier for the Nix installation on which the expression is being - evaluated, such as `"i686-linux"` or `"x86_64-darwin"`. - - Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). - -- [`builtins.currentTime`]{#builtins-currentTime} (integer) - - Return the [Unix time](https://en.wikipedia.org/wiki/Unix_time) at first evaluation. - Repeated references to that name will re-use the initially obtained value. - - Example: - - ```console - $ nix repl - Welcome to Nix 2.15.1 Type :? for help. - - nix-repl> builtins.currentTime - 1683705525 - - nix-repl> builtins.currentTime - 1683705525 - ``` - - The [store path](@docroot@/glossary.md#gloss-store-path) of a derivation depending on `currentTime` will differ for each evaluation. - - Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 8a0ff4cce..aaeb419ad 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -703,28 +703,34 @@ Path EvalState::toRealPath(const Path & path, const NixStringContext & context) } -Value * EvalState::addConstant(const std::string & name, Value & v) +Value * EvalState::addConstant(const std::string & name, Value & v, Constant info) { Value * v2 = allocValue(); *v2 = v; - addConstant(name, v2); + addConstant(name, v2, info); return v2; } -void EvalState::addConstant(const std::string & name, Value * v) +void EvalState::addConstant(const std::string & name, Value * v, Constant info) { - staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl); - baseEnv.values[baseEnvDispl++] = v; auto name2 = name.substr(0, 2) == "__" ? name.substr(2) : name; - baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v)); -} + constantInfos.push_back({name2, info}); -Value * EvalState::addPrimOp(const std::string & name, - size_t arity, PrimOpFun primOp) -{ - return addPrimOp(PrimOp { .fun = primOp, .arity = arity, .name = name }); + if (!(evalSettings.pureEval && info.impureOnly)) { + /* Check the type, if possible. + + We might know the type of a thunk in advance, so be allowed + to just write it down in that case. */ + if (auto gotType = v->type(true); gotType != nThunk) + assert(info.type == gotType); + + /* Install value the base environment. */ + staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl); + baseEnv.values[baseEnvDispl++] = v; + baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v)); + } } @@ -738,7 +744,10 @@ Value * EvalState::addPrimOp(PrimOp && primOp) vPrimOp->mkPrimOp(new PrimOp(primOp)); Value v; v.mkApp(vPrimOp, vPrimOp); - return addConstant(primOp.name, v); + return addConstant(primOp.name, v, { + .type = nThunk, // FIXME + .doc = primOp.doc, + }); } auto envName = symbols.create(primOp.name); @@ -764,13 +773,13 @@ std::optional EvalState::getDoc(Value & v) { if (v.isPrimOp()) { auto v2 = &v; - if (v2->primOp->doc) + if (auto * doc = v2->primOp->doc) return Doc { .pos = {}, .name = v2->primOp->name, .arity = v2->primOp->arity, .args = v2->primOp->args, - .doc = v2->primOp->doc, + .doc = doc, }; } return {}; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 0c07ae081..e3676c1b7 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -25,15 +25,72 @@ struct DerivedPath; enum RepairFlag : bool; +/** + * Function that implements a primop. + */ typedef void (* PrimOpFun) (EvalState & state, const PosIdx pos, Value * * args, Value & v); +/** + * Info about a primitive operation, and its implementation + */ struct PrimOp { - PrimOpFun fun; - size_t arity; + /** + * Name of the primop. `__` prefix is treated specially. + */ std::string name; + + /** + * Names of the parameters of a primop, for primops that take a + * fixed number of arguments to be substituted for these parameters. + */ std::vector args; + + /** + * Aritiy of the primop. + * + * If `args` is not empty, this field will be computed from that + * field instead, so it doesn't need to be manually set. + */ + size_t arity = 0; + + /** + * Optional free-form documentation about the primop. + */ const char * doc = nullptr; + + /** + * Implementation of the primop. + */ + PrimOpFun fun; + + /** + * Optional experimental for this to be gated on. + */ + std::optional experimentalFeature; +}; + +/** + * Info about a constant + */ +struct Constant +{ + /** + * Optional type of the constant (known since it is a fixed value). + * + * @todo we should use an enum for this. + */ + ValueType type = nThunk; + + /** + * Optional free-form documentation about the constant. + */ + const char * doc = nullptr; + + /** + * Whether the constant is impure, and not available in pure mode. + */ + bool impureOnly = false; }; #if HAVE_BOEHMGC @@ -513,18 +570,23 @@ public: */ std::shared_ptr staticBaseEnv; // !!! should be private + /** + * Name and documentation about every constant. + * + * Constants from primops are hard to crawl, and their docs will go + * here too. + */ + std::vector> constantInfos; + private: unsigned int baseEnvDispl = 0; void createBaseEnv(); - Value * addConstant(const std::string & name, Value & v); + Value * addConstant(const std::string & name, Value & v, Constant info); - void addConstant(const std::string & name, Value * v); - - Value * addPrimOp(const std::string & name, - size_t arity, PrimOpFun primOp); + void addConstant(const std::string & name, Value * v, Constant info); Value * addPrimOp(PrimOp && primOp); @@ -538,6 +600,10 @@ public: std::optional name; size_t arity; std::vector args; + /** + * Unlike the other `doc` fields in this file, this one should never be + * `null`. + */ const char * doc; }; @@ -740,7 +806,12 @@ struct EvalSettings : Config Setting nixPath{ this, getDefaultNixPath(), "nix-path", - "List of directories to be searched for `<...>` file references."}; + R"( + List of directories to be searched for `<...>` file references + + In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of + [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtin-constants-nixPath). + )"}; Setting restrictEval{ this, false, "restrict-eval", diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 60bb6a71e..5aa44d6a1 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -788,9 +788,6 @@ static RegisterPrimOp r2({ ```nix (builtins.getFlake "github:edolstra/dwarffs").rev ``` - - This function is only available if you enable the experimental feature - `flakes`. )", .fun = prim_getFlake, .experimentalFeature = Xp::Flakes, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 9dd7bae02..5dfad470a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -238,7 +238,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v } } -static RegisterPrimOp primop_scopedImport(RegisterPrimOp::Info { +static RegisterPrimOp primop_scopedImport(PrimOp { .name = "scopedImport", .arity = 2, .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) @@ -692,7 +692,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value * * a v.listElems()[n++] = i; } -static RegisterPrimOp primop_genericClosure(RegisterPrimOp::Info { +static RegisterPrimOp primop_genericClosure(PrimOp { .name = "__genericClosure", .args = {"attrset"}, .arity = 1, @@ -809,7 +809,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * * } } -static RegisterPrimOp primop_addErrorContext(RegisterPrimOp::Info { +static RegisterPrimOp primop_addErrorContext(PrimOp { .name = "__addErrorContext", .arity = 2, .fun = prim_addErrorContext, @@ -1400,7 +1400,7 @@ drvName, Bindings * attrs, Value & v) v.mkAttrs(result); } -static RegisterPrimOp primop_derivationStrict(RegisterPrimOp::Info { +static RegisterPrimOp primop_derivationStrict(PrimOp { .name = "derivationStrict", .arity = 1, .fun = prim_derivationStrict, @@ -1667,9 +1667,52 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V v.mkPath(state.checkSourcePath(state.findFile(searchPath, path, pos))); } -static RegisterPrimOp primop_findFile(RegisterPrimOp::Info { +static RegisterPrimOp primop_findFile(PrimOp { .name = "__findFile", - .arity = 2, + .args = {"search path", "lookup path"}, + .doc = R"( + Look up the given path with the given search path. + + A search path is represented list of [attribute sets](./values.md#attribute-set) with two attributes, `prefix`, and `path`. + `prefix` is a relative path. + `path` denotes a file system location; the exact syntax depends on the command line interface. + + Examples of search path attribute sets: + + - ``` + { + prefix = "nixos-config"; + path = "/etc/nixos/configuration.nix"; + } + ``` + + - ``` + { + prefix = ""; + path = "/nix/var/nix/profiles/per-user/root/channels"; + } + ``` + + The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/values.html#type-path) of the match. + + This is the process for each entry: + If the lookup path matches `prefix`, then the remainder of the lookup path (the "suffix") is searched for within the directory denoted by `patch`. + Note that the `path` may need to be downloaded at this point to look inside. + If the suffix is found inside that directory, then the entry is a match; + the combined absolute path of the directory (now downloaded if need be) and the suffix is returned. + + The syntax + + ```nix + + ``` + + is equivalent to: + + ```nix + builtins.findFile builtins.nixPath "nixpkgs" + ``` + )", .fun = prim_findFile, }); @@ -2388,7 +2431,7 @@ static void prim_unsafeGetAttrPos(EvalState & state, const PosIdx pos, Value * * state.mkPos(v, i->pos); } -static RegisterPrimOp primop_unsafeGetAttrPos(RegisterPrimOp::Info { +static RegisterPrimOp primop_unsafeGetAttrPos(PrimOp { .name = "__unsafeGetAttrPos", .arity = 2, .fun = prim_unsafeGetAttrPos, @@ -4061,10 +4104,10 @@ static RegisterPrimOp primop_splitVersion({ RegisterPrimOp::PrimOps * RegisterPrimOp::primOps; -RegisterPrimOp::RegisterPrimOp(Info && info) +RegisterPrimOp::RegisterPrimOp(PrimOp && primOp) { if (!primOps) primOps = new PrimOps; - primOps->push_back(std::move(info)); + primOps->push_back(std::move(primOp)); } @@ -4077,54 +4120,202 @@ void EvalState::createBaseEnv() /* `builtins' must be first! */ v.mkAttrs(buildBindings(128).finish()); - addConstant("builtins", v); + addConstant("builtins", v, { + .type = nAttrs, + .doc = R"( + Contains all the [built-in functions](@docroot@/language/builtins.md) and values. + + Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations: + + ```nix + # if hasContext is not available, we assume `s` has a context + if builtins ? hasContext then builtins.hasContext s else true + ``` + )", + }); v.mkBool(true); - addConstant("true", v); + addConstant("true", v, { + .type = nBool, + .doc = R"( + Primitive value. + + It can be returned by + [comparison operators](@docroot@/language/operators.md#Comparison) + and used in + [conditional expressions](@docroot@/language/constructs.md#Conditionals). + + The name `true` is not special, and can be shadowed: + + ```nix-repl + nix-repl> let true = 1; in true + 1 + ``` + )", + }); v.mkBool(false); - addConstant("false", v); + addConstant("false", v, { + .type = nBool, + .doc = R"( + Primitive value. + + It can be returned by + [comparison operators](@docroot@/language/operators.md#Comparison) + and used in + [conditional expressions](@docroot@/language/constructs.md#Conditionals). + + The name `false` is not special, and can be shadowed: + + ```nix-repl + nix-repl> let false = 1; in false + 1 + ``` + )", + }); v.mkNull(); - addConstant("null", v); + addConstant("null", v, { + .type = nNull, + .doc = R"( + Primitive value. + + The name `null` is not special, and can be shadowed: + + ```nix-repl + nix-repl> let null = 1; in null + 1 + ``` + )", + }); if (!evalSettings.pureEval) { v.mkInt(time(0)); - addConstant("__currentTime", v); - - v.mkString(settings.thisSystem.get()); - addConstant("__currentSystem", v); } + addConstant("__currentTime", v, { + .type = nInt, + .doc = R"( + Return the [Unix time](https://en.wikipedia.org/wiki/Unix_time) at first evaluation. + Repeated references to that name will re-use the initially obtained value. + + Example: + + ```console + $ nix repl + Welcome to Nix 2.15.1 Type :? for help. + + nix-repl> builtins.currentTime + 1683705525 + + nix-repl> builtins.currentTime + 1683705525 + ``` + + The [store path](@docroot@/glossary.md#gloss-store-path) of a derivation depending on `currentTime` will differ for each evaluation, unless both evaluate `builtins.currentTime` in the same second. + )", + .impureOnly = true, + }); + + if (!evalSettings.pureEval) { + v.mkString(settings.thisSystem.get()); + } + addConstant("__currentSystem", v, { + .type = nString, + .doc = R"( + The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-pure-eval). + + It can be used to set the `system` attribute for [`builtins.derivation`](@docroot@/language/derivations.md) such that the resulting derivation can be built on the same system that evaluates the Nix expression: + + ```nix + builtins.derivation { + # ... + system = builtins.currentSystem; + } + ``` + + It can be overridden in order to create derivations for different system than the current one: + + ```console + $ nix-instantiate --system "mips64-linux" --eval --expr 'builtins.currentSystem' + "mips64-linux" + ``` + )", + .impureOnly = true, + }); v.mkString(nixVersion); - addConstant("__nixVersion", v); + addConstant("__nixVersion", v, { + .type = nString, + .doc = R"( + The version of Nix. + + For example, where the command line returns the current Nix version, + + ```shell-session + $ nix --version + nix (Nix) 2.16.0 + ``` + + the Nix language evaluator returns the same value: + + ```nix-repl + nix-repl> builtins.nixVersion + "2.16.0" + ``` + )", + }); v.mkString(store->storeDir); - addConstant("__storeDir", v); + addConstant("__storeDir", v, { + .type = nString, + .doc = R"( + Logical file system location of the [Nix store](@docroot@/glossary.md#gloss-store) currently in use. + + This value is determined by the `store` parameter in [Store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md): + + ```shell-session + $ nix-instantiate --store 'dummy://?store=/blah' --eval --expr builtins.storeDir + "/blah" + ``` + )", + }); /* Language version. This should be increased every time a new language feature gets added. It's not necessary to increase it when primops get added, because you can just use `builtins ? primOp' to check. */ v.mkInt(6); - addConstant("__langVersion", v); + addConstant("__langVersion", v, { + .type = nInt, + .doc = R"( + The current version of the Nix language. + )", + }); // Miscellaneous if (evalSettings.enableNativeCode) { - addPrimOp("__importNative", 2, prim_importNative); - addPrimOp("__exec", 1, prim_exec); + addPrimOp({ + .name = "__importNative", + .arity = 2, + .fun = prim_importNative, + }); + addPrimOp({ + .name = "__exec", + .arity = 1, + .fun = prim_exec, + }); } addPrimOp({ - .fun = evalSettings.traceVerbose ? prim_trace : prim_second, - .arity = 2, .name = "__traceVerbose", .args = { "e1", "e2" }, + .arity = 2, .doc = R"( Evaluate *e1* and print its abstract syntax representation on standard error if `--trace-verbose` is enabled. Then return *e2*. This function is useful for debugging. )", + .fun = evalSettings.traceVerbose ? prim_trace : prim_second, }); /* Add a value containing the current Nix expression search path. */ @@ -4136,26 +4327,46 @@ void EvalState::createBaseEnv() attrs.alloc("prefix").mkString(i.prefix); (v.listElems()[n++] = allocValue())->mkAttrs(attrs); } - addConstant("__nixPath", v); + addConstant("__nixPath", v, { + .type = nList, + .doc = R"( + The search path used to resolve angle bracket path lookups. + + Angle bracket expressions can be + [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) + using this and + [`builtins.findFile`](./builtins.html#builtins-findFile): + + ```nix + + ``` + + is equivalent to: + + ```nix + builtins.findFile builtins.nixPath "nixpkgs" + ``` + )", + }); if (RegisterPrimOp::primOps) for (auto & primOp : *RegisterPrimOp::primOps) - if (!primOp.experimentalFeature - || experimentalFeatureSettings.isEnabled(*primOp.experimentalFeature)) + if (experimentalFeatureSettings.isEnabled(primOp.experimentalFeature)) { - addPrimOp({ - .fun = primOp.fun, - .arity = std::max(primOp.args.size(), primOp.arity), - .name = primOp.name, - .args = primOp.args, - .doc = primOp.doc, - }); + auto primOpAdjusted = primOp; + primOpAdjusted.arity = std::max(primOp.args.size(), primOp.arity); + addPrimOp(std::move(primOpAdjusted)); } /* Add a wrapper around the derivation primop that computes the - `drvPath' and `outPath' attributes lazily. */ + `drvPath' and `outPath' attributes lazily. + + Null docs because it is documented separately. + */ auto vDerivation = allocValue(); - addConstant("derivation", vDerivation); + addConstant("derivation", vDerivation, { + .type = nFunction, + }); /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 73b7b866c..930e7f32a 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -10,17 +10,7 @@ namespace nix { struct RegisterPrimOp { - struct Info - { - std::string name; - std::vector args; - size_t arity = 0; - const char * doc; - PrimOpFun fun; - std::optional experimentalFeature; - }; - - typedef std::vector PrimOps; + typedef std::vector PrimOps; static PrimOps * primOps; /** @@ -28,7 +18,7 @@ struct RegisterPrimOp * will get called during EvalState initialization, so there * may be primops not yet added and builtins is not yet sorted. */ - RegisterPrimOp(Info && info); + RegisterPrimOp(PrimOp && primOp); }; /* These primops are disabled without enableNativeCode, but plugins diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 4cf1f1e0b..bae849f61 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -154,9 +154,6 @@ static RegisterPrimOp primop_fetchClosure({ specifying a binary cache from which the path can be fetched. Also, requiring a content-addressed final store path avoids the need for users to configure binary cache public keys. - - This function is only available if you enable the experimental - feature `fetch-closure`. )", .fun = prim_fetchClosure, .experimentalFeature = Xp::FetchClosure, diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 89c0c36fd..44c8071c2 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -218,8 +218,11 @@ public: /** * Returns the normal type of a Value. This only returns nThunk if * the Value hasn't been forceValue'd + * + * @param invalidIsThunk Instead of aborting an an invalid (probably + * 0, so uninitialized) internal type, return `nThunk`. */ - inline ValueType type() const + inline ValueType type(bool invalidIsThunk = false) const { switch (internalType) { case tInt: return nInt; @@ -234,7 +237,10 @@ public: case tFloat: return nFloat; case tThunk: case tApp: case tBlackhole: return nThunk; } - abort(); + if (invalidIsThunk) + return nThunk; + else + abort(); } /** diff --git a/src/nix/main.cc b/src/nix/main.cc index ce0bed2a3..650c79d14 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -352,7 +352,7 @@ void mainWrapped(int argc, char * * argv) return; } - if (argc == 2 && std::string(argv[1]) == "__dump-builtins") { + if (argc == 2 && std::string(argv[1]) == "__dump-language") { experimentalFeatureSettings.experimentalFeatures = { Xp::Flakes, Xp::FetchClosure, @@ -360,17 +360,34 @@ void mainWrapped(int argc, char * * argv) evalSettings.pureEval = false; EvalState state({}, openStore("dummy://")); auto res = nlohmann::json::object(); - auto builtins = state.baseEnv.values[0]->attrs; - for (auto & builtin : *builtins) { - auto b = nlohmann::json::object(); - if (!builtin.value->isPrimOp()) continue; - auto primOp = builtin.value->primOp; - if (!primOp->doc) continue; - b["arity"] = primOp->arity; - b["args"] = primOp->args; - b["doc"] = trim(stripIndentation(primOp->doc)); - res[state.symbols[builtin.name]] = std::move(b); - } + res["builtins"] = ({ + auto builtinsJson = nlohmann::json::object(); + auto builtins = state.baseEnv.values[0]->attrs; + for (auto & builtin : *builtins) { + auto b = nlohmann::json::object(); + if (!builtin.value->isPrimOp()) continue; + auto primOp = builtin.value->primOp; + if (!primOp->doc) continue; + b["arity"] = primOp->arity; + b["args"] = primOp->args; + b["doc"] = trim(stripIndentation(primOp->doc)); + b["experimental-feature"] = primOp->experimentalFeature; + builtinsJson[state.symbols[builtin.name]] = std::move(b); + } + std::move(builtinsJson); + }); + res["constants"] = ({ + auto constantsJson = nlohmann::json::object(); + for (auto & [name, info] : state.constantInfos) { + auto c = nlohmann::json::object(); + if (!info.doc) continue; + c["doc"] = trim(stripIndentation(info.doc)); + c["type"] = showType(info.type, false); + c["impure-only"] = info.impureOnly; + constantsJson[name] = std::move(c); + } + std::move(constantsJson); + }); logger->cout("%s", res); return; } From 80c9259756811c1165167db1bb66c1fef0accb65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 27 Jun 2023 12:01:08 +0200 Subject: [PATCH 058/909] Allow to sign path as unprivileged user User can now sign path as unprivileged/allowed user refs #1708 --- doc/manual/src/release-notes/rl-next.md | 3 +++ src/libstore/daemon.cc | 2 -- tests/nixos/authorization.nix | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index bde9057c6..8479b166a 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,3 +1,6 @@ # Release X.Y (202?-??-??) - [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand + +- Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths. + Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths. diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 75c3d2aca..ad3dee1a2 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -864,8 +864,6 @@ static void performOp(TunnelLogger * logger, ref store, auto path = store->parseStorePath(readString(from)); StringSet sigs = readStrings(from); logger->startWork(); - if (!trusted) - throw Error("you are not privileged to add signatures"); store->addSignatures(path, sigs); logger->stopWork(); to << 1; diff --git a/tests/nixos/authorization.nix b/tests/nixos/authorization.nix index 7e8744dd9..fdeae06ed 100644 --- a/tests/nixos/authorization.nix +++ b/tests/nixos/authorization.nix @@ -75,5 +75,20 @@ su --login bob -c '(! nix-store --verify --repair 2>&1)' | tee diag 1>&2 grep -F "you are not privileged to repair paths" diag """) + + machine.succeed(""" + set -x + su --login mallory -c ' + nix-store --generate-binary-cache-key cache1.example.org sk1 pk1 + (! nix store sign --key-file sk1 ${pathFour} 2>&1)' | tee diag 1>&2 + grep -F "cannot open connection to remote store 'daemon'" diag + """) + + machine.succeed(""" + su --login bob -c ' + nix-store --generate-binary-cache-key cache1.example.org sk1 pk1 + nix store sign --key-file sk1 ${pathFour} + ' + """) ''; } From 2ccc02515f4a52d59f3473493f254942209fc81b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 27 Jun 2023 18:22:42 -0400 Subject: [PATCH 059/909] Trailing commas in redirects This avoids diff noise when more are added. Unlike with JSON, this is allowed in JS. --- doc/manual/redirects.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index d75cd231a..f54f6ea6a 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -330,23 +330,23 @@ const redirects = { "ssec-relnotes-2.0": "release-notes/rl-2.0.html", "ssec-relnotes-2.1": "release-notes/rl-2.1.html", "ssec-relnotes-2.2": "release-notes/rl-2.2.html", - "ssec-relnotes-2.3": "release-notes/rl-2.3.html" + "ssec-relnotes-2.3": "release-notes/rl-2.3.html", }, "language/values.html": { "simple-values": "#primitives", "lists": "#list", "strings": "#string", "lists": "#list", - "attribute-sets": "#attribute-set" + "attribute-sets": "#attribute-set", }, "installation/installing-binary.html": { "linux": "uninstall.html#linux", "macos": "uninstall.html#macos", - "uninstalling": "uninstall.html" + "uninstalling": "uninstall.html", } "contributing/hacking.html": { - "nix-with-flakes": "#building-nix-with-flakes" - "classic-nix": "#building-nix" + "nix-with-flakes": "#building-nix-with-flakes", + "classic-nix": "#building-nix", } }; From ca49e13414b4d1d9b2cdd72e36920e28c4ef117e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 21 Jun 2023 23:33:33 -0400 Subject: [PATCH 060/909] Split testing into its own page in the contribution guide `hacking.md` has gotten really big! --- doc/manual/redirects.js | 8 ++ doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/contributing/hacking.md | 165 ------------------------ doc/manual/src/contributing/testing.md | 167 +++++++++++++++++++++++++ 4 files changed, 176 insertions(+), 165 deletions(-) create mode 100644 doc/manual/src/contributing/testing.md diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index f54f6ea6a..dcdb5d6e9 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -347,6 +347,14 @@ const redirects = { "contributing/hacking.html": { "nix-with-flakes": "#building-nix-with-flakes", "classic-nix": "#building-nix", + "running-tests": "testing.html#running-tests", + "unit-tests": "testing.html#unit-tests", + "functional-tests": "testing.html#functional-tests", + "debugging-failing-functional-tests": "testing.html#debugging-failing-functional-tests", + "integration-tests": "testing.html#integration-tests", + "installer-tests": "testing.html#installer-tests", + "one-time-setup": "testing.html#one-time-setup", + "using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing", } }; diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index bbb603713..1bd8fa774 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -104,6 +104,7 @@ - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) + - [Testing](contributing/testing.md) - [Experimental Features](contributing/experimental-features.md) - [CLI guideline](contributing/cli-guideline.md) - [C++ style guide](contributing/cxx.md) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 6c6f2eb52..7b2440971 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -190,171 +190,6 @@ Configure your editor to use the `clangd` from the shell, either by running it i > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). > Editor-specific setup is typically opinionated, so we will not cover it here in more detail. -## Running tests - -### Unit-tests - -The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined -under `src/{library_name}/tests` using the -[googletest](https://google.github.io/googletest/) and -[rapidcheck](https://github.com/emil-e/rapidcheck) frameworks. - -You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option. - -### Functional tests - -The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. -Each test is a bash script. - -The whole test suite can be run with: - -```shell-session -$ make install && make installcheck -ran test tests/foo.sh... [PASS] -ran test tests/bar.sh... [PASS] -... -``` - -Individual tests can be run with `make`: - -```shell-session -$ make tests/${testName}.sh.test -ran test tests/${testName}.sh... [PASS] -``` - -or without `make`: - -```shell-session -$ ./mk/run-test.sh tests/${testName}.sh -ran test tests/${testName}.sh... [PASS] -``` - -To see the complete output, one can also run: - -```shell-session -$ ./mk/debug-test.sh tests/${testName}.sh -+ foo -output from foo -+ bar -output from bar -... -``` - -The test script will then be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails. - -#### Debugging failing functional tests - -When a functional test fails, it usually does so somewhere in the middle of the script. - -To figure out what's wrong, it is convenient to run the test regularly up to the failing `nix` command, and then run that command with a debugger like GDB. - -For example, if the script looks like: - -```bash -foo -nix blah blub -bar -``` -edit it like so: - -```diff - foo --nix blah blub -+gdb --args nix blah blub - bar -``` - -Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: - -```shell-session -$ ./mk/debug-test.sh tests/${testName}.sh -... -+ gdb blash blub -GNU gdb (GDB) 12.1 -... -(gdb) -``` - -One can debug the Nix invocation in all the usual ways. -For example, enter `run` to start the Nix invocation. - -### Integration tests - -The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. -These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. -Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). - -You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` - -### Installer tests - -After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch. - -Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): - -- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache: - - `x86_64-linux` - - `armv6l-linux` - - `armv7l-linux` - - `x86_64-darwin` - -- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. - -#### One-time setup - -1. Have a GitHub account with a fork of the [Nix repository](https://github.com/NixOS/nix). -2. At cachix.org: - - Create or log in to an account. - - Create a Cachix cache using the format `-nix-install-tests`. - - Navigate to the new cache > Settings > Auth Tokens. - - Generate a new Cachix auth token and copy the generated value. -3. At github.com: - - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret. - - Name the secret `CACHIX_AUTH_TOKEN`. - - Paste the copied value of the Cachix cache auth token. - -#### Using the CI-generated installer for manual testing - -After the CI run completes, you can check the output to extract the installer URL: -1. Click into the detailed view of the CI run. -2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them). -3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`) -4. Copy the value of `install_url` -5. To generate an install command, plug this `install_url` and your GitHub username into this template: - - ```console - curl -L | sh -s -- --tarball-url-prefix https://-nix-install-tests.cachix.org/serve - ``` - - - ### Checking links in the manual The build checks for broken internal links. diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md new file mode 100644 index 000000000..e5f80a928 --- /dev/null +++ b/doc/manual/src/contributing/testing.md @@ -0,0 +1,167 @@ +# Running tests + +## Unit-tests + +The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined +under `src/{library_name}/tests` using the +[googletest](https://google.github.io/googletest/) and +[rapidcheck](https://github.com/emil-e/rapidcheck) frameworks. + +You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option. + +## Functional tests + +The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. +Each test is a bash script. + +The whole test suite can be run with: + +```shell-session +$ make install && make installcheck +ran test tests/foo.sh... [PASS] +ran test tests/bar.sh... [PASS] +... +``` + +Individual tests can be run with `make`: + +```shell-session +$ make tests/${testName}.sh.test +ran test tests/${testName}.sh... [PASS] +``` + +or without `make`: + +```shell-session +$ ./mk/run-test.sh tests/${testName}.sh +ran test tests/${testName}.sh... [PASS] +``` + +To see the complete output, one can also run: + +```shell-session +$ ./mk/debug-test.sh tests/${testName}.sh ++ foo +output from foo ++ bar +output from bar +... +``` + +The test script will then be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails. + +### Debugging failing functional tests + +When a functional test fails, it usually does so somewhere in the middle of the script. + +To figure out what's wrong, it is convenient to run the test regularly up to the failing `nix` command, and then run that command with a debugger like GDB. + +For example, if the script looks like: + +```bash +foo +nix blah blub +bar +``` +edit it like so: + +```diff + foo +-nix blah blub ++gdb --args nix blah blub + bar +``` + +Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: + +```shell-session +$ ./mk/debug-test.sh tests/${testName}.sh +... ++ gdb blash blub +GNU gdb (GDB) 12.1 +... +(gdb) +``` + +One can debug the Nix invocation in all the usual ways. +For example, enter `run` to start the Nix invocation. + +## Integration tests + +The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. +These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. +Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). + +You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` + +## Installer tests + +After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch. + +Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): + +- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache: + - `x86_64-linux` + - `armv6l-linux` + - `armv7l-linux` + - `x86_64-darwin` + +- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. + +### One-time setup + +1. Have a GitHub account with a fork of the [Nix repository](https://github.com/NixOS/nix). +2. At cachix.org: + - Create or log in to an account. + - Create a Cachix cache using the format `-nix-install-tests`. + - Navigate to the new cache > Settings > Auth Tokens. + - Generate a new Cachix auth token and copy the generated value. +3. At github.com: + - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret. + - Name the secret `CACHIX_AUTH_TOKEN`. + - Paste the copied value of the Cachix cache auth token. + +## Working on documentation + +### Using the CI-generated installer for manual testing + +After the CI run completes, you can check the output to extract the installer URL: +1. Click into the detailed view of the CI run. +2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them). +3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`) +4. Copy the value of `install_url` +5. To generate an install command, plug this `install_url` and your GitHub username into this template: + + ```console + curl -L | sh -s -- --tarball-url-prefix https://-nix-install-tests.cachix.org/serve + ``` + + + From 3468cbaf479e3a2e9b3c7d6b00b2cb1976cce43e Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Wed, 28 Jun 2023 22:38:44 +0800 Subject: [PATCH 061/909] libexpr: fix leaking `es2` in stripIndentation (parser.y) --- src/libexpr/parser.y | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 5c9597f8a..ed6421ced 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -275,7 +275,12 @@ static Expr * stripIndentation(const PosIdx pos, SymbolTable & symbols, } /* If this is a single string, then don't do a concatenation. */ - return es2->size() == 1 && dynamic_cast((*es2)[0].second) ? (*es2)[0].second : new ExprConcatStrings(pos, true, es2); + if (es2->size() == 1 && dynamic_cast((*es2)[0].second)) { + auto *const result = (*es2)[0].second; + delete es2; + return result; + } + return new ExprConcatStrings(pos, true, es2); } From 685f1bb3864d52b80e74f920a150f8ffc7a479c3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 30 Jun 2023 15:10:07 +0200 Subject: [PATCH 062/909] labeler.yml: tests -> with-tests --- .github/labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index fce0d3aeb..12120bdb3 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -16,7 +16,7 @@ "new-cli": - src/nix/**/* -"tests": +"with-tests": # Unit tests - src/*/tests/**/* # Functional and integration tests From eebfe989a5dc3aac622b9b5f2edef4461d8968c1 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 30 Jun 2023 21:11:59 +0800 Subject: [PATCH 063/909] linkOrCopy: Fallback upon cross-device link error (EXDEV) Fix building derivations in local chroot store on OpenAFS, where hard linking accross directories causes cross-device link error (EXDEV). --- src/libstore/build/local-derivation-goal.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index ea7c52098..ee66ee500 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -395,8 +395,9 @@ static void linkOrCopy(const Path & from, const Path & to) bind-mount in this case? It can also fail with EPERM in BeegFS v7 and earlier versions + or fail with EXDEV in OpenAFS which don't allow hard-links to other directories */ - if (errno != EMLINK && errno != EPERM) + if (errno != EMLINK && errno != EPERM && errno != EXDEV) throw SysError("linking '%s' to '%s'", to, from); copyPath(from, to); } From 0f6d596df5866d3f08e743ac0f1a282daf6b2155 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 19 May 2023 09:49:18 +0200 Subject: [PATCH 064/909] fetchClosure: Factor out attribute hint --- src/libexpr/primops/fetchClosure.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index bae849f61..26c2b90bf 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -16,11 +16,13 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg for (auto & attr : *args[0]->attrs) { const auto & attrName = state.symbols[attr.name]; + auto attrHint = [&]() -> std::string { + return "while evaluating the '" + attrName + "' attribute passed to builtins.fetchClosure"; + }; if (attrName == "fromPath") { NixStringContext context; - fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, - "while evaluating the 'fromPath' attribute passed to builtins.fetchClosure"); + fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint()); } else if (attrName == "toPath") { @@ -28,14 +30,13 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg toCA = true; if (attr.value->type() != nString || attr.value->string.s != std::string("")) { NixStringContext context; - toPath = state.coerceToStorePath(attr.pos, *attr.value, context, - "while evaluating the 'toPath' attribute passed to builtins.fetchClosure"); + toPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint()); } } else if (attrName == "fromStore") fromStoreUrl = state.forceStringNoCtx(*attr.value, attr.pos, - "while evaluating the 'fromStore' attribute passed to builtins.fetchClosure"); + attrHint()); else throw Error({ From 7e5b6d2c4550a3007946271768e114531d7b89f0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 19 May 2023 10:03:25 +0200 Subject: [PATCH 065/909] fetchClosure: Refactor: rename toCA -> enableRewriting --- src/libexpr/primops/fetchClosure.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 26c2b90bf..8f8e033a5 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -11,7 +11,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg std::optional fromStoreUrl; std::optional fromPath; - bool toCA = false; + bool enableRewriting = false; std::optional toPath; for (auto & attr : *args[0]->attrs) { @@ -27,7 +27,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg else if (attrName == "toPath") { state.forceValue(*attr.value, attr.pos); - toCA = true; + enableRewriting = true; if (attr.value->type() != nString || attr.value->string.s != std::string("")) { NixStringContext context; toPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint()); @@ -75,7 +75,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg auto fromStore = openStore(parsedURL.to_string()); - if (toCA) { + if (enableRewriting) { if (!toPath || !state.store->isValidPath(*toPath)) { auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath }); auto i = remappings.find(*fromPath); From ea30f152b738fe65f216f3173d3a19adaaef5323 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 19 May 2023 10:48:53 +0200 Subject: [PATCH 066/909] fetchClosure: Allow input addressed paths in pure mode When explicitly requested by the caller, as suggested in the meeting (https://github.com/NixOS/nix/pull/8090#issuecomment-1531139324) > @edolstra: { toPath } vs { fromPath } is too implicit I've opted for the `inputAddressed = true` requirement, because it we did not agree on renaming the path attributes. > @roberth: more explicit > @edolstra: except for the direction; not immediately clear in which direction the rewriting happens This is in fact the most explicit syntax and a bit redundant, which is good, because that redundancy lets us deliver an error message that reminds expression authors that CA provides a better experience to their users. --- src/libexpr/primops/fetchClosure.cc | 54 +++++++++++++++++++++++------ tests/fetchClosure.sh | 16 +++++++-- 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 8f8e033a5..96c1df544 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -13,6 +13,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg std::optional fromPath; bool enableRewriting = false; std::optional toPath; + bool inputAddressed = false; for (auto & attr : *args[0]->attrs) { const auto & attrName = state.symbols[attr.name]; @@ -38,6 +39,9 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg fromStoreUrl = state.forceStringNoCtx(*attr.value, attr.pos, attrHint()); + else if (attrName == "inputAddressed") + inputAddressed = state.forceBool(*attr.value, attr.pos, attrHint()); + else throw Error({ .msg = hintfmt("attribute '%s' isn't supported in call to 'fetchClosure'", attrName), @@ -51,6 +55,18 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg .errPos = state.positions[pos] }); + if (inputAddressed) { + if (toPath && toPath != fromPath) + throw Error({ + .msg = hintfmt("attribute '%s' is set to true, but 'toPath' does not match 'fromPath'. 'toPath' should be equal, or should be omitted. Instead 'toPath' was '%s' and 'fromPath' was '%s'", + "inputAddressed", + state.store->printStorePath(*toPath), + state.store->printStorePath(*fromPath)), + .errPos = state.positions[pos] + }); + assert(!enableRewriting); + } + if (!fromStoreUrl) throw Error({ .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"), @@ -104,15 +120,28 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg toPath = fromPath; } - /* In pure mode, require a CA path. */ - if (evalSettings.pureEval) { + /* We want input addressing to be explicit, to inform readers and to give + expression authors an opportunity to improve their user experience. */ + if (!inputAddressed) { auto info = state.store->queryPathInfo(*toPath); - if (!info->isContentAddressed(*state.store)) - throw Error({ - .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't", - state.store->printStorePath(*toPath)), - .errPos = state.positions[pos] - }); + if (!info->isContentAddressed(*state.store)) { + if (enableRewriting) { + throw Error({ + // Ideally we'd compute the path for them, but this error message is unlikely to occur in practice, so we keep it simple. + .msg = hintfmt("Rewriting was requested, but 'toPath' is not content addressed. This is impossible. Please change 'toPath' to the correct path, or to a non-existing path, and try again", + state.store->printStorePath(*toPath)), + .errPos = state.positions[pos] + }); + } else { + // We just checked toPath, but we report fromPath, because that's what the user certainly passed. + assert (toPath == fromPath); + throw Error({ + .msg = hintfmt("The 'fromPath' value '%s' is input addressed, but input addressing was not requested. If you do intend to return an input addressed store path, add 'inputAddressed = true;' to the 'fetchClosure' arguments. Note that content addressing does not require users to configure a trusted binary cache public key on their systems, and is therefore preferred.", + state.store->printStorePath(*fromPath)), + .errPos = state.positions[pos] + }); + } + } } state.mkStorePathString(*toPath, v); @@ -138,7 +167,7 @@ static RegisterPrimOp primop_fetchClosure({ `/nix/store/ldbh...`. If `fromPath` is already content-addressed, or if you are - allowing impure evaluation (`--impure`), then `toPath` may be + allowing input addressing (`inputAddressed = true;`), then `toPath` may be omitted. To find out the correct value for `toPath` given a `fromPath`, @@ -153,8 +182,11 @@ static RegisterPrimOp primop_fetchClosure({ allows you to use a previously built store path in a Nix expression. However, it is more reproducible because it requires specifying a binary cache from which the path can be fetched. - Also, requiring a content-addressed final store path avoids the - need for users to configure binary cache public keys. + Also, the default requirement of a content-addressed final store path + avoids the need for users to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys). + + This function is only available if you enable the experimental + feature `fetch-closure`. )", .fun = prim_fetchClosure, .experimentalFeature = Xp::FetchClosure, diff --git a/tests/fetchClosure.sh b/tests/fetchClosure.sh index 21650eb06..4b8198e94 100644 --- a/tests/fetchClosure.sh +++ b/tests/fetchClosure.sh @@ -35,16 +35,28 @@ clearStore if [[ "$NIX_REMOTE" != "daemon" ]]; then - # In impure mode, we can use non-CA paths. - [[ $(nix eval --raw --no-require-sigs --impure --expr " + # We can use non-CA paths when we ask explicitly. + [[ $(nix eval --raw --no-require-sigs --expr " builtins.fetchClosure { fromStore = \"file://$cacheDir\"; fromPath = $nonCaPath; + inputAddressed = true; } ") = $nonCaPath ]] [ -e $nonCaPath ] + # .. but only if we ask explicitly. + expectStderr 1 nix eval --raw --no-require-sigs --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + } + " | grepQuiet -E "The .fromPath. value .* is input addressed, but input addressing was not requested. If you do intend to return an input addressed store path, add .inputAddressed = true;. to the .fetchClosure. arguments." + + [ -e $nonCaPath ] + + fi # 'toPath' set to empty string should fail but print the expected path. From 508aa58e671583bf06fa3597629839827c8d1bd7 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 19 May 2023 16:25:23 +0200 Subject: [PATCH 067/909] fetchClosure: Always check that inputAddressed matches the result --- src/libexpr/primops/fetchClosure.cc | 42 +++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 96c1df544..c66fed633 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -13,7 +13,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg std::optional fromPath; bool enableRewriting = false; std::optional toPath; - bool inputAddressed = false; + std::optional inputAddressedMaybe; for (auto & attr : *args[0]->attrs) { const auto & attrName = state.symbols[attr.name]; @@ -40,7 +40,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg attrHint()); else if (attrName == "inputAddressed") - inputAddressed = state.forceBool(*attr.value, attr.pos, attrHint()); + inputAddressedMaybe = state.forceBool(*attr.value, attr.pos, attrHint()); else throw Error({ @@ -55,6 +55,8 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg .errPos = state.positions[pos] }); + bool inputAddressed = inputAddressedMaybe.value_or(false); + if (inputAddressed) { if (toPath && toPath != fromPath) throw Error({ @@ -120,15 +122,39 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg toPath = fromPath; } - /* We want input addressing to be explicit, to inform readers and to give - expression authors an opportunity to improve their user experience. */ - if (!inputAddressed) { - auto info = state.store->queryPathInfo(*toPath); + auto info = state.store->queryPathInfo(*toPath); + + /* If inputAddressed is set, make sure the result is as expected. */ + if (inputAddressedMaybe) { + bool expectCA = !inputAddressed; + if (info->isContentAddressed(*state.store) != expectCA) { + if (inputAddressed) + throw Error({ + .msg = hintfmt("The 'fetchClosure' result, '%s' is not input addressed, despite 'inputAddressed' being set to true. It is ok or even preferable to return a content addressed path, so you probably want to remove the 'inputAddressed' attribute", + state.store->printStorePath(*toPath)), + .errPos = state.positions[pos] + }); + else + throw Error({ + .msg = hintfmt("The 'fetchClosure' result, '%s' is input addressed, despite 'inputAddressed' being set to false. Please make sure you passed the correct path(s) or set 'inputAddressed' to true if you intended to return an input addressed path", + state.store->printStorePath(*toPath)), + .errPos = state.positions[pos] + }); + } + } else { + /* + While it's fine to omit the inputAddressed attribute for CA paths, + we want input addressing to be explicit, to inform readers and to give + expression authors an opportunity to improve their user experience; + see message below. + */ if (!info->isContentAddressed(*state.store)) { if (enableRewriting) { + // We don't perform the rewriting when outPath already exists, as an optimisation. + // However, we can quickly detect a mistake if the toPath is input addressed. + // Ideally we'd compute the path for them here, but this error message is unlikely to occur in practice, so we keep it simple. throw Error({ - // Ideally we'd compute the path for them, but this error message is unlikely to occur in practice, so we keep it simple. - .msg = hintfmt("Rewriting was requested, but 'toPath' is not content addressed. This is impossible. Please change 'toPath' to the correct path, or to a non-existing path, and try again", + .msg = hintfmt("The 'toPath' value '%s' is input addressed, so it can't possibly be the result of rewriting. You may set 'toPath' to an empty string to figure out the correct path.", state.store->printStorePath(*toPath)), .errPos = state.positions[pos] }); From 8dca95386c9ab8b13886716ec31e1b2936a3ef10 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 5 Jun 2023 11:11:53 +0200 Subject: [PATCH 068/909] fetchClosure: Disallow toPath for inputAddressed = true --- src/libexpr/primops/fetchClosure.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index c66fed633..6d2271853 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -58,12 +58,11 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg bool inputAddressed = inputAddressedMaybe.value_or(false); if (inputAddressed) { - if (toPath && toPath != fromPath) + if (toPath) throw Error({ - .msg = hintfmt("attribute '%s' is set to true, but 'toPath' does not match 'fromPath'. 'toPath' should be equal, or should be omitted. Instead 'toPath' was '%s' and 'fromPath' was '%s'", + .msg = hintfmt("attribute '%s' is set to true, but '%s' is also set. Please remove one of them", "inputAddressed", - state.store->printStorePath(*toPath), - state.store->printStorePath(*fromPath)), + "toPath"), .errPos = state.positions[pos] }); assert(!enableRewriting); From 55888633dd323c83cd1db3773d82dc954da34888 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 5 Jun 2023 11:37:41 +0200 Subject: [PATCH 069/909] makeContentAddressed: Add single path helper --- src/libexpr/primops/fetchClosure.cc | 10 ++++------ src/libstore/make-content-addressed.cc | 11 +++++++++++ src/libstore/make-content-addressed.hh | 13 ++++++++++++- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 6d2271853..6a15c826f 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -94,14 +94,12 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg if (enableRewriting) { if (!toPath || !state.store->isValidPath(*toPath)) { - auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath }); - auto i = remappings.find(*fromPath); - assert(i != remappings.end()); - if (toPath && *toPath != i->second) + auto rewrittenPath = makeContentAddressed(*fromStore, *state.store, *fromPath); + if (toPath && *toPath != rewrittenPath) throw Error({ .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", state.store->printStorePath(*fromPath), - state.store->printStorePath(i->second), + state.store->printStorePath(rewrittenPath), state.store->printStorePath(*toPath)), .errPos = state.positions[pos] }); @@ -111,7 +109,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg "rewriting '%s' to content-addressed form yielded '%s'; " "please set this in the 'toPath' attribute passed to 'fetchClosure'", state.store->printStorePath(*fromPath), - state.store->printStorePath(i->second)), + state.store->printStorePath(rewrittenPath)), .errPos = state.positions[pos] }); } diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 53fe04704..626a22480 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -80,4 +80,15 @@ std::map makeContentAddressed( return remappings; } +StorePath makeContentAddressed( + Store & srcStore, + Store & dstStore, + const StorePath & fromPath) +{ + auto remappings = makeContentAddressed(srcStore, dstStore, StorePathSet { fromPath }); + auto i = remappings.find(fromPath); + assert(i != remappings.end()); + return i->second; +} + } diff --git a/src/libstore/make-content-addressed.hh b/src/libstore/make-content-addressed.hh index 2ce6ec7bc..60bb2b477 100644 --- a/src/libstore/make-content-addressed.hh +++ b/src/libstore/make-content-addressed.hh @@ -5,9 +5,20 @@ namespace nix { +/** Rewrite a closure of store paths to be completely content addressed. + */ std::map makeContentAddressed( Store & srcStore, Store & dstStore, - const StorePathSet & storePaths); + const StorePathSet & rootPaths); + +/** Rewrite a closure of a store path to be completely content addressed. + * + * This is a convenience function for the case where you only have one root path. + */ +StorePath makeContentAddressed( + Store & srcStore, + Store & dstStore, + const StorePath & rootPath); } From 5bdca46117c701cd1cd47700c755367442537157 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 5 Jun 2023 12:35:03 +0200 Subject: [PATCH 070/909] fetchClosure: Split into three cases --- src/libexpr/primops/fetchClosure.cc | 177 ++++++++++++++++------------ 1 file changed, 101 insertions(+), 76 deletions(-) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 6a15c826f..e8c32de96 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -5,6 +5,101 @@ namespace nix { +/** + * Handler for the content addressed case. + * + * @param state The evaluator state and store to write to. + * @param fromStore The store containing the path to rewrite. + * @param fromPath The source path to be rewritten. + * @param toPathMaybe The path to write the rewritten path to. If empty, the error shows the actual path. + * @param v The return `Value` + */ +static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, const std::optional & toPathMaybe, Value &v) { + + // establish toPath or throw + + if (!toPathMaybe || !state.store->isValidPath(*toPathMaybe)) { + auto rewrittenPath = makeContentAddressed(fromStore, *state.store, fromPath); + if (toPathMaybe && *toPathMaybe != rewrittenPath) + throw Error({ + .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", + state.store->printStorePath(fromPath), + state.store->printStorePath(rewrittenPath), + state.store->printStorePath(*toPathMaybe)), + .errPos = state.positions[pos] + }); + if (!toPathMaybe) + throw Error({ + .msg = hintfmt( + "rewriting '%s' to content-addressed form yielded '%s'; " + "please set this in the 'toPath' attribute passed to 'fetchClosure'", + state.store->printStorePath(fromPath), + state.store->printStorePath(rewrittenPath)), + .errPos = state.positions[pos] + }); + } + + auto toPath = *toPathMaybe; + + // check and return + + auto resultInfo = state.store->queryPathInfo(toPath); + + if (!resultInfo->isContentAddressed(*state.store)) { + // We don't perform the rewriting when outPath already exists, as an optimisation. + // However, we can quickly detect a mistake if the toPath is input addressed. + throw Error({ + .msg = hintfmt("The 'toPath' value '%s' is input addressed, so it can't possibly be the result of rewriting. You may set 'toPath' to an empty string to figure out the correct path.", + state.store->printStorePath(toPath)), + .errPos = state.positions[pos] + }); + } + + state.mkStorePathString(toPath, v); +} + +/** + * Fetch the closure and make sure it's content addressed. + */ +static void runFetchClosureWithContentAddressedPath(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { + + if (!state.store->isValidPath(fromPath)) + copyClosure(fromStore, *state.store, RealisedPath::Set { fromPath }); + + auto info = state.store->queryPathInfo(fromPath); + + if (!info->isContentAddressed(*state.store)) { + throw Error({ + .msg = hintfmt("The 'fromPath' value '%s' is input addressed, but input addressing was not requested. If you do intend to return an input addressed store path, add 'inputAddressed = true;' to the 'fetchClosure' arguments. Note that content addressing does not require users to configure a trusted binary cache public key on their systems, and is therefore preferred.", + state.store->printStorePath(fromPath)), + .errPos = state.positions[pos] + }); + } + + state.mkStorePathString(fromPath, v); +} + +/** + * Fetch the closure and make sure it's input addressed. + */ +static void runFetchClosureWithInputAddressedPath(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { + + if (!state.store->isValidPath(fromPath)) + copyClosure(fromStore, *state.store, RealisedPath::Set { fromPath }); + + auto info = state.store->queryPathInfo(fromPath); + + if (info->isContentAddressed(*state.store)) { + throw Error({ + .msg = hintfmt("The 'fetchClosure' result, '%s' is not input addressed, despite 'inputAddressed' being set to true. It is preferable to return a content addressed path, so remove the 'inputAddressed' attribute to ensure content addressing is used in the future", + state.store->printStorePath(fromPath)), + .errPos = state.positions[pos] + }); + } + + state.mkStorePathString(fromPath, v); +} + static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.fetchClosure"); @@ -92,82 +187,12 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg auto fromStore = openStore(parsedURL.to_string()); - if (enableRewriting) { - if (!toPath || !state.store->isValidPath(*toPath)) { - auto rewrittenPath = makeContentAddressed(*fromStore, *state.store, *fromPath); - if (toPath && *toPath != rewrittenPath) - throw Error({ - .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", - state.store->printStorePath(*fromPath), - state.store->printStorePath(rewrittenPath), - state.store->printStorePath(*toPath)), - .errPos = state.positions[pos] - }); - if (!toPath) - throw Error({ - .msg = hintfmt( - "rewriting '%s' to content-addressed form yielded '%s'; " - "please set this in the 'toPath' attribute passed to 'fetchClosure'", - state.store->printStorePath(*fromPath), - state.store->printStorePath(rewrittenPath)), - .errPos = state.positions[pos] - }); - } - } else { - if (!state.store->isValidPath(*fromPath)) - copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath }); - toPath = fromPath; - } - - auto info = state.store->queryPathInfo(*toPath); - - /* If inputAddressed is set, make sure the result is as expected. */ - if (inputAddressedMaybe) { - bool expectCA = !inputAddressed; - if (info->isContentAddressed(*state.store) != expectCA) { - if (inputAddressed) - throw Error({ - .msg = hintfmt("The 'fetchClosure' result, '%s' is not input addressed, despite 'inputAddressed' being set to true. It is ok or even preferable to return a content addressed path, so you probably want to remove the 'inputAddressed' attribute", - state.store->printStorePath(*toPath)), - .errPos = state.positions[pos] - }); - else - throw Error({ - .msg = hintfmt("The 'fetchClosure' result, '%s' is input addressed, despite 'inputAddressed' being set to false. Please make sure you passed the correct path(s) or set 'inputAddressed' to true if you intended to return an input addressed path", - state.store->printStorePath(*toPath)), - .errPos = state.positions[pos] - }); - } - } else { - /* - While it's fine to omit the inputAddressed attribute for CA paths, - we want input addressing to be explicit, to inform readers and to give - expression authors an opportunity to improve their user experience; - see message below. - */ - if (!info->isContentAddressed(*state.store)) { - if (enableRewriting) { - // We don't perform the rewriting when outPath already exists, as an optimisation. - // However, we can quickly detect a mistake if the toPath is input addressed. - // Ideally we'd compute the path for them here, but this error message is unlikely to occur in practice, so we keep it simple. - throw Error({ - .msg = hintfmt("The 'toPath' value '%s' is input addressed, so it can't possibly be the result of rewriting. You may set 'toPath' to an empty string to figure out the correct path.", - state.store->printStorePath(*toPath)), - .errPos = state.positions[pos] - }); - } else { - // We just checked toPath, but we report fromPath, because that's what the user certainly passed. - assert (toPath == fromPath); - throw Error({ - .msg = hintfmt("The 'fromPath' value '%s' is input addressed, but input addressing was not requested. If you do intend to return an input addressed store path, add 'inputAddressed = true;' to the 'fetchClosure' arguments. Note that content addressing does not require users to configure a trusted binary cache public key on their systems, and is therefore preferred.", - state.store->printStorePath(*fromPath)), - .errPos = state.positions[pos] - }); - } - } - } - - state.mkStorePathString(*toPath, v); + if (enableRewriting) + runFetchClosureWithRewrite(state, pos, *fromStore, *fromPath, toPath, v); + else if (inputAddressed) + runFetchClosureWithInputAddressedPath(state, pos, *fromStore, *fromPath, v); + else + runFetchClosureWithContentAddressedPath(state, pos, *fromStore, *fromPath, v); } static RegisterPrimOp primop_fetchClosure({ From dc79636007ff6982300980ba7eb1c2be3e45bece Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 5 Jun 2023 12:55:58 +0200 Subject: [PATCH 071/909] fetchClosure: Refactor: replace enableRewriting A single variable is nice and self-contained. --- src/libexpr/primops/fetchClosure.cc | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index e8c32de96..e048a2629 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -100,14 +100,15 @@ static void runFetchClosureWithInputAddressedPath(EvalState & state, const PosId state.mkStorePathString(fromPath, v); } +typedef std::optional StorePathOrGap; + static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.fetchClosure"); std::optional fromStoreUrl; std::optional fromPath; - bool enableRewriting = false; - std::optional toPath; + std::optional toPath; std::optional inputAddressedMaybe; for (auto & attr : *args[0]->attrs) { @@ -123,8 +124,11 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg else if (attrName == "toPath") { state.forceValue(*attr.value, attr.pos); - enableRewriting = true; - if (attr.value->type() != nString || attr.value->string.s != std::string("")) { + bool isEmptyString = attr.value->type() == nString && attr.value->string.s == std::string(""); + if (isEmptyString) { + toPath = StorePathOrGap {}; + } + else { NixStringContext context; toPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint()); } @@ -160,7 +164,6 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg "toPath"), .errPos = state.positions[pos] }); - assert(!enableRewriting); } if (!fromStoreUrl) @@ -187,8 +190,8 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg auto fromStore = openStore(parsedURL.to_string()); - if (enableRewriting) - runFetchClosureWithRewrite(state, pos, *fromStore, *fromPath, toPath, v); + if (toPath) + runFetchClosureWithRewrite(state, pos, *fromStore, *fromPath, *toPath, v); else if (inputAddressed) runFetchClosureWithInputAddressedPath(state, pos, *fromStore, *fromPath, v); else From 32c69e2b17ad7bd9721b584c3733629fe6d8c7f7 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 5 Jun 2023 16:25:02 +0200 Subject: [PATCH 072/909] doc: Typo --- doc/manual/src/language/builtins-prefix.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/language/builtins-prefix.md b/doc/manual/src/language/builtins-prefix.md index 35e3dccc3..7b2321466 100644 --- a/doc/manual/src/language/builtins-prefix.md +++ b/doc/manual/src/language/builtins-prefix.md @@ -3,7 +3,7 @@ This section lists the functions built into the Nix language evaluator. All built-in functions are available through the global [`builtins`](./builtin-constants.md#builtins-builtins) constant. -For convenience, some built-ins are can be accessed directly: +For convenience, some built-ins can be accessed directly: - [`derivation`](#builtins-derivation) - [`import`](#builtins-import) From 50de11d662ced6bfef792af54ad57553f1c3208a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 5 Jun 2023 16:25:22 +0200 Subject: [PATCH 073/909] doc: Improve `fetchClosure` documentation --- doc/manual/src/release-notes/rl-next.md | 2 + src/libexpr/primops.cc | 2 + src/libexpr/primops/fetchClosure.cc | 66 +++++++++++++++++++------ 3 files changed, 55 insertions(+), 15 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 8479b166a..e64839fa2 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -2,5 +2,7 @@ - [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand +- The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure mode](../command-ref/conf-file.md#conf-pure-eval). + - Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths. Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths. diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5dfad470a..f4bdf8fc6 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1502,6 +1502,8 @@ static RegisterPrimOp primop_storePath({ in a new path (e.g. `/nix/store/ld01dnzc…-source-source`). Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). + + See also [`builtins.fetchClosure`](#builtins-fetchClosure). )", .fun = prim_storePath, }); diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index e048a2629..e344c9513 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -202,40 +202,76 @@ static RegisterPrimOp primop_fetchClosure({ .name = "__fetchClosure", .args = {"args"}, .doc = R"( - Fetch a Nix store closure from a binary cache, rewriting it into - content-addressed form. For example, + Fetch a Nix store closure from a binary cache. - ```nix - builtins.fetchClosure { - fromStore = "https://cache.nixos.org"; - fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1; - toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1; - } - ``` + This function can be used in three ways: - fetches `/nix/store/r2jd...` from the specified binary cache, + - Fetch any store path and rewrite it to a fully content-addressed store path. Example: + + ```nix + builtins.fetchClosure { + fromStore = "https://cache.nixos.org"; + fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1; + toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1; + } + ``` + + - Fetch a content-addressed store path. + + ```nix + builtins.fetchClosure { + fromStore = "https://cache.nixos.org"; + fromPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1; + } + ``` + + - Fetch an [input-addressed store path](@docroot@/glossary.md#gloss-input-addressed-store-object) as is. This depends on user configuration, which is less preferable. + + ```nix + builtins.fetchClosure { + fromStore = "https://cache.nixos.org"; + fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1; + inputAddressed = true; + } + ``` + + **Rewriting a store path** + + This first example fetches `/nix/store/r2jd...` from the specified binary cache, and rewrites it into the content-addressed store path `/nix/store/ldbh...`. - If `fromPath` is already content-addressed, or if you are - allowing input addressing (`inputAddressed = true;`), then `toPath` may be - omitted. + By rewriting the store paths, you can add the contents to any store without requiring that you or perhaps your users configure any extra trust. To find out the correct value for `toPath` given a `fromPath`, - you can use `nix store make-content-addressed`: + you can use [`nix store make-content-addressed`](@docroot@/command-ref/new-cli/nix3-store-make-content-addressed.md): ```console # nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1 rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1' ``` - This function is similar to `builtins.storePath` in that it + Alternatively, you may set `toPath = ""` and find the correct `toPath` in the error message. + + **Not rewriting** + + `toPath` may be omitted when either + - `fromPath` is already content-addressed, or + - `inputAddressed = true;` is set. + + Fetching an [input addressed path](@docroot@/glossary.md#gloss-input-addressed-store-object) is not recommended because it requires that the source be trusted by the Nix configuration. + + **`builtins.storePath`** + + This function is similar to [`builtins.storePath`](#builtins-storePath) in that it allows you to use a previously built store path in a Nix expression. However, it is more reproducible because it requires specifying a binary cache from which the path can be fetched. Also, the default requirement of a content-addressed final store path avoids the need for users to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys). + **Experimental feature** + This function is only available if you enable the experimental feature `fetch-closure`. )", From 40052c76132d79561aa50ec9349ad22b51c64505 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 7 Jun 2023 12:47:18 +0200 Subject: [PATCH 074/909] fetchClosure: Docs and error message improvements Co-authored-by: Valentin Gagarin --- doc/manual/src/release-notes/rl-next.md | 2 +- src/libexpr/primops/fetchClosure.cc | 38 ++++++++++++++++--------- tests/fetchClosure.sh | 2 +- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index e64839fa2..139d07188 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -2,7 +2,7 @@ - [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand -- The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure mode](../command-ref/conf-file.md#conf-pure-eval). +* The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure evaluation mode](../command-ref/conf-file.md#conf-pure-eval), as those are not impure. - Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths. Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths. diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index e344c9513..5114df553 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -8,11 +8,11 @@ namespace nix { /** * Handler for the content addressed case. * - * @param state The evaluator state and store to write to. - * @param fromStore The store containing the path to rewrite. - * @param fromPath The source path to be rewritten. - * @param toPathMaybe The path to write the rewritten path to. If empty, the error shows the actual path. - * @param v The return `Value` + * @param state Evaluator state and store to write to. + * @param fromStore Store containing the path to rewrite. + * @param fromPath Source path to be rewritten. + * @param toPathMaybe Path to write the rewritten path to. If empty, the error shows the actual path. + * @param v Return `Value` */ static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, const std::optional & toPathMaybe, Value &v) { @@ -31,8 +31,8 @@ static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Stor if (!toPathMaybe) throw Error({ .msg = hintfmt( - "rewriting '%s' to content-addressed form yielded '%s'; " - "please set this in the 'toPath' attribute passed to 'fetchClosure'", + "rewriting '%s' to content-addressed form yielded '%s'\n" + "Use this value for the 'toPath' attribute passed to 'fetchClosure'", state.store->printStorePath(fromPath), state.store->printStorePath(rewrittenPath)), .errPos = state.positions[pos] @@ -49,7 +49,9 @@ static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Stor // We don't perform the rewriting when outPath already exists, as an optimisation. // However, we can quickly detect a mistake if the toPath is input addressed. throw Error({ - .msg = hintfmt("The 'toPath' value '%s' is input addressed, so it can't possibly be the result of rewriting. You may set 'toPath' to an empty string to figure out the correct path.", + .msg = hintfmt( + "The 'toPath' value '%s' is input-addressed, so it can't possibly be the result of rewriting to a content-addressed path.\n\n" + "Set 'toPath' to an empty string to make Nix report the correct content-addressed path.", state.store->printStorePath(toPath)), .errPos = state.positions[pos] }); @@ -70,7 +72,9 @@ static void runFetchClosureWithContentAddressedPath(EvalState & state, const Pos if (!info->isContentAddressed(*state.store)) { throw Error({ - .msg = hintfmt("The 'fromPath' value '%s' is input addressed, but input addressing was not requested. If you do intend to return an input addressed store path, add 'inputAddressed = true;' to the 'fetchClosure' arguments. Note that content addressing does not require users to configure a trusted binary cache public key on their systems, and is therefore preferred.", + .msg = hintfmt( + "The 'fromPath' value '%s' is input-addressed, but 'inputAddressed' is set to 'false' (default).\n\n" + "If you do intend to fetch an input-addressed store path, add 'inputAddressed = true;' to the 'fetchClosure' arguments. Note that content addressing does not require users to configure a trusted binary cache public key on their systems, and is therefore preferred.", state.store->printStorePath(fromPath)), .errPos = state.positions[pos] }); @@ -91,7 +95,9 @@ static void runFetchClosureWithInputAddressedPath(EvalState & state, const PosId if (info->isContentAddressed(*state.store)) { throw Error({ - .msg = hintfmt("The 'fetchClosure' result, '%s' is not input addressed, despite 'inputAddressed' being set to true. It is preferable to return a content addressed path, so remove the 'inputAddressed' attribute to ensure content addressing is used in the future", + .msg = hintfmt( + "The store object referred to by 'fromPath' at '%s' is not input-addressed, but 'inputAddressed' is set to 'true'.\n\n" + "Remove the 'inputAddressed' attribute (it defaults to 'false') to expect 'fromPath' to be content-addressed", state.store->printStorePath(fromPath)), .errPos = state.positions[pos] }); @@ -202,11 +208,13 @@ static RegisterPrimOp primop_fetchClosure({ .name = "__fetchClosure", .args = {"args"}, .doc = R"( - Fetch a Nix store closure from a binary cache. + Fetch a store path [closure](@docroot@/glossary.md#gloss-closure) from a binary cache. This function can be used in three ways: - - Fetch any store path and rewrite it to a fully content-addressed store path. Example: + - Fetch any store path and rewrite it to a fully content-addressed store path. + + Example: ```nix builtins.fetchClosure { @@ -218,6 +226,8 @@ static RegisterPrimOp primop_fetchClosure({ - Fetch a content-addressed store path. +Example: + ```nix builtins.fetchClosure { fromStore = "https://cache.nixos.org"; @@ -227,6 +237,8 @@ static RegisterPrimOp primop_fetchClosure({ - Fetch an [input-addressed store path](@docroot@/glossary.md#gloss-input-addressed-store-object) as is. This depends on user configuration, which is less preferable. + Example: + ```nix builtins.fetchClosure { fromStore = "https://cache.nixos.org"; @@ -244,7 +256,7 @@ static RegisterPrimOp primop_fetchClosure({ By rewriting the store paths, you can add the contents to any store without requiring that you or perhaps your users configure any extra trust. To find out the correct value for `toPath` given a `fromPath`, - you can use [`nix store make-content-addressed`](@docroot@/command-ref/new-cli/nix3-store-make-content-addressed.md): + use [`nix store make-content-addressed`](@docroot@/command-ref/new-cli/nix3-store-make-content-addressed.md): ```console # nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1 diff --git a/tests/fetchClosure.sh b/tests/fetchClosure.sh index 4b8198e94..0f1c64372 100644 --- a/tests/fetchClosure.sh +++ b/tests/fetchClosure.sh @@ -52,7 +52,7 @@ if [[ "$NIX_REMOTE" != "daemon" ]]; then fromStore = \"file://$cacheDir\"; fromPath = $nonCaPath; } - " | grepQuiet -E "The .fromPath. value .* is input addressed, but input addressing was not requested. If you do intend to return an input addressed store path, add .inputAddressed = true;. to the .fetchClosure. arguments." + " | grepQuiet -E "The .fromPath. value .* is input-addressed, but .inputAddressed. is set to .false." [ -e $nonCaPath ] From 1db81f710725eb83597b1761076e9ca28317a52e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 19 Jun 2023 23:22:37 +0200 Subject: [PATCH 075/909] tests/fetchClosure: Improve coverage of new and some existing flows --- tests/fetchClosure.sh | 79 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 69 insertions(+), 10 deletions(-) diff --git a/tests/fetchClosure.sh b/tests/fetchClosure.sh index 0f1c64372..a02d1ce7a 100644 --- a/tests/fetchClosure.sh +++ b/tests/fetchClosure.sh @@ -33,8 +33,26 @@ clearStore [ ! -e $nonCaPath ] [ -e $caPath ] +clearStore + +# The daemon will reject input addressed paths unless configured to trust the +# cache key or the user. This behavior should be covered by another test, so we +# skip this part when using the daemon. if [[ "$NIX_REMOTE" != "daemon" ]]; then + # If we want to return a non-CA path, we have to be explicit about it. + expectStderr 1 nix eval --raw --no-require-sigs --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + } + " | grepQuiet -E "The .fromPath. value .* is input-addressed, but .inputAddressed. is set to .false." + + # TODO: Should the closure be rejected, despite single user mode? + # [ ! -e $nonCaPath ] + + [ ! -e $caPath ] + # We can use non-CA paths when we ask explicitly. [[ $(nix eval --raw --no-require-sigs --expr " builtins.fetchClosure { @@ -45,20 +63,13 @@ if [[ "$NIX_REMOTE" != "daemon" ]]; then ") = $nonCaPath ]] [ -e $nonCaPath ] - - # .. but only if we ask explicitly. - expectStderr 1 nix eval --raw --no-require-sigs --expr " - builtins.fetchClosure { - fromStore = \"file://$cacheDir\"; - fromPath = $nonCaPath; - } - " | grepQuiet -E "The .fromPath. value .* is input-addressed, but .inputAddressed. is set to .false." - - [ -e $nonCaPath ] + [ ! -e $caPath ] fi +[ ! -e $caPath ] + # 'toPath' set to empty string should fail but print the expected path. expectStderr 1 nix eval -v --json --expr " builtins.fetchClosure { @@ -71,6 +82,10 @@ expectStderr 1 nix eval -v --json --expr " # If fromPath is CA, then toPath isn't needed. nix copy --to file://$cacheDir $caPath +clearStore + +[ ! -e $caPath ] + [[ $(nix eval -v --raw --expr " builtins.fetchClosure { fromStore = \"file://$cacheDir\"; @@ -78,6 +93,8 @@ nix copy --to file://$cacheDir $caPath } ") = $caPath ]] +[ -e $caPath ] + # Check that URL query parameters aren't allowed. clearStore narCache=$TEST_ROOT/nar-cache @@ -89,3 +106,45 @@ rm -rf $narCache } ") (! [ -e $narCache ]) + +# If toPath is specified but wrong, we check it (only) when the path is missing. +clearStore + +badPath=$(echo $caPath | sed -e 's!/store/................................-!/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-!') + +[ ! -e $badPath ] + +expectStderr 1 nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + toPath = $badPath; + } +" | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath.*while.*$badPath.*was expected" + +[ ! -e $badPath ] + +# We only check it when missing, as a performance optimization similar to what we do for fixed output derivations. So if it's already there, we don't check it. +# It would be nice for this to fail, but checking it would be too(?) slow. +[ -e $caPath ] + +[[ $(nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $badPath; + toPath = $caPath; + } +") = $caPath ]] + + +# However, if the output address is unexpected, we can report it + + +expectStderr 1 nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $caPath; + inputAddressed = true; + } +" | grepQuiet 'error.*The store object referred to by.*fromPath.* at .* is not input-addressed, but .*inputAddressed.* is set to .*true.*' + From fefb94713295d63d30a8bba257c41c3eb54e0998 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 19 Jun 2023 23:22:56 +0200 Subject: [PATCH 076/909] tests/signing.sh: Check signature checking error message We should check error messages, so that we know the command fails for the right reason. Alternatively, a mere typo can run the test undetected. --- tests/signing.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/signing.sh b/tests/signing.sh index 9b673c609..f14b53e7f 100644 --- a/tests/signing.sh +++ b/tests/signing.sh @@ -84,7 +84,7 @@ info=$(nix path-info --store file://$cacheDir --json $outPath2) # Copying to a diverted store should fail due to a lack of signatures by trusted keys. chmod -R u+w $TEST_ROOT/store0 || true rm -rf $TEST_ROOT/store0 -(! nix copy --to $TEST_ROOT/store0 $outPath) +expectStderr 1 nix copy --to $TEST_ROOT/store0 $outPath | grepQuiet -E 'cannot add path .* because it lacks a signature by a trusted key' # But succeed if we supply the public keys. nix copy --to $TEST_ROOT/store0 $outPath --trusted-public-keys $pk1 From a6c17097d2ee76a347d86c42a37fd6de0d0fd3ef Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 30 Jun 2023 23:36:27 +0200 Subject: [PATCH 077/909] tests: Don't install test-libstoreconsumer program Sorry about that. Fixes https://github.com/NixOS/nix/issues/8616 --- tests/test-libstoreconsumer/local.mk | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test-libstoreconsumer/local.mk b/tests/test-libstoreconsumer/local.mk index cd2d0c7f8..edc140723 100644 --- a/tests/test-libstoreconsumer/local.mk +++ b/tests/test-libstoreconsumer/local.mk @@ -2,6 +2,9 @@ programs += test-libstoreconsumer test-libstoreconsumer_DIR := $(d) +# do not install +test-libstoreconsumer_INSTALL_DIR := + test-libstoreconsumer_SOURCES := \ $(wildcard $(d)/*.cc) \ From 87b82db8812736922de851a27681dd0b2955c2f0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 May 2022 10:33:58 +0200 Subject: [PATCH 078/909] nix profile list: Improve readability of the output --- src/nix/profile-list.md | 46 ++++++++++++++++++++++++++++++----------- src/nix/profile.cc | 14 +++++++++---- 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/src/nix/profile-list.md b/src/nix/profile-list.md index fa786162f..5d7fcc0ec 100644 --- a/src/nix/profile-list.md +++ b/src/nix/profile-list.md @@ -6,26 +6,48 @@ R""( ```console # nix profile list - 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.spotify /nix/store/akpdsid105phbbvknjsdh7hl4v3fhjkr-spotify-1.1.46.916.g416cacf1 - 1 flake:nixpkgs#legacyPackages.x86_64-linux.zoom-us github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.zoom-us /nix/store/89pmjmbih5qpi7accgacd17ybpgp4xfm-zoom-us-5.4.53350.1027 - 2 flake:blender-bin#packages.x86_64-linux.default github:edolstra/nix-warez/d09d7eea893dcb162e89bc67f6dc1ced14abfc27?dir=blender#packages.x86_64-linux.default /nix/store/zfgralhqjnam662kqsgq6isjw8lhrflz-blender-bin-2.91.0 + Index: 0 + Flake attribute: legacyPackages.x86_64-linux.gdb + Original flake URL: flake:nixpkgs + Locked flake URL: github:NixOS/nixpkgs/7b38b03d76ab71bdc8dc325e3f6338d984cc35ca + Store paths: /nix/store/indzcw5wvlhx6vwk7k4iq29q15chvr3d-gdb-11.1 + + Index: 1 + Flake attribute: packages.x86_64-linux.default + Original flake URL: flake:blender-bin + Locked flake URL: github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender + Store paths: /nix/store/i798sxl3j40wpdi1rgf391id1b5klw7g-blender-bin-3.1.2 ``` + Note that you can unambiguously rebuild a package from a profile + through its locked flake URL and flake attribute, e.g. + + ```console + # nix build github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender#packages.x86_64-linux.default + ``` + + will build the package with index 1 shown above. + # Description This command shows what packages are currently installed in a -profile. The output consists of one line per package, with the -following fields: +profile. For each installed package, it shows the following +information: -* An integer that can be used to unambiguously identify the package in - invocations of `nix profile remove` and `nix profile upgrade`. +* `Index`: An integer that can be used to unambiguously identify the + package in invocations of `nix profile remove` and `nix profile + upgrade`. -* The original ("unlocked") flake reference and output attribute path - used at installation time. +* `Flake attribute`: The flake output attribute path that provides the + package (e.g. `packages.x86_64-linux.hello`). -* The locked flake reference to which the unlocked flake reference was - resolved. +* `Original flake URL`: The original ("unlocked") flake reference + specified by the user when the package was first installed via `nix + profile install`. -* The store path(s) of the package. +* `Locked flake URL`: The locked flake reference to which the original + flake reference was resolved. + +* `Store paths`: The store path(s) of the package. )"" diff --git a/src/nix/profile.cc b/src/nix/profile.cc index f3b73f10d..b0cedb24a 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -655,10 +655,16 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); - logger->cout("%d %s %s %s", i, - element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-", - element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-", - concatStringsSep(" ", store->printStorePathSet(element.storePaths))); + if (i) logger->cout(""); + logger->cout("Index: " ANSI_BOLD "%s" ANSI_NORMAL "%s", + i, + element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL); + if (element.source) { + logger->cout("Flake attribute: %s%s", element.source->attrPath, element.source->outputs.to_string()); + logger->cout("Original flake URL: %s", element.source->originalRef.to_string()); + logger->cout("Locked flake URL: %s", element.source->resolvedRef.to_string()); + } + logger->cout("Store paths: %s", concatStringsSep(" ", store->printStorePathSet(element.storePaths))); } } }; From 3c90340fe64fb18e66f02a648ebce8ad629482a5 Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Mon, 3 Jul 2023 16:02:54 +0800 Subject: [PATCH 079/909] libexpr: use `thread_local` to make the parser thread-safe If we call `adjustLoc`, the global variable `prev_yylloc` is shared between threads and racy. Currently, nix itself does not concurrently parsing files, but this is helpful for libexpr users. (The parser is thread-safe except this.) --- src/libexpr/lexer.l | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 462b3b602..a3a8608d9 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -36,7 +36,7 @@ static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data) #define CUR_POS makeCurPos(*yylloc, data) // backup to recover from yyless(0) -YYLTYPE prev_yylloc; +thread_local YYLTYPE prev_yylloc; static void initLoc(YYLTYPE * loc) { From b8e8f27159251091141ccd6db12dd2d07204771a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 May 2022 10:35:28 +0200 Subject: [PATCH 080/909] Rename 'resolvedRef' to 'lockedRef' 'resolvedRef' was incorrect, since a resolved ref is one after registry resolution, which may still be unlocked (e.g. 'nixpkgs' -> 'github:NixOS/nixpkgs'). --- src/libcmd/installable-flake.cc | 2 +- src/libcmd/installable-flake.hh | 2 +- src/nix/profile.cc | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index eb944240b..4da9b131b 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -151,7 +151,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() }, ExtraPathInfoFlake::Flake { .originalRef = flakeRef, - .resolvedRef = getLockedFlake()->flake.lockedRef, + .lockedRef = getLockedFlake()->flake.lockedRef, }), }}; } diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh index 7ac4358d2..314918c14 100644 --- a/src/libcmd/installable-flake.hh +++ b/src/libcmd/installable-flake.hh @@ -19,7 +19,7 @@ struct ExtraPathInfoFlake : ExtraPathInfoValue */ struct Flake { FlakeRef originalRef; - FlakeRef resolvedRef; + FlakeRef lockedRef; }; Flake flake; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index b0cedb24a..ae2e8f4f1 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -21,7 +21,7 @@ struct ProfileElementSource { FlakeRef originalRef; // FIXME: record original attrpath. - FlakeRef resolvedRef; + FlakeRef lockedRef; std::string attrPath; ExtendedOutputsSpec outputs; @@ -181,7 +181,7 @@ struct ProfileManifest obj["priority"] = element.priority; if (element.source) { obj["originalUrl"] = element.source->originalRef.to_string(); - obj["url"] = element.source->resolvedRef.to_string(); + obj["url"] = element.source->lockedRef.to_string(); obj["attrPath"] = element.source->attrPath; obj["outputs"] = element.source->outputs; } @@ -349,7 +349,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile if (auto * info2 = dynamic_cast(&*info)) { element.source = ProfileElementSource { .originalRef = info2->flake.originalRef, - .resolvedRef = info2->flake.resolvedRef, + .lockedRef = info2->flake.lockedRef, .attrPath = info2->value.attrPath, .outputs = info2->value.extendedOutputsSpec, }; @@ -588,14 +588,14 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf assert(infop); auto & info = *infop; - if (element.source->resolvedRef == info.flake.resolvedRef) continue; + if (element.source->lockedRef == info.flake.lockedRef) continue; printInfo("upgrading '%s' from flake '%s' to '%s'", - element.source->attrPath, element.source->resolvedRef, info.flake.resolvedRef); + element.source->attrPath, element.source->lockedRef, info.flake.lockedRef); element.source = ProfileElementSource { .originalRef = installable->flakeRef, - .resolvedRef = info.flake.resolvedRef, + .lockedRef = info.flake.lockedRef, .attrPath = info.value.attrPath, .outputs = installable->extendedOutputsSpec, }; @@ -662,7 +662,7 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro if (element.source) { logger->cout("Flake attribute: %s%s", element.source->attrPath, element.source->outputs.to_string()); logger->cout("Original flake URL: %s", element.source->originalRef.to_string()); - logger->cout("Locked flake URL: %s", element.source->resolvedRef.to_string()); + logger->cout("Locked flake URL: %s", element.source->lockedRef.to_string()); } logger->cout("Store paths: %s", concatStringsSep(" ", store->printStorePathSet(element.storePaths))); } From a353412c43e776f4fdc709dc7affd1dbb9d588be Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 May 2022 10:44:26 +0200 Subject: [PATCH 081/909] nix profile list: Add --json flag This just dumps the profile manifest to stdout. --- src/nix/profile.cc | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index ae2e8f4f1..b833b5192 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -168,7 +168,7 @@ struct ProfileManifest } } - std::string toJSON(Store & store) const + nlohmann::json toJSON(Store & store) const { auto array = nlohmann::json::array(); for (auto & element : elements) { @@ -190,7 +190,7 @@ struct ProfileManifest nlohmann::json json; json["version"] = 2; json["elements"] = array; - return json.dump(); + return json; } StorePath build(ref store) @@ -210,7 +210,7 @@ struct ProfileManifest buildProfile(tempDir, std::move(pkgs)); - writeFile(tempDir + "/manifest.json", toJSON(*store)); + writeFile(tempDir + "/manifest.json", toJSON(*store).dump()); /* Add the symlink tree to the store. */ StringSink sink; @@ -635,7 +635,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf } }; -struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile +struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile, MixJSON { std::string description() override { @@ -653,18 +653,22 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro { ProfileManifest manifest(*getEvalState(), *profile); - for (size_t i = 0; i < manifest.elements.size(); ++i) { - auto & element(manifest.elements[i]); - if (i) logger->cout(""); - logger->cout("Index: " ANSI_BOLD "%s" ANSI_NORMAL "%s", - i, - element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL); - if (element.source) { - logger->cout("Flake attribute: %s%s", element.source->attrPath, element.source->outputs.to_string()); - logger->cout("Original flake URL: %s", element.source->originalRef.to_string()); - logger->cout("Locked flake URL: %s", element.source->lockedRef.to_string()); + if (json) { + std::cout << manifest.toJSON(*store).dump() << "\n"; + } else { + for (size_t i = 0; i < manifest.elements.size(); ++i) { + auto & element(manifest.elements[i]); + if (i) logger->cout(""); + logger->cout("Index: " ANSI_BOLD "%s" ANSI_NORMAL "%s", + i, + element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL); + if (element.source) { + logger->cout("Flake attribute: %s%s", element.source->attrPath, element.source->outputs.to_string()); + logger->cout("Original flake URL: %s", element.source->originalRef.to_string()); + logger->cout("Locked flake URL: %s", element.source->lockedRef.to_string()); + } + logger->cout("Store paths: %s", concatStringsSep(" ", store->printStorePathSet(element.storePaths))); } - logger->cout("Store paths: %s", concatStringsSep(" ", store->printStorePathSet(element.storePaths))); } } }; From 5fbfbb4c7c246090df3debf7688f5ee9f01a4cbb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 May 2022 13:53:56 +0200 Subject: [PATCH 082/909] Fix test --- tests/nix-profile.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index 9da3f802b..7c478a0cd 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -47,8 +47,9 @@ cp ./config.nix $flake1Dir/ # Test upgrading from nix-env. nix-env -f ./user-envs.nix -i foo-1.0 -nix profile list | grep '0 - - .*-foo-1.0' +nix profile list | grep -A2 'Index:.*0' | grep 'Store paths:.*foo-1.0' nix profile install $flake1Dir -L +nix profile list | grep -A4 'Index:.*1' | grep 'Locked flake URL:.*narHash' [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] [ -e $TEST_HOME/.nix-profile/share/man ] (! [ -e $TEST_HOME/.nix-profile/include ]) From 82d6699976aa51cb6c06272d3455b32299b2c4ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Tue, 4 Jul 2023 10:39:08 +0200 Subject: [PATCH 083/909] Document the path flakeref format (#8640) * Document the path flakeref format Fix https://github.com/NixOS/nix/issues/8482 Co-authored-by: Valentin Gagarin --- src/nix/flake.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index 456fd0ea1..92f477917 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -71,8 +71,6 @@ inputs.nixpkgs = { Here are some examples of flake references in their URL-like representation: -* `.`: The flake in the current directory. -* `/home/alice/src/patchelf`: A flake in some other directory. * `nixpkgs`: The `nixpkgs` entry in the flake registry. * `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: The `nixpkgs` entry in the flake registry, with its Git revision overridden to a @@ -93,6 +91,23 @@ Here are some examples of flake references in their URL-like representation: * `https://github.com/NixOS/patchelf/archive/master.tar.gz`: A tarball flake. +## Path-like syntax + +Flakes corresponding to a local path can also be referred to by a direct path reference, either `/absolute/path/to/the/flake` or `./relative/path/to/the/flake` (note that the leading `./` is mandatory for relative paths to avoid any ambiguity). + +The semantic of such a path is as follows: + +* If the directory is part of a Git repository, then the input will be treated as a `git+file:` URL, otherwise it will be treated as a `path:` url; +* If the directory doesn't contain a `flake.nix` file, then Nix will search for such a file upwards in the file system hierarchy until it finds any of: + 1. The Git repository root, or + 2. The filesystem root (/), or + 3. A folder on a different mount point. + +### Examples + +* `.`: The flake to which the current directory belongs to. +* `/home/alice/src/patchelf`: A flake in some other directory. + ## Flake reference attributes The following generic flake reference attributes are supported: From ab78d8804e35f889965a561719d84ba6cf904ddc Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 6 Jul 2023 11:26:51 +0200 Subject: [PATCH 084/909] .gitignore: Add .cache/ --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 969194650..ec952b918 100644 --- a/.gitignore +++ b/.gitignore @@ -138,3 +138,6 @@ nix-rust/target result .vscode/ + +# clangd and possibly more +.cache/ From 4b2f155f0a22179e46d2c55cb1bd074c885fd08e Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Thu, 6 Jul 2023 09:00:49 -0700 Subject: [PATCH 085/909] nix-daemon.service: Add TasksMax=infinity --- misc/systemd/nix-daemon.service.in | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in index f46413630..4b4474475 100644 --- a/misc/systemd/nix-daemon.service.in +++ b/misc/systemd/nix-daemon.service.in @@ -10,6 +10,7 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket ExecStart=@@bindir@/nix-daemon nix-daemon --daemon KillMode=process LimitNOFILE=1048576 +TasksMax=infinity [Install] WantedBy=multi-user.target From 537e8beb770ed92dd1d5a20796a86620c4c61389 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 7 Jul 2023 11:00:40 +0200 Subject: [PATCH 086/909] fetchClosure: Apply suggestions from code review Co-authored-by: Valentin Gagarin --- src/libexpr/primops/fetchClosure.cc | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 5114df553..22bae19b7 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -74,7 +74,10 @@ static void runFetchClosureWithContentAddressedPath(EvalState & state, const Pos throw Error({ .msg = hintfmt( "The 'fromPath' value '%s' is input-addressed, but 'inputAddressed' is set to 'false' (default).\n\n" - "If you do intend to fetch an input-addressed store path, add 'inputAddressed = true;' to the 'fetchClosure' arguments. Note that content addressing does not require users to configure a trusted binary cache public key on their systems, and is therefore preferred.", + "If you do intend to fetch an input-addressed store path, add\n\n" + " inputAddressed = true;\n\n" + "to the 'fetchClosure' arguments.\n\n" + "Note that to ensure authenticity input-addressed store paths, users must configure a trusted binary cache public key on their systems. This is not needed for content-addressed paths.", state.store->printStorePath(fromPath)), .errPos = state.positions[pos] }); @@ -226,7 +229,7 @@ static RegisterPrimOp primop_fetchClosure({ - Fetch a content-addressed store path. -Example: + Example: ```nix builtins.fetchClosure { @@ -263,7 +266,7 @@ Example: rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1' ``` - Alternatively, you may set `toPath = ""` and find the correct `toPath` in the error message. + Alternatively, set `toPath = ""` and find the correct `toPath` in the error message. **Not rewriting** @@ -279,13 +282,7 @@ Example: allows you to use a previously built store path in a Nix expression. However, it is more reproducible because it requires specifying a binary cache from which the path can be fetched. - Also, the default requirement of a content-addressed final store path - avoids the need for users to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys). - - **Experimental feature** - - This function is only available if you enable the experimental - feature `fetch-closure`. + Also, using content-addressed store paths does not require users to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity. )", .fun = prim_fetchClosure, .experimentalFeature = Xp::FetchClosure, From b4b02d084f066d6391074ac807e532e36e4eccd9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 7 Jul 2023 11:40:40 +0200 Subject: [PATCH 087/909] fetchClosure: Interleave the examples in the docs --- src/libexpr/primops/fetchClosure.cc | 79 ++++++++++++++--------------- 1 file changed, 37 insertions(+), 42 deletions(-) diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 22bae19b7..7fe8203f4 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -211,52 +211,42 @@ static RegisterPrimOp primop_fetchClosure({ .name = "__fetchClosure", .args = {"args"}, .doc = R"( - Fetch a store path [closure](@docroot@/glossary.md#gloss-closure) from a binary cache. + Fetch a store path [closure](@docroot@/glossary.md#gloss-closure) from a binary cache, and return the store path as a string with context. - This function can be used in three ways: + This function can be invoked in three ways, that we will discuss in order of preference. - - Fetch any store path and rewrite it to a fully content-addressed store path. - - Example: - - ```nix - builtins.fetchClosure { - fromStore = "https://cache.nixos.org"; - fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1; - toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1; - } - ``` - - - Fetch a content-addressed store path. + **Fetch a content-addressed store path** Example: - ```nix - builtins.fetchClosure { - fromStore = "https://cache.nixos.org"; - fromPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1; - } - ``` + ```nix + builtins.fetchClosure { + fromStore = "https://cache.nixos.org"; + fromPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1; + } + ``` - - Fetch an [input-addressed store path](@docroot@/glossary.md#gloss-input-addressed-store-object) as is. This depends on user configuration, which is less preferable. + This is the simplest invocation, and it does not require the user of the expression to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity. - Example: + If your store path is [input addressed](@docroot@/glossary.md#gloss-input-addressed-store-object) instead of content addressed, consider the other two invocations. - ```nix - builtins.fetchClosure { - fromStore = "https://cache.nixos.org"; - fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1; - inputAddressed = true; - } - ``` + **Fetch any store path and rewrite it to a fully content-addressed store path** - **Rewriting a store path** + Example: - This first example fetches `/nix/store/r2jd...` from the specified binary cache, + ```nix + builtins.fetchClosure { + fromStore = "https://cache.nixos.org"; + fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1; + toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1; + } + ``` + + This example fetches `/nix/store/r2jd...` from the specified binary cache, and rewrites it into the content-addressed store path `/nix/store/ldbh...`. - By rewriting the store paths, you can add the contents to any store without requiring that you or perhaps your users configure any extra trust. + Like the previous example, no extra configuration or privileges are required. To find out the correct value for `toPath` given a `fromPath`, use [`nix store make-content-addressed`](@docroot@/command-ref/new-cli/nix3-store-make-content-addressed.md): @@ -268,20 +258,25 @@ static RegisterPrimOp primop_fetchClosure({ Alternatively, set `toPath = ""` and find the correct `toPath` in the error message. - **Not rewriting** + **Fetch an input-addressed store path as is** - `toPath` may be omitted when either - - `fromPath` is already content-addressed, or - - `inputAddressed = true;` is set. + Example: - Fetching an [input addressed path](@docroot@/glossary.md#gloss-input-addressed-store-object) is not recommended because it requires that the source be trusted by the Nix configuration. + ```nix + builtins.fetchClosure { + fromStore = "https://cache.nixos.org"; + fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1; + inputAddressed = true; + } + ``` + + It is possible to fetch an [input-addressed store path](@docroot@/glossary.md#gloss-input-addressed-store-object) and return it as is. + However, this is the least preferred way of invoking `fetchClosure`, because it requires that the input-addressed paths are trusted by the Nix configuration. **`builtins.storePath`** - This function is similar to [`builtins.storePath`](#builtins-storePath) in that it - allows you to use a previously built store path in a Nix - expression. However, it is more reproducible because it requires - specifying a binary cache from which the path can be fetched. + `fetchClosure` is similar to [`builtins.storePath`](#builtins-storePath) in that it allows you to use a previously built store path in a Nix expression. + However, `fetchClosure` is more reproducible because it specifies a binary cache from which the path can be fetched. Also, using content-addressed store paths does not require users to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity. )", .fun = prim_fetchClosure, From 903700c5e168e2ab5bd6bee5e09b5908cb2908d6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 5 Jul 2023 18:53:44 -0400 Subject: [PATCH 088/909] Simplify `ContentAddress` Whereas `ContentAddressWithReferences` is a sum type complex because different varieties support different notions of reference, and `ContentAddressMethod` is a nested enum to support that, `ContentAddress` can be a simple pair of a method and hash. `ContentAddress` does not need to be a sum type on the outside because the choice of method doesn't effect what type of hashes we can use. Co-Authored-By: Cale Gibbard --- perl/lib/Nix/Store.xs | 6 +- src/libexpr/primops.cc | 13 +- src/libexpr/primops/fetchTree.cc | 6 +- src/libfetchers/fetchers.cc | 6 +- src/libfetchers/tarball.cc | 6 +- src/libstore/binary-cache-store.cc | 16 +-- src/libstore/build/local-derivation-goal.cc | 6 +- src/libstore/content-address.cc | 125 ++++++-------------- src/libstore/content-address.hh | 91 +++++--------- src/libstore/derivations.cc | 22 ++-- src/libstore/local-store.cc | 83 +++++++------ src/libstore/local-store.hh | 8 +- src/libstore/make-content-addressed.cc | 6 +- src/libstore/path-info.cc | 21 ++-- src/libstore/store-api.cc | 28 ++--- src/libstore/tests/derivation.cc | 6 +- src/nix-store/nix-store.cc | 6 +- src/nix/add-to-store.cc | 6 +- src/nix/prefetch.cc | 8 +- src/nix/profile.cc | 6 +- 20 files changed, 182 insertions(+), 293 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 41ecbbeb4..c38ea2d2b 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -294,10 +294,8 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) auto h = Hash::parseAny(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { - .hash = { - .method = method, - .hash = h, - }, + .method = method, + .hash = h, .references = {}, }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5dfad470a..a8b4a069f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1300,9 +1300,10 @@ drvName, Bindings * attrs, Value & v) auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); DerivationOutput::CAFixed dof { - .ca = ContentAddress::fromParts( - std::move(method), - std::move(h)), + .ca = ContentAddress { + .method = std::move(method), + .hash = std::move(h), + }, }; drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out")); @@ -2162,10 +2163,8 @@ static void addPath( std::optional expectedStorePath; if (expectedHash) expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { - .hash = { - .method = method, - .hash = *expectedHash, - }, + .method = method, + .hash = *expectedHash, .references = {}, }); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 579a45f92..5e668c629 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -254,10 +254,8 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v auto expectedPath = state.store->makeFixedOutputPath( name, FixedOutputInfo { - .hash = { - .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, - .hash = *expectedHash, - }, + .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, + .hash = *expectedHash, .references = {} }); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 2860c1ceb..f86c0604e 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -217,10 +217,8 @@ StorePath Input::computeStorePath(Store & store) const if (!narHash) throw Error("cannot compute store path for unlocked input '%s'", to_string()); return store.makeFixedOutputPath(getName(), FixedOutputInfo { - .hash = { - .method = FileIngestionMethod::Recursive, - .hash = *narHash, - }, + .method = FileIngestionMethod::Recursive, + .hash = *narHash, .references = {}, }); } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index e42aca6db..a012234e0 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -77,10 +77,8 @@ DownloadFileResult downloadFile( *store, name, FixedOutputInfo { - .hash = { - .method = FileIngestionMethod::Flat, - .hash = hash, - }, + .method = FileIngestionMethod::Flat, + .hash = hash, .references = {}, }, hashString(htSHA256, sink.s), diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index fcd763a9d..b4fea693f 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -309,10 +309,8 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n *this, name, FixedOutputInfo { - .hash = { - .method = method, - .hash = nar.first, - }, + .method = method, + .hash = nar.first, .references = { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus @@ -428,10 +426,8 @@ StorePath BinaryCacheStore::addToStore( *this, name, FixedOutputInfo { - .hash = { - .method = method, - .hash = h, - }, + .method = method, + .hash = h, .references = { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus @@ -465,8 +461,8 @@ StorePath BinaryCacheStore::addTextToStore( *this, std::string { name }, TextInfo { - { .hash = textHash }, - references, + .hash = textHash, + .references = references, }, nar.first, }; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index ee66ee500..7b98a8601 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2538,16 +2538,16 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() }, [&](const DerivationOutput::CAFixed & dof) { - auto wanted = dof.ca.getHash(); + auto & wanted = dof.ca.hash; auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { - .method = dof.ca.getMethod(), + .method = dof.ca.method, .hashType = wanted.type, }); /* Check wanted hash */ assert(newInfo0.ca); - auto got = newInfo0.ca->getHash(); + auto & got = newInfo0.ca->hash; if (wanted != got) { /* Throw an error after registering the path as valid. */ diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 04f7ac214..080456e18 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -4,11 +4,6 @@ namespace nix { -std::string FixedOutputHash::printMethodAlgo() const -{ - return makeFileIngestionPrefix(method) + printHashType(hash.type); -} - std::string makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { @@ -42,21 +37,6 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) return method; } -std::string ContentAddress::render() const -{ - return std::visit(overloaded { - [](const TextHash & th) { - return "text:" - + th.hash.to_string(Base32, true); - }, - [](const FixedOutputHash & fsh) { - return "fixed:" - + makeFileIngestionPrefix(fsh.method) - + fsh.hash.to_string(Base32, true); - } - }, raw); -} - std::string ContentAddressMethod::render(HashType ht) const { return std::visit(overloaded { @@ -69,6 +49,20 @@ std::string ContentAddressMethod::render(HashType ht) const }, raw); } +std::string ContentAddress::render() const +{ + return std::visit(overloaded { + [](const TextIngestionMethod &) -> std::string { + return "text:"; + }, + [](const FileIngestionMethod & method) { + return "fixed:" + + makeFileIngestionPrefix(method); + }, + }, method.raw) + + this->hash.to_string(Base32, true); +} + /** * Parses content address strings up to the hash. */ @@ -118,22 +112,12 @@ ContentAddress ContentAddress::parse(std::string_view rawCa) { auto rest = rawCa; - auto [caMethod, hashType_] = parseContentAddressMethodPrefix(rest); - auto hashType = hashType_; // work around clang bug + auto [caMethod, hashType] = parseContentAddressMethodPrefix(rest); - return std::visit(overloaded { - [&](TextIngestionMethod &) { - return ContentAddress(TextHash { - .hash = Hash::parseNonSRIUnprefixed(rest, hashType) - }); - }, - [&](FileIngestionMethod & fim) { - return ContentAddress(FixedOutputHash { - .method = fim, - .hash = Hash::parseNonSRIUnprefixed(rest, hashType), - }); - }, - }, caMethod.raw); + return ContentAddress { + .method = std::move(caMethod).raw, + .hash = Hash::parseNonSRIUnprefixed(rest, hashType), + }; } std::pair ContentAddressMethod::parse(std::string_view caMethod) @@ -156,52 +140,10 @@ std::string renderContentAddress(std::optional ca) return ca ? ca->render() : ""; } -ContentAddress ContentAddress::fromParts( - ContentAddressMethod method, Hash hash) noexcept -{ - return std::visit(overloaded { - [&](TextIngestionMethod _) -> ContentAddress { - return TextHash { - .hash = std::move(hash), - }; - }, - [&](FileIngestionMethod m2) -> ContentAddress { - return FixedOutputHash { - .method = std::move(m2), - .hash = std::move(hash), - }; - }, - }, method.raw); -} - -ContentAddressMethod ContentAddress::getMethod() const -{ - return std::visit(overloaded { - [](const TextHash & th) -> ContentAddressMethod { - return TextIngestionMethod {}; - }, - [](const FixedOutputHash & fsh) -> ContentAddressMethod { - return fsh.method; - }, - }, raw); -} - -const Hash & ContentAddress::getHash() const -{ - return std::visit(overloaded { - [](const TextHash & th) -> auto & { - return th.hash; - }, - [](const FixedOutputHash & fsh) -> auto & { - return fsh.hash; - }, - }, raw); -} - std::string ContentAddress::printMethodAlgo() const { - return getMethod().renderPrefix() - + printHashType(getHash().type); + return method.renderPrefix() + + printHashType(hash.type); } bool StoreReferences::empty() const @@ -217,19 +159,20 @@ size_t StoreReferences::size() const ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept { return std::visit(overloaded { - [&](const TextHash & h) -> ContentAddressWithReferences { + [&](const TextIngestionMethod &) -> ContentAddressWithReferences { return TextInfo { - .hash = h, + .hash = ca.hash, .references = {}, }; }, - [&](const FixedOutputHash & h) -> ContentAddressWithReferences { + [&](const FileIngestionMethod & method) -> ContentAddressWithReferences { return FixedOutputInfo { - .hash = h, + .method = method, + .hash = ca.hash, .references = {}, }; }, - }, ca.raw); + }, ca.method.raw); } std::optional ContentAddressWithReferences::fromPartsOpt( @@ -241,7 +184,7 @@ std::optional ContentAddressWithReferences::fromPa return std::nullopt; return ContentAddressWithReferences { TextInfo { - .hash = { .hash = std::move(hash) }, + .hash = std::move(hash), .references = std::move(refs.others), } }; @@ -249,10 +192,8 @@ std::optional ContentAddressWithReferences::fromPa [&](FileIngestionMethod m2) -> std::optional { return ContentAddressWithReferences { FixedOutputInfo { - .hash = { - .method = m2, - .hash = std::move(hash), - }, + .method = m2, + .hash = std::move(hash), .references = std::move(refs), } }; @@ -267,7 +208,7 @@ ContentAddressMethod ContentAddressWithReferences::getMethod() const return TextIngestionMethod {}; }, [](const FixedOutputInfo & fsh) -> ContentAddressMethod { - return fsh.hash.method; + return fsh.method; }, }, raw); } @@ -276,10 +217,10 @@ Hash ContentAddressWithReferences::getHash() const { return std::visit(overloaded { [](const TextInfo & th) { - return th.hash.hash; + return th.hash; }, [](const FixedOutputInfo & fsh) { - return fsh.hash.hash; + return fsh.hash; }, }, raw); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index e1e80448b..01b771e52 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -113,37 +113,6 @@ struct ContentAddressMethod * Mini content address */ -/** - * Somewhat obscure, used by \ref Derivation derivations and - * `builtins.toFile` currently. - */ -struct TextHash { - /** - * Hash of the contents of the text/file. - */ - Hash hash; - - GENERATE_CMP(TextHash, me->hash); -}; - -/** - * Used by most store objects that are content-addressed. - */ -struct FixedOutputHash { - /** - * How the file system objects are serialized - */ - FileIngestionMethod method; - /** - * Hash of that serialization - */ - Hash hash; - - std::string printMethodAlgo() const; - - GENERATE_CMP(FixedOutputHash, me->method, me->hash); -}; - /** * We've accumulated several types of content-addressed paths over the * years; fixed-output derivations support multiple hash algorithms and @@ -158,19 +127,17 @@ struct FixedOutputHash { */ struct ContentAddress { - typedef std::variant< - TextHash, - FixedOutputHash - > Raw; + /** + * How the file system objects are serialized + */ + ContentAddressMethod method; - Raw raw; + /** + * Hash of that serialization + */ + Hash hash; - GENERATE_CMP(ContentAddress, me->raw); - - /* The moral equivalent of `using Raw::Raw;` */ - ContentAddress(auto &&... arg) - : raw(std::forward(arg)...) - { } + GENERATE_CMP(ContentAddress, me->method, me->hash); /** * Compute the content-addressability assertion @@ -183,20 +150,6 @@ struct ContentAddress static std::optional parseOpt(std::string_view rawCaOpt); - /** - * Create a `ContentAddress` from 2 parts: - * - * @param method Way ingesting the file system data. - * - * @param hash Hash of ingested file system data. - */ - static ContentAddress fromParts( - ContentAddressMethod method, Hash hash) noexcept; - - ContentAddressMethod getMethod() const; - - const Hash & getHash() const; - std::string printMethodAlgo() const; }; @@ -219,7 +172,8 @@ std::string renderContentAddress(std::optional ca); * References to other store objects are tracked with store paths, self * references however are tracked with a boolean. */ -struct StoreReferences { +struct StoreReferences +{ /** * References to other store objects */ @@ -246,8 +200,13 @@ struct StoreReferences { }; // This matches the additional info that we need for makeTextPath -struct TextInfo { - TextHash hash; +struct TextInfo +{ + /** + * Hash of the contents of the text/file. + */ + Hash hash; + /** * References to other store objects only; self references * disallowed @@ -257,8 +216,18 @@ struct TextInfo { GENERATE_CMP(TextInfo, me->hash, me->references); }; -struct FixedOutputInfo { - FixedOutputHash hash; +struct FixedOutputInfo +{ + /** + * How the file system objects are serialized + */ + FileIngestionMethod method; + + /** + * Hash of that serialization + */ + Hash hash; + /** * References to other store objects or this one. */ diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 6f63685d4..b831b2fe5 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -232,9 +232,10 @@ static DerivationOutput parseDerivationOutput(const Store & store, validatePath(pathS); auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType); return DerivationOutput::CAFixed { - .ca = ContentAddress::fromParts( - std::move(method), - std::move(hash)), + .ca = ContentAddress { + .method = std::move(method), + .hash = std::move(hash), + }, }; } else { experimentalFeatureSettings.require(Xp::CaDerivations); @@ -395,7 +396,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, [&](const DerivationOutput::CAFixed & dof) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); s += ','; printUnquotedString(s, dof.ca.printMethodAlgo()); - s += ','; printUnquotedString(s, dof.ca.getHash().to_string(Base16, false)); + s += ','; printUnquotedString(s, dof.ca.hash.to_string(Base16, false)); }, [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); @@ -628,7 +629,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut auto & dof = std::get(i.second.raw()); auto hash = hashString(htSHA256, "fixed:out:" + dof.ca.printMethodAlgo() + ":" - + dof.ca.getHash().to_string(Base16, false) + ":" + + dof.ca.hash.to_string(Base16, false) + ":" + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } @@ -780,7 +781,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr [&](const DerivationOutput::CAFixed & dof) { out << store.printStorePath(dof.path(store, drv.name, i.first)) << dof.ca.printMethodAlgo() - << dof.ca.getHash().to_string(Base16, false); + << dof.ca.hash.to_string(Base16, false); }, [&](const DerivationOutput::CAFloating & dof) { out << "" @@ -970,7 +971,7 @@ nlohmann::json DerivationOutput::toJSON( [&](const DerivationOutput::CAFixed & dof) { res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); res["hashAlgo"] = dof.ca.printMethodAlgo(); - res["hash"] = dof.ca.getHash().to_string(Base16, false); + res["hash"] = dof.ca.hash.to_string(Base16, false); // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { @@ -1017,9 +1018,10 @@ DerivationOutput DerivationOutput::fromJSON( else if (keys == (std::set { "path", "hashAlgo", "hash" })) { auto [method, hashType] = methodAlgo(); auto dof = DerivationOutput::CAFixed { - .ca = ContentAddress::fromParts( - std::move(method), - Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType)), + .ca = ContentAddress { + .method = std::move(method), + .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType), + }, }; if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"])) throw Error("Path doesn't match derivation output"); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e69460e6c..c468cb51e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1249,27 +1249,17 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, printStorePath(info.path), info.narSize, hashResult.second); if (info.ca) { - if (auto foHash = std::get_if(&info.ca->raw)) { - auto actualFoHash = hashCAPath( - foHash->method, - foHash->hash.type, - info.path - ); - if (foHash->hash != actualFoHash.hash) { - throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), - foHash->hash.to_string(Base32, true), - actualFoHash.hash.to_string(Base32, true)); - } - } - if (auto textHash = std::get_if(&info.ca->raw)) { - auto actualTextHash = hashString(htSHA256, readFile(realPath)); - if (textHash->hash != actualTextHash) { - throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), - textHash->hash.to_string(Base32, true), - actualTextHash.to_string(Base32, true)); - } + auto & specified = *info.ca; + auto actualHash = hashCAPath( + specified.method, + specified.hash.type, + info.path + ); + if (specified.hash != actualHash.hash) { + throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", + printStorePath(info.path), + specified.hash.to_string(Base32, true), + actualHash.hash.to_string(Base32, true)); } } @@ -1349,10 +1339,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto [hash, size] = hashSink->finish(); ContentAddressWithReferences desc = FixedOutputInfo { - .hash = { - .method = method, - .hash = hash, - }, + .method = method, + .hash = hash, .references = { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus @@ -1428,8 +1416,8 @@ StorePath LocalStore::addTextToStore( { auto hash = hashString(htSHA256, s); auto dstPath = makeTextPath(name, TextInfo { - { .hash = hash }, - references, + .hash = hash, + .references = references, }); addTempRoot(dstPath); @@ -1459,7 +1447,10 @@ StorePath LocalStore::addTextToStore( ValidPathInfo info { dstPath, narHash }; info.narSize = sink.s.size(); info.references = references; - info.ca = TextHash { .hash = hash }; + info.ca = { + .method = TextIngestionMethod {}, + .hash = hash, + }; registerValidPath(info); } @@ -1856,33 +1847,39 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id, } } -FixedOutputHash LocalStore::hashCAPath( - const FileIngestionMethod & method, const HashType & hashType, +ContentAddress LocalStore::hashCAPath( + const ContentAddressMethod & method, const HashType & hashType, const StorePath & path) { return hashCAPath(method, hashType, Store::toRealPath(path), path.hashPart()); } -FixedOutputHash LocalStore::hashCAPath( - const FileIngestionMethod & method, +ContentAddress LocalStore::hashCAPath( + const ContentAddressMethod & method, const HashType & hashType, const Path & path, const std::string_view pathHash ) { HashModuloSink caSink ( hashType, std::string(pathHash) ); - switch (method) { - case FileIngestionMethod::Recursive: - dumpPath(path, caSink); - break; - case FileIngestionMethod::Flat: - readFile(path, caSink); - break; - } - auto hash = caSink.finish().first; - return FixedOutputHash{ + std::visit(overloaded { + [&](const TextIngestionMethod &) { + readFile(path, caSink); + }, + [&](const FileIngestionMethod & m2) { + switch (m2) { + case FileIngestionMethod::Recursive: + dumpPath(path, caSink); + break; + case FileIngestionMethod::Flat: + readFile(path, caSink); + break; + } + }, + }, method.raw); + return ContentAddress { .method = method, - .hash = hash, + .hash = caSink.finish().first, }; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 8a3b0b43f..d76662f33 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -345,13 +345,13 @@ private: void signRealisation(Realisation &); // XXX: Make a generic `Store` method - FixedOutputHash hashCAPath( - const FileIngestionMethod & method, + ContentAddress hashCAPath( + const ContentAddressMethod & method, const HashType & hashType, const StorePath & path); - FixedOutputHash hashCAPath( - const FileIngestionMethod & method, + ContentAddress hashCAPath( + const ContentAddressMethod & method, const HashType & hashType, const Path & path, const std::string_view pathHash diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 53fe04704..349c2f187 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -52,10 +52,8 @@ std::map makeContentAddressed( dstStore, path.name(), FixedOutputInfo { - .hash = { - .method = FileIngestionMethod::Recursive, - .hash = narModuloHash, - }, + .method = FileIngestionMethod::Recursive, + .hash = narModuloHash, .references = std::move(refs), }, Hash::dummy, diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 981bbfb14..ccb57104f 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -29,14 +29,14 @@ std::optional ValidPathInfo::contentAddressWithRef return std::nullopt; return std::visit(overloaded { - [&](const TextHash & th) -> ContentAddressWithReferences { + [&](const TextIngestionMethod &) -> ContentAddressWithReferences { assert(references.count(path) == 0); return TextInfo { - .hash = th, + .hash = ca->hash, .references = references, }; }, - [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { + [&](const FileIngestionMethod & m2) -> ContentAddressWithReferences { auto refs = references; bool hasSelfReference = false; if (refs.count(path)) { @@ -44,14 +44,15 @@ std::optional ValidPathInfo::contentAddressWithRef refs.erase(path); } return FixedOutputInfo { - .hash = foh, + .method = m2, + .hash = ca->hash, .references = { .others = std::move(refs), .self = hasSelfReference, }, }; }, - }, ca->raw); + }, ca->method.raw); } bool ValidPathInfo::isContentAddressed(const Store & store) const @@ -110,13 +111,19 @@ ValidPathInfo::ValidPathInfo( std::visit(overloaded { [this](TextInfo && ti) { this->references = std::move(ti.references); - this->ca = std::move((TextHash &&) ti); + this->ca = ContentAddress { + .method = TextIngestionMethod {}, + .hash = std::move(ti.hash), + }; }, [this](FixedOutputInfo && foi) { this->references = std::move(foi.references.others); if (foi.references.self) this->references.insert(path); - this->ca = std::move((FixedOutputHash &&) foi); + this->ca = ContentAddress { + .method = std::move(foi.method), + .hash = std::move(foi.hash), + }; }, }, std::move(ca).raw); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5bee1af9f..b16f5a6b7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -184,15 +184,15 @@ static std::string makeType( StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (info.hash.hash.type == htSHA256 && info.hash.method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", info.references), info.hash.hash, name); + if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { assert(info.references.size() == 0); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(info.hash.method) - + info.hash.hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.method) + + info.hash.to_string(Base16, true) + ":"), name); } } @@ -200,13 +200,13 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const { - assert(info.hash.hash.type == htSHA256); + assert(info.hash.type == htSHA256); return makeStorePath( makeType(*this, "text", StoreReferences { .others = info.references, .self = false, }), - info.hash.hash, + info.hash, name); } @@ -232,10 +232,8 @@ std::pair Store::computeStorePathForPath(std::string_view name, ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); FixedOutputInfo caInfo { - .hash = { - .method = method, - .hash = h, - }, + .method = method, + .hash = h, .references = {}, }; return std::make_pair(makeFixedOutputPath(name, caInfo), h); @@ -248,8 +246,8 @@ StorePath Store::computeStorePathForText( const StorePathSet & references) const { return makeTextPath(name, TextInfo { - { .hash = hashString(htSHA256, s) }, - references, + .hash = hashString(htSHA256, s), + .references = references, }); } @@ -441,10 +439,8 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, *this, name, FixedOutputInfo { - .hash = { - .method = method, - .hash = hash, - }, + .method = method, + .hash = hash, .references = {}, }, narHash, diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 6328ad370..0e28c1f08 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -81,7 +81,7 @@ TEST_JSON(DerivationTest, caFixedFlat, "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" })", (DerivationOutput::CAFixed { - .ca = FixedOutputHash { + .ca = { .method = FileIngestionMethod::Flat, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, @@ -95,7 +95,7 @@ TEST_JSON(DerivationTest, caFixedNAR, "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" })", (DerivationOutput::CAFixed { - .ca = FixedOutputHash { + .ca = { .method = FileIngestionMethod::Recursive, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, @@ -109,7 +109,7 @@ TEST_JSON(DynDerivationTest, caFixedText, "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" })", (DerivationOutput::CAFixed { - .ca = TextHash { + .ca = { .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, }), diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index caa0248f1..94956df66 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -220,10 +220,8 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) std::string name = *i++; cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { - .hash = { - .method = method, - .hash = Hash::parseAny(hash, hashAlgo), - }, + .method = method, + .hash = Hash::parseAny(hash, hashAlgo), .references = {}, }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 16e48a39b..39e5cc99d 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -45,10 +45,8 @@ struct CmdAddToStore : MixDryRun, StoreCommand *store, std::move(*namePart), FixedOutputInfo { - .hash = { - .method = std::move(ingestionMethod), - .hash = std::move(hash), - }, + .method = std::move(ingestionMethod), + .hash = std::move(hash), .references = {}, }, narHash, diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 3b2e225f6..b67d381ca 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -71,10 +71,8 @@ std::tuple prefetchFile( if (expectedHash) { hashType = expectedHash->type; storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { - .hash = { - .method = ingestionMethod, - .hash = *expectedHash, - }, + .method = ingestionMethod, + .hash = *expectedHash, .references = {}, }); if (store->isValidPath(*storePath)) @@ -127,7 +125,7 @@ std::tuple prefetchFile( auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); storePath = info.path; assert(info.ca); - hash = info.ca->getHash(); + hash = info.ca->hash; } return {storePath.value(), hash.value()}; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index b833b5192..476ddcd60 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -222,10 +222,8 @@ struct ProfileManifest *store, "profile", FixedOutputInfo { - .hash = { - .method = FileIngestionMethod::Recursive, - .hash = narHash, - }, + .method = FileIngestionMethod::Recursive, + .hash = narHash, .references = { .others = std::move(references), // profiles never refer to themselves From 3b3822ea1ddf34f8fcb44c67cda2b778b5b41a34 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 7 Jul 2023 15:08:25 +0200 Subject: [PATCH 089/909] tests: Reformat exit code error message Now looks like: Expected exit code '123' but got '0' from command 'echo' 'hi' --- tests/common/vars-and-functions.sh.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/common/vars-and-functions.sh.in b/tests/common/vars-and-functions.sh.in index a9e6c802f..33a764c3f 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/common/vars-and-functions.sh.in @@ -195,7 +195,7 @@ expect() { shift "$@" && res=0 || res="$?" if [[ $res -ne $expected ]]; then - echo "Expected '$expected' but got '$res' while running '${*@Q}'" >&2 + echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 return 1 fi return 0 @@ -209,7 +209,7 @@ expectStderr() { shift "$@" 2>&1 && res=0 || res="$?" if [[ $res -ne $expected ]]; then - echo "Expected '$expected' but got '$res' while running '${*@Q}'" >&2 + echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 return 1 fi return 0 From 9fc82de49388a58240234d10893673566432f7ab Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 7 Jul 2023 15:07:30 +0200 Subject: [PATCH 090/909] signing.sh: Revert test improvement because it fails on GHA + macOS --- tests/signing.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/signing.sh b/tests/signing.sh index f14b53e7f..942b51630 100644 --- a/tests/signing.sh +++ b/tests/signing.sh @@ -84,7 +84,11 @@ info=$(nix path-info --store file://$cacheDir --json $outPath2) # Copying to a diverted store should fail due to a lack of signatures by trusted keys. chmod -R u+w $TEST_ROOT/store0 || true rm -rf $TEST_ROOT/store0 -expectStderr 1 nix copy --to $TEST_ROOT/store0 $outPath | grepQuiet -E 'cannot add path .* because it lacks a signature by a trusted key' + +# Fails or very flaky only on GHA + macOS: +# expectStderr 1 nix copy --to $TEST_ROOT/store0 $outPath | grepQuiet -E 'cannot add path .* because it lacks a signature by a trusted key' +# but this works: +(! nix copy --to $TEST_ROOT/store0 $outPath) # But succeed if we supply the public keys. nix copy --to $TEST_ROOT/store0 $outPath --trusted-public-keys $pk1 From d76bf29c5f55f63f10741290c8d415c880a80ac2 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Fri, 7 Jul 2023 07:52:16 -0700 Subject: [PATCH 091/909] Choose a reasonable number similar to LimitNOFile --- misc/systemd/nix-daemon.service.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in index 4b4474475..45fbea02c 100644 --- a/misc/systemd/nix-daemon.service.in +++ b/misc/systemd/nix-daemon.service.in @@ -10,7 +10,7 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket ExecStart=@@bindir@/nix-daemon nix-daemon --daemon KillMode=process LimitNOFILE=1048576 -TasksMax=infinity +TasksMax=1048576 [Install] WantedBy=multi-user.target From 87dcd090470ed6e56a2744cbe1490d2cf235d5c0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 23 Jun 2023 12:31:09 -0400 Subject: [PATCH 092/909] Clean up `resolveSearchPathElem` We should use `std::optional` not `std::pair` for an optional string. --- src/libexpr/eval.cc | 14 +++++++------- src/libexpr/eval.hh | 8 ++++++-- src/libexpr/parser.y | 45 +++++++++++++++++++++++--------------------- 3 files changed, 37 insertions(+), 30 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 706a19024..97a264085 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -571,22 +571,22 @@ EvalState::EvalState( allowedPaths = PathSet(); for (auto & i : searchPath) { - auto r = resolveSearchPathElem(i); - if (!r.first) continue; + auto r = resolveSearchPathElem(i.path); + if (!r) continue; - auto path = r.second; + auto path = *std::move(r); - if (store->isInStore(r.second)) { + if (store->isInStore(path)) { try { StorePathSet closure; - store->computeFSClosure(store->toStorePath(r.second).first, closure); + store->computeFSClosure(store->toStorePath(path).first, closure); for (auto & path : closure) allowPath(path); } catch (InvalidPath &) { - allowPath(r.second); + allowPath(path); } } else - allowPath(r.second); + allowPath(path); } } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e3676c1b7..e1a540a7f 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -317,7 +317,7 @@ private: SearchPath searchPath; - std::map> searchPathResolved; + std::map> searchPathResolved; /** * Cache used by checkSourcePath(). @@ -434,9 +434,13 @@ public: SourcePath findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); /** + * Try to resolve a search path value (not the optinal key part) + * * If the specified search path element is a URI, download it. + * + * If it is not found, return `std::nullopt` */ - std::pair resolveSearchPathElem(const SearchPathElem & elem); + std::optional resolveSearchPathElem(const std::string & value); /** * Evaluate an expression to normal form diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 0d0004f9f..f839e804c 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -772,9 +772,9 @@ SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view p continue; suffix = path.size() == s ? "" : concatStrings("/", path.substr(s)); } - auto r = resolveSearchPathElem(i); - if (!r.first) continue; - Path res = r.second + suffix; + auto r = resolveSearchPathElem(i.path); + if (!r) continue; + Path res = *r + suffix; if (pathExists(res)) return CanonPath(canonPath(res)); } @@ -791,49 +791,52 @@ SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view p } -std::pair EvalState::resolveSearchPathElem(const SearchPathElem & elem) +std::optional EvalState::resolveSearchPathElem(const std::string & value) { - auto i = searchPathResolved.find(elem.path); + auto i = searchPathResolved.find(value); if (i != searchPathResolved.end()) return i->second; - std::pair res; + std::optional res; - if (EvalSettings::isPseudoUrl(elem.path)) { + if (EvalSettings::isPseudoUrl(value)) { try { auto storePath = fetchers::downloadTarball( - store, EvalSettings::resolvePseudoUrl(elem.path), "source", false).tree.storePath; - res = { true, store->toRealPath(storePath) }; + store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath; + res = { store->toRealPath(storePath) }; } catch (FileTransferError & e) { logWarning({ - .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.path) + .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) }); - res = { false, "" }; + res = std::nullopt; } } - else if (hasPrefix(elem.path, "flake:")) { + else if (hasPrefix(value, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(elem.path.substr(6), {}, true, false); - debug("fetching flake search path element '%s''", elem.path); + auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); + debug("fetching flake search path element '%s''", value); auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; - res = { true, store->toRealPath(storePath) }; + res = { store->toRealPath(storePath) }; } else { - auto path = absPath(elem.path); + auto path = absPath(value); if (pathExists(path)) - res = { true, path }; + res = { path }; else { logWarning({ - .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.path) + .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", value) }); - res = { false, "" }; + res = std::nullopt; } } - debug("resolved search path element '%s' to '%s'", elem.path, res.second); + if (res) + debug("resolved search path element '%s' to '%s'", value, *res); + else + debug("failed to resolve search path element '%s'", value); - searchPathResolved[elem.path] = res; + searchPathResolved[value] = res; return res; } From be518e73ae331ac2f46e6b3a0ffdfeead26e3186 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 23 Jun 2023 13:51:25 -0400 Subject: [PATCH 093/909] Clean up `SearchPath` - Better types - Own header / C++ file pair - Test factored out methods - Pass parsed thing around more than strings Co-authored-by: Robert Hensing --- src/libcmd/common-eval-args.cc | 4 +- src/libcmd/common-eval-args.hh | 3 +- src/libcmd/repl.cc | 8 +-- src/libcmd/repl.hh | 2 +- src/libexpr/eval.cc | 12 ++-- src/libexpr/eval.hh | 18 ++---- src/libexpr/parser.y | 47 +++++--------- src/libexpr/primops.cc | 14 ++-- src/libexpr/search-path.cc | 56 ++++++++++++++++ src/libexpr/search-path.hh | 108 +++++++++++++++++++++++++++++++ src/libexpr/tests/search-path.cc | 90 ++++++++++++++++++++++++++ src/nix/upgrade-nix.cc | 2 +- 12 files changed, 300 insertions(+), 64 deletions(-) create mode 100644 src/libexpr/search-path.cc create mode 100644 src/libexpr/search-path.hh create mode 100644 src/libexpr/tests/search-path.cc diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 7f97364a1..3df2c71a5 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -105,7 +105,9 @@ MixEvalArgs::MixEvalArgs() )", .category = category, .labels = {"path"}, - .handler = {[&](std::string s) { searchPath.push_back(s); }} + .handler = {[&](std::string s) { + searchPath.elements.emplace_back(SearchPath::Elem::parse(s)); + }} }); addFlag({ diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index b65cb5b20..6359b2579 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -3,6 +3,7 @@ #include "args.hh" #include "common-args.hh" +#include "search-path.hh" namespace nix { @@ -19,7 +20,7 @@ struct MixEvalArgs : virtual Args, virtual MixRepair Bindings * getAutoArgs(EvalState & state); - Strings searchPath; + SearchPath searchPath; std::optional evalStoreUrl; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 4b160a100..f9e9c2bf8 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -68,7 +68,7 @@ struct NixRepl const Path historyFile; - NixRepl(const Strings & searchPath, nix::ref store,ref state, + NixRepl(const SearchPath & searchPath, nix::ref store,ref state, std::function getValues); virtual ~NixRepl(); @@ -104,7 +104,7 @@ std::string removeWhitespace(std::string s) } -NixRepl::NixRepl(const Strings & searchPath, nix::ref store, ref state, +NixRepl::NixRepl(const SearchPath & searchPath, nix::ref store, ref state, std::function getValues) : AbstractNixRepl(state) , debugTraceIndex(0) @@ -1024,7 +1024,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m std::unique_ptr AbstractNixRepl::create( - const Strings & searchPath, nix::ref store, ref state, + const SearchPath & searchPath, nix::ref store, ref state, std::function getValues) { return std::make_unique( @@ -1044,7 +1044,7 @@ void AbstractNixRepl::runSimple( NixRepl::AnnotatedValues values; return values; }; - const Strings & searchPath = {}; + SearchPath searchPath = {}; auto repl = std::make_unique( searchPath, openStore(), diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index 731c8e6db..6d88883fe 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -25,7 +25,7 @@ struct AbstractNixRepl typedef std::vector> AnnotatedValues; static std::unique_ptr create( - const Strings & searchPath, nix::ref store, ref state, + const SearchPath & searchPath, nix::ref store, ref state, std::function getValues); static void runSimple( diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 97a264085..be1bdb806 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -498,7 +498,7 @@ ErrorBuilder & ErrorBuilder::withFrame(const Env & env, const Expr & expr) EvalState::EvalState( - const Strings & _searchPath, + const SearchPath & _searchPath, ref store, std::shared_ptr buildStore) : sWith(symbols.create("")) @@ -563,15 +563,17 @@ EvalState::EvalState( /* Initialise the Nix expression search path. */ if (!evalSettings.pureEval) { - for (auto & i : _searchPath) addToSearchPath(i); - for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); + for (auto & i : _searchPath.elements) + addToSearchPath(SearchPath::Elem {i}); + for (auto & i : evalSettings.nixPath.get()) + addToSearchPath(SearchPath::Elem::parse(i)); } if (evalSettings.restrictEval || evalSettings.pureEval) { allowedPaths = PathSet(); - for (auto & i : searchPath) { - auto r = resolveSearchPathElem(i.path); + for (auto & i : searchPath.elements) { + auto r = resolveSearchPathPath(i.path); if (!r) continue; auto path = *std::move(r); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e1a540a7f..277e77ad5 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -9,6 +9,7 @@ #include "config.hh" #include "experimental-features.hh" #include "input-accessor.hh" +#include "search-path.hh" #include #include @@ -122,15 +123,6 @@ std::string printValue(const EvalState & state, const Value & v); std::ostream & operator << (std::ostream & os, const ValueType t); -struct SearchPathElem -{ - std::string prefix; - // FIXME: maybe change this to an std::variant. - std::string path; -}; -typedef std::list SearchPath; - - /** * Initialise the Boehm GC, if applicable. */ @@ -344,12 +336,12 @@ private: public: EvalState( - const Strings & _searchPath, + const SearchPath & _searchPath, ref store, std::shared_ptr buildStore = nullptr); ~EvalState(); - void addToSearchPath(const std::string & s); + void addToSearchPath(SearchPath::Elem && elem); SearchPath getSearchPath() { return searchPath; } @@ -431,7 +423,7 @@ public: * Look up a file in the search path. */ SourcePath findFile(const std::string_view path); - SourcePath findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); + SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); /** * Try to resolve a search path value (not the optinal key part) @@ -440,7 +432,7 @@ public: * * If it is not found, return `std::nullopt` */ - std::optional resolveSearchPathElem(const std::string & value); + std::optional resolveSearchPathPath(const SearchPath::Path & path); /** * Evaluate an expression to normal form diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index f839e804c..77bcc31e0 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -734,22 +734,9 @@ Expr * EvalState::parseStdin() } -void EvalState::addToSearchPath(const std::string & s) +void EvalState::addToSearchPath(SearchPath::Elem && elem) { - size_t pos = s.find('='); - std::string prefix; - Path path; - if (pos == std::string::npos) { - path = s; - } else { - prefix = std::string(s, 0, pos); - path = std::string(s, pos + 1); - } - - searchPath.emplace_back(SearchPathElem { - .prefix = prefix, - .path = path, - }); + searchPath.elements.emplace_back(std::move(elem)); } @@ -759,22 +746,19 @@ SourcePath EvalState::findFile(const std::string_view path) } -SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos) +SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos) { - for (auto & i : searchPath) { - std::string suffix; - if (i.prefix.empty()) - suffix = concatStrings("/", path); - else { - auto s = i.prefix.size(); - if (path.compare(0, s, i.prefix) != 0 || - (path.size() > s && path[s] != '/')) - continue; - suffix = path.size() == s ? "" : concatStrings("/", path.substr(s)); - } - auto r = resolveSearchPathElem(i.path); - if (!r) continue; - Path res = *r + suffix; + for (auto & i : searchPath.elements) { + auto suffixOpt = i.prefix.suffixIfPotentialMatch(path); + + if (!suffixOpt) continue; + auto suffix = *suffixOpt; + + auto rOpt = resolveSearchPathPath(i.path); + if (!rOpt) continue; + auto r = *rOpt; + + Path res = suffix == "" ? r : concatStrings(r, "/", suffix); if (pathExists(res)) return CanonPath(canonPath(res)); } @@ -791,8 +775,9 @@ SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view p } -std::optional EvalState::resolveSearchPathElem(const std::string & value) +std::optional EvalState::resolveSearchPathPath(const SearchPath::Path & value0) { + auto & value = value0.s; auto i = searchPathResolved.find(value); if (i != searchPathResolved.end()) return i->second; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5dfad470a..b98b06db9 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1656,9 +1656,9 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V })); } - searchPath.emplace_back(SearchPathElem { - .prefix = prefix, - .path = path, + searchPath.elements.emplace_back(SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = prefix }, + .path = SearchPath::Path { .s = path }, }); } @@ -4319,12 +4319,12 @@ void EvalState::createBaseEnv() }); /* Add a value containing the current Nix expression search path. */ - mkList(v, searchPath.size()); + mkList(v, searchPath.elements.size()); int n = 0; - for (auto & i : searchPath) { + for (auto & i : searchPath.elements) { auto attrs = buildBindings(2); - attrs.alloc("path").mkString(i.path); - attrs.alloc("prefix").mkString(i.prefix); + attrs.alloc("path").mkString(i.path.s); + attrs.alloc("prefix").mkString(i.prefix.s); (v.listElems()[n++] = allocValue())->mkAttrs(attrs); } addConstant("__nixPath", v, { diff --git a/src/libexpr/search-path.cc b/src/libexpr/search-path.cc new file mode 100644 index 000000000..36bb4c3a5 --- /dev/null +++ b/src/libexpr/search-path.cc @@ -0,0 +1,56 @@ +#include "search-path.hh" +#include "util.hh" + +namespace nix { + +std::optional SearchPath::Prefix::suffixIfPotentialMatch( + std::string_view path) const +{ + auto n = s.size(); + + /* Non-empty prefix and suffix must be separated by a /, or the + prefix is not a valid path prefix. */ + bool needSeparator = n > 0 && (path.size() - n) > 0; + + if (needSeparator && path[n] != '/') { + return std::nullopt; + } + + /* Prefix must be prefix of this path. */ + if (path.compare(0, n, s) != 0) { + return std::nullopt; + } + + /* Skip next path separator. */ + return { + path.substr(needSeparator ? n + 1 : n) + }; +} + + +SearchPath::Elem SearchPath::Elem::parse(std::string_view rawElem) +{ + size_t pos = rawElem.find('='); + + return SearchPath::Elem { + .prefix = Prefix { + .s = pos == std::string::npos + ? std::string { "" } + : std::string { rawElem.substr(0, pos) }, + }, + .path = Path { + .s = std::string { rawElem.substr(pos + 1) }, + }, + }; +} + + +SearchPath parseSearchPath(const Strings & rawElems) +{ + SearchPath res; + for (auto & rawElem : rawElems) + res.elements.emplace_back(SearchPath::Elem::parse(rawElem)); + return res; +} + +} diff --git a/src/libexpr/search-path.hh b/src/libexpr/search-path.hh new file mode 100644 index 000000000..ce78135b5 --- /dev/null +++ b/src/libexpr/search-path.hh @@ -0,0 +1,108 @@ +#pragma once +///@file + +#include + +#include "types.hh" +#include "comparator.hh" + +namespace nix { + +/** + * A "search path" is a list of ways look for something, used with + * `builtins.findFile` and `< >` lookup expressions. + */ +struct SearchPath +{ + /** + * A single element of a `SearchPath`. + * + * Each element is tried in succession when looking up a path. The first + * element to completely match wins. + */ + struct Elem; + + /** + * The first part of a `SearchPath::Elem` pair. + * + * Called a "prefix" because it takes the form of a prefix of a file + * path (first `n` path components). When looking up a path, to use + * a `SearchPath::Elem`, its `Prefix` must match the path. + */ + struct Prefix; + + /** + * The second part of a `SearchPath::Elem` pair. + * + * It is either a path or a URL (with certain restrictions / extra + * structure). + * + * If the prefix of the path we are looking up matches, we then + * check if the rest of the path points to something that exists + * within the directory denoted by this. If so, the + * `SearchPath::Elem` as a whole matches, and that *something* being + * pointed to by the rest of the path we are looking up is the + * result. + */ + struct Path; + + /** + * The list of search path elements. Each one is checked for a path + * when looking up. (The actual lookup entry point is in `EvalState` + * not in this class.) + */ + std::list elements; + + /** + * Parse a string into a `SearchPath` + */ + static SearchPath parse(const Strings & rawElems); +}; + +struct SearchPath::Prefix +{ + /** + * Underlying string + * + * @todo Should we normalize this when constructing a `SearchPath::Prefix`? + */ + std::string s; + + GENERATE_CMP(SearchPath::Prefix, me->s); + + /** + * If the path possibly matches this search path element, return the + * suffix that we should look for inside the resolved value of the + * element + * Note the double optionality in the name. While we might have a matching prefix, the suffix may not exist. + */ + std::optional suffixIfPotentialMatch(std::string_view path) const; +}; + +struct SearchPath::Path +{ + /** + * The location of a search path item, as a path or URL. + * + * @todo Maybe change this to `std::variant`. + */ + std::string s; + + GENERATE_CMP(SearchPath::Path, me->s); +}; + +struct SearchPath::Elem +{ + + Prefix prefix; + Path path; + + GENERATE_CMP(SearchPath::Elem, me->prefix, me->path); + + /** + * Parse a string into a `SearchPath::Elem` + */ + static SearchPath::Elem parse(std::string_view rawElem); +}; + +} diff --git a/src/libexpr/tests/search-path.cc b/src/libexpr/tests/search-path.cc new file mode 100644 index 000000000..dbe7ab95f --- /dev/null +++ b/src/libexpr/tests/search-path.cc @@ -0,0 +1,90 @@ +#include +#include + +#include "search-path.hh" + +namespace nix { + +TEST(SearchPathElem, parse_justPath) { + ASSERT_EQ( + SearchPath::Elem::parse("foo"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "" }, + .path = SearchPath::Path { .s = "foo" }, + })); +} + +TEST(SearchPathElem, parse_emptyPrefix) { + ASSERT_EQ( + SearchPath::Elem::parse("=foo"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "" }, + .path = SearchPath::Path { .s = "foo" }, + })); +} + +TEST(SearchPathElem, parse_oneEq) { + ASSERT_EQ( + SearchPath::Elem::parse("foo=bar"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "foo" }, + .path = SearchPath::Path { .s = "bar" }, + })); +} + +TEST(SearchPathElem, parse_twoEqs) { + ASSERT_EQ( + SearchPath::Elem::parse("foo=bar=baz"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "foo" }, + .path = SearchPath::Path { .s = "bar=baz" }, + })); +} + + +TEST(SearchPathElem, suffixIfPotentialMatch_justPath) { + SearchPath::Prefix prefix { .s = "" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("any/thing"), std::optional { "any/thing" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_misleadingPrefix1) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX"), std::nullopt); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_misleadingPrefix2) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX/bar"), std::nullopt); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_partialPrefix) { + SearchPath::Prefix prefix { .s = "fooX" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::nullopt); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_exactPrefix) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::optional { "" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_multiKey) { + SearchPath::Prefix prefix { .s = "foo/bar" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "baz" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_trailingSlash) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/"), std::optional { "" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_trailingDoubleSlash) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo//"), std::optional { "/" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_trailingPath) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "bar/baz" }); +} + +} diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 3997c98bf..d05c23fb7 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -146,7 +146,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand auto req = FileTransferRequest(storePathsUrl); auto res = getFileTransfer()->download(req); - auto state = std::make_unique(Strings(), store); + auto state = std::make_unique(SearchPath{}, store); auto v = state->allocValue(); state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v); Bindings & bindings(*state->allocBindings(0)); From 3d74e7b811ea8fc13e31127175be12ba8d39351c Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Mon, 10 Jul 2023 12:02:29 +0800 Subject: [PATCH 094/909] libexpr: remove std::move() for `basePath` in parser, it has no effect --- src/libexpr/parser.y | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 0d0004f9f..3336d3315 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -663,7 +663,7 @@ Expr * EvalState::parse( ParseData data { .state = *this, .symbols = symbols, - .basePath = std::move(basePath), + .basePath = basePath, .origin = {origin}, }; From 3fa0266e7a736e85e2faaa6c92ab23e809f89930 Mon Sep 17 00:00:00 2001 From: Bader AlAttar <68021685+balattar@users.noreply.github.com> Date: Mon, 10 Jul 2023 02:33:04 -0700 Subject: [PATCH 095/909] Fix some grammar in installables doc (#8682) --- src/nix/nix.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/nix.md b/src/nix/nix.md index 6d9e40dbc..e0f459d6b 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -63,7 +63,7 @@ The following types of installable are supported by most commands: - [Nix file](#nix-file), optionally qualified by an attribute path - [Nix expression](#nix-expression), optionally qualified by an attribute path -For most commands, if no installable is specified, `.` as assumed. +For most commands, if no installable is specified, `.` is assumed. That is, Nix will operate on the default flake output attribute of the flake in the current directory. ### Flake output attribute From c2c8187118c4bf2961aa175ff8667e1402a767c7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 21 Jun 2023 23:15:24 -0400 Subject: [PATCH 096/909] Fix test file name It's UTF-8, not UFT-8. --- tests/lang/{parse-fail-uft8.nix => parse-fail-utf8.nix} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/lang/{parse-fail-uft8.nix => parse-fail-utf8.nix} (100%) diff --git a/tests/lang/parse-fail-uft8.nix b/tests/lang/parse-fail-utf8.nix similarity index 100% rename from tests/lang/parse-fail-uft8.nix rename to tests/lang/parse-fail-utf8.nix From 07dabcc90ed8f2a2e7b98d858a47de3e75d2c3a2 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 11 Jul 2023 10:44:03 +0100 Subject: [PATCH 097/909] Always attempt setgroups but allow failure to be ignored. --- src/libstore/build/local-derivation-goal.cc | 9 ++++++--- src/libstore/globals.hh | 2 +- tests/supplementary-groups.sh | 8 ++++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 53e6998e8..068b47f93 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -909,9 +909,12 @@ void LocalDerivationGoal::startBuilder() /* Drop additional groups here because we can't do it after we've created the new user namespace. */ - if (settings.dropSupplementaryGroups) - if (setgroups(0, 0) == -1) - throw SysError("setgroups failed. Set the drop-supplementary-groups option to false to skip this step."); + if (setgroups(0, 0) == -1) { + if (errno != EPERM) + throw SysError("setgroups failed"); + if (settings.requireDropSupplementaryGroups) + throw Error("setgroups failed. Set the require-drop-supplementary-groups option to false to skip this step."); + } ProcessOptions options; options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index a19b43086..dbabf116a 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -524,7 +524,7 @@ public: Setting sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; - Setting dropSupplementaryGroups{this, getuid() == 0, "drop-supplementary-groups", + Setting requireDropSupplementaryGroups{this, true, "require-drop-supplementary-groups", R"( Whether to drop supplementary groups when building with sandboxing. This is normally a good idea if we are root and have the capability to diff --git a/tests/supplementary-groups.sh b/tests/supplementary-groups.sh index 47debc5e3..47c6ef605 100644 --- a/tests/supplementary-groups.sh +++ b/tests/supplementary-groups.sh @@ -20,14 +20,14 @@ unshare --mount --map-root-user bash < Date: Tue, 11 Jul 2023 10:57:03 +0100 Subject: [PATCH 098/909] Update description for require-drop-supplementary-groups. --- src/libstore/globals.hh | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index dbabf116a..601626d00 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -526,21 +526,15 @@ public: Setting requireDropSupplementaryGroups{this, true, "require-drop-supplementary-groups", R"( - Whether to drop supplementary groups when building with sandboxing. - This is normally a good idea if we are root and have the capability to - do so. + Following the principle of least privilege, + Nix will attempt to drop supplementary groups when building with sandboxing. - But if this "root" is mapped from a non-root user in a larger - namespace, we won't be able drop additional groups; they will be - mapped to nogroup in the child namespace. There does not seem to be a - workaround for this. + However this can fail under some circumstances. + For example, if the user lacks the CAP_SETGID capability. + Search setgroups(2) for EPERM to find more detailed information on this. - (But who can tell from reading user_namespaces(7)? See also https://lwn.net/Articles/621612/.) - - TODO: It might be good to create a middle ground option that allows - `setgroups` to fail if all additional groups are "nogroup" / the value - of `/proc/sys/fs/overflowuid`. This would handle the common - nested-sandboxing case identified above. + If you encounter such a failure, + you can instruct Nix to continue without dropping supplementary groups by setting this option to `false`. )"}; #if __linux__ From 2b4c59dd997c72069b6039783fea4c3b35f5cee7 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 11 Jul 2023 11:09:25 +0100 Subject: [PATCH 099/909] Be clearer about the security implications. --- src/libstore/globals.hh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 601626d00..dec132ff0 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -533,8 +533,9 @@ public: For example, if the user lacks the CAP_SETGID capability. Search setgroups(2) for EPERM to find more detailed information on this. - If you encounter such a failure, - you can instruct Nix to continue without dropping supplementary groups by setting this option to `false`. + If you encounter such a failure, setting this option to `false` will let you ignore it and continue. + But before doing so, you should consider the security implications carefully. + Not dropping supplementary groups means the build sandbox will be less restricted than intended. )"}; #if __linux__ From a193ec4052d9efa895681c438cc335296c7affea Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 11 Jul 2023 11:13:39 +0100 Subject: [PATCH 100/909] Default should depend on whether we are root. --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index dec132ff0..9a9b4903f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -524,7 +524,7 @@ public: Setting sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; - Setting requireDropSupplementaryGroups{this, true, "require-drop-supplementary-groups", + Setting requireDropSupplementaryGroups{this, getuid() == 0, "require-drop-supplementary-groups", R"( Following the principle of least privilege, Nix will attempt to drop supplementary groups when building with sandboxing. From b8e8dfc3e8581a08bc179f75fbe61e04030088de Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 11 Jul 2023 11:24:11 +0100 Subject: [PATCH 101/909] Say a bit about default value in setting description. --- src/libstore/globals.hh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 9a9b4903f..81fa154bb 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -536,6 +536,10 @@ public: If you encounter such a failure, setting this option to `false` will let you ignore it and continue. But before doing so, you should consider the security implications carefully. Not dropping supplementary groups means the build sandbox will be less restricted than intended. + + This option defaults to `true` when the user is root + (since root usually has permissions to call setgroups) + and `false` otherwise. )"}; #if __linux__ From c1d39de1fbceebbb6b46abc4a21fb8e34423df99 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 15 Jun 2023 12:48:06 +0100 Subject: [PATCH 102/909] Check _NIX_TEST_NO_SANDBOX when setting _canUseSandbox. --- tests/common/vars-and-functions.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common/vars-and-functions.sh.in b/tests/common/vars-and-functions.sh.in index dc7ce13cc..ad0623871 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/common/vars-and-functions.sh.in @@ -141,7 +141,7 @@ restartDaemon() { startDaemon } -if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then +if [[ -z "${_NIX_TEST_NO_SANDBOX:-}" ]] && [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then _canUseSandbox=1 fi From 41412dc4ae0fec2d08335c19724276d99e0c6056 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 15 Jun 2023 12:58:59 +0100 Subject: [PATCH 103/909] Skip build-remote-trustless unless sandbox is supported. --- tests/build-remote-trustless-should-fail-0.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/build-remote-trustless-should-fail-0.sh b/tests/build-remote-trustless-should-fail-0.sh index fad1def59..b14101f83 100644 --- a/tests/build-remote-trustless-should-fail-0.sh +++ b/tests/build-remote-trustless-should-fail-0.sh @@ -2,6 +2,7 @@ source common.sh enableFeatures "daemon-trust-override" +requireSandboxSupport restartDaemon [[ $busybox =~ busybox ]] || skipTest "no busybox" From c70484454f52a3bba994ac0c155b4a6f35a5013c Mon Sep 17 00:00:00 2001 From: Mathnerd314 Date: Fri, 4 Sep 2015 14:23:08 -0600 Subject: [PATCH 104/909] Expanded test suite * Lang now verifies errors and parse output * Some new miscellaneous tests * Easy way to update the tests * Document workflow in manual * Use `!` not `~` as separater char for sed It is confusing to use `~` when we are talking about paths and home directories! * Test test suite itself (`test/lang-test/infra.sh`) Additionally, run shellcheck on `tests/lang.sh` to help ensure it is correct, now that is is more complex. Co-authored-by: Robert Hensing Co-authored-by: Valentin Gagarin --- .gitignore | 1 + doc/manual/src/contributing/testing.md | 25 ++++ tests/dependencies.sh | 3 + tests/lang-test-infra.sh | 86 ++++++++++++ tests/lang.sh | 123 +++++++++++++----- tests/lang/empty.exp | 0 tests/lang/eval-fail-abort.err.exp | 10 ++ tests/lang/eval-fail-antiquoted-path.err.exp | 1 + tests/lang/eval-fail-assert.err.exp | 36 +++++ tests/lang/eval-fail-bad-antiquote-1.err.exp | 10 ++ tests/lang/eval-fail-bad-antiquote-2.err.exp | 1 + tests/lang/eval-fail-bad-antiquote-3.err.exp | 10 ++ ...al-fail-bad-string-interpolation-1.err.exp | 10 ++ ...al-fail-bad-string-interpolation-2.err.exp | 1 + ...al-fail-bad-string-interpolation-3.err.exp | 10 ++ tests/lang/eval-fail-blackhole.err.exp | 18 +++ tests/lang/eval-fail-deepseq.err.exp | 26 ++++ ...-foldlStrict-strict-op-application.err.exp | 38 ++++++ .../eval-fail-fromTOML-timestamps.err.exp | 12 ++ tests/lang/eval-fail-hashfile-missing.err.exp | 19 +++ tests/lang/eval-fail-list.err.exp | 10 ++ tests/lang/eval-fail-list.nix | 1 + tests/lang/eval-fail-missing-arg.err.exp | 16 +++ tests/lang/eval-fail-nonexist-path.err.exp | 1 + tests/lang/eval-fail-path-slash.err.exp | 8 ++ tests/lang/eval-fail-recursion.err.exp | 16 +++ tests/lang/eval-fail-recursion.nix | 1 + tests/lang/eval-fail-remove.err.exp | 19 +++ tests/lang/eval-fail-scope-5.err.exp | 36 +++++ tests/lang/eval-fail-seq.err.exp | 18 +++ tests/lang/eval-fail-set-override.err.exp | 6 + tests/lang/eval-fail-set-override.nix | 1 + tests/lang/eval-fail-set.err.exp | 7 + tests/lang/eval-fail-set.nix | 1 + tests/lang/eval-fail-substring.err.exp | 12 ++ tests/lang/eval-fail-to-path.err.exp | 14 ++ tests/lang/eval-fail-undeclared-arg.err.exp | 17 +++ tests/lang/eval-okay-fromjson.nix | 14 +- tests/lang/eval-okay-overrides.nix | 2 +- tests/lang/eval-okay-print.err.exp | 1 + tests/lang/eval-okay-print.exp | 1 + tests/lang/eval-okay-print.nix | 1 + tests/lang/eval-okay-search-path.flags | 2 +- tests/lang/framework.sh | 33 +++++ tests/lang/parse-fail-dup-attrs-1.err.exp | 7 + tests/lang/parse-fail-dup-attrs-2.err.exp | 7 + tests/lang/parse-fail-dup-attrs-3.err.exp | 7 + tests/lang/parse-fail-dup-attrs-4.err.exp | 7 + tests/lang/parse-fail-dup-attrs-6.err.exp | 1 + tests/lang/parse-fail-dup-attrs-7.err.exp | 7 + tests/lang/parse-fail-dup-formals.err.exp | 6 + tests/lang/parse-fail-eof-in-string.err.exp | 7 + .../parse-fail-mixed-nested-attrs1.err.exp | 8 ++ .../parse-fail-mixed-nested-attrs2.err.exp | 8 ++ tests/lang/parse-fail-patterns-1.err.exp | 7 + .../parse-fail-regression-20060610.err.exp | 8 ++ tests/lang/parse-fail-undef-var-2.err.exp | 7 + tests/lang/parse-fail-undef-var.err.exp | 7 + tests/lang/parse-fail-utf8.err.exp | 6 + tests/lang/parse-okay-1.exp | 1 + tests/lang/parse-okay-crlf.exp | 1 + tests/lang/parse-okay-dup-attrs-5.exp | 1 + tests/lang/parse-okay-dup-attrs-6.exp | 1 + .../lang/parse-okay-mixed-nested-attrs-1.exp | 1 + .../lang/parse-okay-mixed-nested-attrs-2.exp | 1 + .../lang/parse-okay-mixed-nested-attrs-3.exp | 1 + tests/lang/parse-okay-regression-20041027.exp | 1 + tests/lang/parse-okay-regression-751.exp | 1 + tests/lang/parse-okay-subversion.exp | 1 + tests/lang/parse-okay-url.exp | 1 + tests/local.mk | 1 + tests/misc.sh | 6 + tests/restricted.sh | 2 + 73 files changed, 762 insertions(+), 36 deletions(-) create mode 100644 tests/lang-test-infra.sh mode change 100644 => 100755 tests/lang.sh create mode 100644 tests/lang/empty.exp create mode 100644 tests/lang/eval-fail-abort.err.exp create mode 100644 tests/lang/eval-fail-antiquoted-path.err.exp create mode 100644 tests/lang/eval-fail-assert.err.exp create mode 100644 tests/lang/eval-fail-bad-antiquote-1.err.exp create mode 100644 tests/lang/eval-fail-bad-antiquote-2.err.exp create mode 100644 tests/lang/eval-fail-bad-antiquote-3.err.exp create mode 100644 tests/lang/eval-fail-bad-string-interpolation-1.err.exp create mode 100644 tests/lang/eval-fail-bad-string-interpolation-2.err.exp create mode 100644 tests/lang/eval-fail-bad-string-interpolation-3.err.exp create mode 100644 tests/lang/eval-fail-blackhole.err.exp create mode 100644 tests/lang/eval-fail-deepseq.err.exp create mode 100644 tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp create mode 100644 tests/lang/eval-fail-fromTOML-timestamps.err.exp create mode 100644 tests/lang/eval-fail-hashfile-missing.err.exp create mode 100644 tests/lang/eval-fail-list.err.exp create mode 100644 tests/lang/eval-fail-list.nix create mode 100644 tests/lang/eval-fail-missing-arg.err.exp create mode 100644 tests/lang/eval-fail-nonexist-path.err.exp create mode 100644 tests/lang/eval-fail-path-slash.err.exp create mode 100644 tests/lang/eval-fail-recursion.err.exp create mode 100644 tests/lang/eval-fail-recursion.nix create mode 100644 tests/lang/eval-fail-remove.err.exp create mode 100644 tests/lang/eval-fail-scope-5.err.exp create mode 100644 tests/lang/eval-fail-seq.err.exp create mode 100644 tests/lang/eval-fail-set-override.err.exp create mode 100644 tests/lang/eval-fail-set-override.nix create mode 100644 tests/lang/eval-fail-set.err.exp create mode 100644 tests/lang/eval-fail-set.nix create mode 100644 tests/lang/eval-fail-substring.err.exp create mode 100644 tests/lang/eval-fail-to-path.err.exp create mode 100644 tests/lang/eval-fail-undeclared-arg.err.exp create mode 100644 tests/lang/eval-okay-print.err.exp create mode 100644 tests/lang/eval-okay-print.exp create mode 100644 tests/lang/eval-okay-print.nix create mode 100644 tests/lang/framework.sh create mode 100644 tests/lang/parse-fail-dup-attrs-1.err.exp create mode 100644 tests/lang/parse-fail-dup-attrs-2.err.exp create mode 100644 tests/lang/parse-fail-dup-attrs-3.err.exp create mode 100644 tests/lang/parse-fail-dup-attrs-4.err.exp create mode 100644 tests/lang/parse-fail-dup-attrs-6.err.exp create mode 100644 tests/lang/parse-fail-dup-attrs-7.err.exp create mode 100644 tests/lang/parse-fail-dup-formals.err.exp create mode 100644 tests/lang/parse-fail-eof-in-string.err.exp create mode 100644 tests/lang/parse-fail-mixed-nested-attrs1.err.exp create mode 100644 tests/lang/parse-fail-mixed-nested-attrs2.err.exp create mode 100644 tests/lang/parse-fail-patterns-1.err.exp create mode 100644 tests/lang/parse-fail-regression-20060610.err.exp create mode 100644 tests/lang/parse-fail-undef-var-2.err.exp create mode 100644 tests/lang/parse-fail-undef-var.err.exp create mode 100644 tests/lang/parse-fail-utf8.err.exp create mode 100644 tests/lang/parse-okay-1.exp create mode 100644 tests/lang/parse-okay-crlf.exp create mode 100644 tests/lang/parse-okay-dup-attrs-5.exp create mode 100644 tests/lang/parse-okay-dup-attrs-6.exp create mode 100644 tests/lang/parse-okay-mixed-nested-attrs-1.exp create mode 100644 tests/lang/parse-okay-mixed-nested-attrs-2.exp create mode 100644 tests/lang/parse-okay-mixed-nested-attrs-3.exp create mode 100644 tests/lang/parse-okay-regression-20041027.exp create mode 100644 tests/lang/parse-okay-regression-751.exp create mode 100644 tests/lang/parse-okay-subversion.exp create mode 100644 tests/lang/parse-okay-url.exp diff --git a/.gitignore b/.gitignore index 969194650..93a9ff9ae 100644 --- a/.gitignore +++ b/.gitignore @@ -95,6 +95,7 @@ perl/Makefile.config # /tests/lang/ /tests/lang/*.out /tests/lang/*.out.xml +/tests/lang/*.err /tests/lang/*.ast /perl/lib/Nix/Config.pm diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index e5f80a928..a5253997d 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -86,6 +86,31 @@ GNU gdb (GDB) 12.1 One can debug the Nix invocation in all the usual ways. For example, enter `run` to start the Nix invocation. +### Characterization testing + +Occasionally, Nix utilizes a technique called [Characterization Testing](https://en.wikipedia.org/wiki/Characterization_test) as part of the functional tests. +This technique is to include the exact output/behavior of a former version of Nix in a test in order to check that Nix continues to produce the same behavior going forward. + +For example, this technique is used for the language tests, to check both the printed final value if evaluation was successful, and any errors and warnings encountered. + +It is frequently useful to regenerate the expected output. +To do that, rerun the failed test with `_NIX_TEST_ACCEPT=1`. +(At least, this is the convention we've used for `tests/lang.sh`. +If we add more characterization testing we should always strive to be consistent.) + +An interesting situation to document is the case when these tests are "overfitted". +The language tests are, again, an example of this. +The expected successful output of evaluation is supposed to be highly stable – we do not intend to make breaking changes to (the stable parts of) the Nix language. +However, the errors and warnings during evaluation (successful or not) are not stable in this way. +We are free to change how they are displayed at any time. + +It may be surprising that we would test non-normative behavior like diagnostic outputs. +Diagnostic outputs are indeed not a stable interface, but they still are important to users. +By recording the expected output, the test suite guards against accidental changes, and ensure the *result* (not just the code that implements it) of the diagnostic code paths are under code review. +Regressions are caught, and improvements always show up in code review. + +To ensure that characterization testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. + ## Integration tests The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. diff --git a/tests/dependencies.sh b/tests/dependencies.sh index f9da0c6bc..d5cd30396 100644 --- a/tests/dependencies.sh +++ b/tests/dependencies.sh @@ -15,6 +15,9 @@ if test -n "$dot"; then $dot < $TEST_ROOT/graph fi +# Test GraphML graph generation +nix-store -q --graphml "$drvPath" > $TEST_ROOT/graphml + outPath=$(nix-store -rvv "$drvPath") || fail "build failed" # Test Graphviz graph generation. diff --git a/tests/lang-test-infra.sh b/tests/lang-test-infra.sh new file mode 100644 index 000000000..30da8977b --- /dev/null +++ b/tests/lang-test-infra.sh @@ -0,0 +1,86 @@ +# Test the function for lang.sh +source common.sh + +source lang/framework.sh + +# We are testing this, so don't want outside world to affect us. +unset _NIX_TEST_ACCEPT + +# We'll only modify this in subshells so we don't need to reset it. +badDiff=0 + +# matches non-empty +echo Hi! > "$TEST_ROOT/got" +cp "$TEST_ROOT/got" "$TEST_ROOT/expected" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) + +# matches empty, non-existant file is the same as empty file +echo -n > "$TEST_ROOT/got" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exist" + (( "$badDiff" == 0 )) +) + +# doesn't matches non-empty, non-existant file is the same as empty file +echo Hi! > "$TEST_ROOT/got" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exist" + (( "$badDiff" == 1 )) +) + +# doesn't match, `badDiff` set, file unchanged +echo Hi! > "$TEST_ROOT/got" +echo Bye! > "$TEST_ROOT/expected" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 1 )) +) +[[ "$(echo Bye! )" == $(< "$TEST_ROOT/expected") ]] + +# _NIX_TEST_ACCEPT=1 matches non-empty +echo Hi! > "$TEST_ROOT/got" +cp "$TEST_ROOT/got" "$TEST_ROOT/expected" +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) + +# _NIX_TEST_ACCEPT doesn't match, `badDiff=1` set, file changed (was previously non-empty) +echo Hi! > "$TEST_ROOT/got" +echo Bye! > "$TEST_ROOT/expected" +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 1 )) +) +[[ "$(echo Hi! )" == $(< "$TEST_ROOT/expected") ]] +# second time succeeds +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) + +# _NIX_TEST_ACCEPT matches empty, non-existant file not created +echo -n > "$TEST_ROOT/got" +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exists" + (( "$badDiff" == 0 )) +) +[[ ! -f "$TEST_ROOT/does-not-exist" ]] + +# _NIX_TEST_ACCEPT doesn't match, output empty, file deleted +echo -n > "$TEST_ROOT/got" +echo Bye! > "$TEST_ROOT/expected" +badDiff=0 +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 1 )) +) +[[ ! -f "$TEST_ROOT/expected" ]] +# second time succeeds +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) diff --git a/tests/lang.sh b/tests/lang.sh old mode 100644 new mode 100755 index 8170cb39d..bfb942d3d --- a/tests/lang.sh +++ b/tests/lang.sh @@ -1,5 +1,17 @@ source common.sh +set -o pipefail + +source lang/framework.sh + +# specialize function a bit +function diffAndAccept() { + local -r testName="$1" + local -r got="lang/$testName.$2" + local -r expected="lang/$testName.$3" + diffAndAcceptInner "$testName" "$got" "$expected" +} + export TEST_VAR=foo # for eval-okay-getenv.nix export NIX_REMOTE=dummy:// export NIX_STORE_DIR=/nix/store @@ -20,63 +32,114 @@ nix-instantiate --eval -E 'let x = { repeating = x; tracing = builtins.trace x t set +x -fail=0 +badDiff=0 +badExitCode=0 for i in lang/parse-fail-*.nix; do echo "parsing $i (should fail)"; - i=$(basename $i .nix) - if ! expect 1 nix-instantiate --parse - < lang/$i.nix; then + i=$(basename "$i" .nix) + if expectStderr 1 nix-instantiate --parse - < "lang/$i.nix" > "lang/$i.err" + then + diffAndAccept "$i" err err.exp + else echo "FAIL: $i shouldn't parse" - fail=1 + badExitCode=1 fi done for i in lang/parse-okay-*.nix; do echo "parsing $i (should succeed)"; - i=$(basename $i .nix) - if ! expect 0 nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then + i=$(basename "$i" .nix) + if + expect 0 nix-instantiate --parse - < "lang/$i.nix" \ + 1> >(sed "s!$(pwd)!/pwd!g" > "lang/$i.out") \ + 2> >(sed "s!$(pwd)!/pwd!g" > "lang/$i.err") + then + diffAndAccept "$i" out exp + diffAndAccept "$i" err err.exp + else echo "FAIL: $i should parse" - fail=1 + badExitCode=1 fi done for i in lang/eval-fail-*.nix; do echo "evaluating $i (should fail)"; - i=$(basename $i .nix) - if ! expect 1 nix-instantiate --eval lang/$i.nix; then + i=$(basename "$i" .nix) + if + expectStderr 1 nix-instantiate --show-trace "lang/$i.nix" \ + | sed "s!$(pwd)!/pwd!g" > "lang/$i.err" + then + diffAndAccept "$i" err err.exp + else echo "FAIL: $i shouldn't evaluate" - fail=1 + badExitCode=1 fi done for i in lang/eval-okay-*.nix; do echo "evaluating $i (should succeed)"; - i=$(basename $i .nix) + i=$(basename "$i" .nix) - if test -e lang/$i.exp; then - flags= - if test -e lang/$i.flags; then - flags=$(cat lang/$i.flags) - fi - if ! expect 0 env NIX_PATH=lang/dir3:lang/dir4 HOME=/fake-home nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then + if test -e "lang/$i.exp.xml"; then + if expect 0 nix-instantiate --eval --xml --no-location --strict \ + "lang/$i.nix" > "lang/$i.out.xml" + then + diffAndAccept "$i" out.xml exp.xml + else echo "FAIL: $i should evaluate" - fail=1 - elif ! diff <(< lang/$i.out sed -e "s|$(pwd)|/pwd|g") lang/$i.exp; then - echo "FAIL: evaluation result of $i not as expected" - fail=1 + badExitCode=1 + fi + elif test ! -e "lang/$i.exp-disabled"; then + declare -a flags=() + if test -e "lang/$i.flags"; then + read -r -a flags < "lang/$i.flags" fi - fi - if test -e lang/$i.exp.xml; then - if ! expect 0 nix-instantiate --eval --xml --no-location --strict \ - lang/$i.nix > lang/$i.out.xml; then + if + expect 0 env \ + NIX_PATH=lang/dir3:lang/dir4 \ + HOME=/fake-home \ + nix-instantiate "${flags[@]}" --eval --strict "lang/$i.nix" \ + 1> "lang/$i.out" \ + 2> "lang/$i.err" + then + sed -i "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" + diffAndAccept "$i" out exp + diffAndAccept "$i" err err.exp + else echo "FAIL: $i should evaluate" - fail=1 - elif ! cmp -s lang/$i.out.xml lang/$i.exp.xml; then - echo "FAIL: XML evaluation result of $i not as expected" - fail=1 + badExitCode=1 fi fi done -exit $fail +if test -n "${_NIX_TEST_ACCEPT-}"; then + if (( "$badDiff" )); then + echo 'Output did mot match, but accepted output as the persisted expected output.' + echo 'That means the next time the tests are run, they should pass.' + else + echo 'NOTE: Environment variable _NIX_TEST_ACCEPT is defined,' + echo 'indicating the unexpected output should be accepted as the expected output going forward,' + echo 'but no tests had unexpected output so there was no expected output to update.' + fi + if (( "$badExitCode" )); then + exit "$badExitCode" + else + skipTest "regenerating golden masters" + fi +else + if (( "$badDiff" )); then + echo '' + echo 'You can rerun this test with:' + echo '' + echo ' _NIX_TEST_ACCEPT=1 make tests/lang.sh.test' + echo '' + echo 'to regenerate the files containing the expected output,' + echo 'and then view the git diff to decide whether a change is' + echo 'good/intentional or bad/unintentional.' + echo 'If the diff contains arbitrary or impure information,' + echo 'please improve the normalization that the test applies to the output.' + fi + exit $(( "$badExitCode" + "$badDiff" )) +fi diff --git a/tests/lang/empty.exp b/tests/lang/empty.exp new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lang/eval-fail-abort.err.exp b/tests/lang/eval-fail-abort.err.exp new file mode 100644 index 000000000..345232d3f --- /dev/null +++ b/tests/lang/eval-fail-abort.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'abort' builtin + + at /pwd/lang/eval-fail-abort.nix:1:14: + + 1| if true then abort "this should fail" else 1 + | ^ + 2| + + error: evaluation aborted with the following error message: 'this should fail' diff --git a/tests/lang/eval-fail-antiquoted-path.err.exp b/tests/lang/eval-fail-antiquoted-path.err.exp new file mode 100644 index 000000000..425deba42 --- /dev/null +++ b/tests/lang/eval-fail-antiquoted-path.err.exp @@ -0,0 +1 @@ +error: getting attributes of path ‘PWD/lang/fnord’: No such file or directory diff --git a/tests/lang/eval-fail-assert.err.exp b/tests/lang/eval-fail-assert.err.exp new file mode 100644 index 000000000..aeecd8167 --- /dev/null +++ b/tests/lang/eval-fail-assert.err.exp @@ -0,0 +1,36 @@ +error: + … while evaluating the attribute 'body' + + at /pwd/lang/eval-fail-assert.nix:4:3: + + 3| + 4| body = x "x"; + | ^ + 5| } + + … from call site + + at /pwd/lang/eval-fail-assert.nix:4:10: + + 3| + 4| body = x "x"; + | ^ + 5| } + + … while calling 'x' + + at /pwd/lang/eval-fail-assert.nix:2:7: + + 1| let { + 2| x = arg: assert arg == "y"; 123; + | ^ + 3| + + error: assertion '(arg == "y")' failed + + at /pwd/lang/eval-fail-assert.nix:2:12: + + 1| let { + 2| x = arg: assert arg == "y"; 123; + | ^ + 3| diff --git a/tests/lang/eval-fail-bad-antiquote-1.err.exp b/tests/lang/eval-fail-bad-antiquote-1.err.exp new file mode 100644 index 000000000..cf94f53bc --- /dev/null +++ b/tests/lang/eval-fail-bad-antiquote-1.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-antiquote-1.nix:1:2: + + 1| "${x: x}" + | ^ + 2| + + error: cannot coerce a function to a string diff --git a/tests/lang/eval-fail-bad-antiquote-2.err.exp b/tests/lang/eval-fail-bad-antiquote-2.err.exp new file mode 100644 index 000000000..c8fe39d12 --- /dev/null +++ b/tests/lang/eval-fail-bad-antiquote-2.err.exp @@ -0,0 +1 @@ +error: operation 'addToStoreFromDump' is not supported by store 'dummy' diff --git a/tests/lang/eval-fail-bad-antiquote-3.err.exp b/tests/lang/eval-fail-bad-antiquote-3.err.exp new file mode 100644 index 000000000..fbefbc826 --- /dev/null +++ b/tests/lang/eval-fail-bad-antiquote-3.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-antiquote-3.nix:1:3: + + 1| ''${x: x}'' + | ^ + 2| + + error: cannot coerce a function to a string diff --git a/tests/lang/eval-fail-bad-string-interpolation-1.err.exp b/tests/lang/eval-fail-bad-string-interpolation-1.err.exp new file mode 100644 index 000000000..eb73e9a52 --- /dev/null +++ b/tests/lang/eval-fail-bad-string-interpolation-1.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-string-interpolation-1.nix:1:2: + + 1| "${x: x}" + | ^ + 2| + + error: cannot coerce a function to a string diff --git a/tests/lang/eval-fail-bad-string-interpolation-2.err.exp b/tests/lang/eval-fail-bad-string-interpolation-2.err.exp new file mode 100644 index 000000000..c8fe39d12 --- /dev/null +++ b/tests/lang/eval-fail-bad-string-interpolation-2.err.exp @@ -0,0 +1 @@ +error: operation 'addToStoreFromDump' is not supported by store 'dummy' diff --git a/tests/lang/eval-fail-bad-string-interpolation-3.err.exp b/tests/lang/eval-fail-bad-string-interpolation-3.err.exp new file mode 100644 index 000000000..ac14f329b --- /dev/null +++ b/tests/lang/eval-fail-bad-string-interpolation-3.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-string-interpolation-3.nix:1:3: + + 1| ''${x: x}'' + | ^ + 2| + + error: cannot coerce a function to a string diff --git a/tests/lang/eval-fail-blackhole.err.exp b/tests/lang/eval-fail-blackhole.err.exp new file mode 100644 index 000000000..f0618d8ac --- /dev/null +++ b/tests/lang/eval-fail-blackhole.err.exp @@ -0,0 +1,18 @@ +error: + … while evaluating the attribute 'body' + + at /pwd/lang/eval-fail-blackhole.nix:2:3: + + 1| let { + 2| body = x; + | ^ + 3| x = y; + + error: infinite recursion encountered + + at /pwd/lang/eval-fail-blackhole.nix:3:7: + + 2| body = x; + 3| x = y; + | ^ + 4| y = x; diff --git a/tests/lang/eval-fail-deepseq.err.exp b/tests/lang/eval-fail-deepseq.err.exp new file mode 100644 index 000000000..5e204ba73 --- /dev/null +++ b/tests/lang/eval-fail-deepseq.err.exp @@ -0,0 +1,26 @@ +error: + … while calling the 'deepSeq' builtin + + at /pwd/lang/eval-fail-deepseq.nix:1:1: + + 1| builtins.deepSeq { x = abort "foo"; } 456 + | ^ + 2| + + … while evaluating the attribute 'x' + + at /pwd/lang/eval-fail-deepseq.nix:1:20: + + 1| builtins.deepSeq { x = abort "foo"; } 456 + | ^ + 2| + + … while calling the 'abort' builtin + + at /pwd/lang/eval-fail-deepseq.nix:1:24: + + 1| builtins.deepSeq { x = abort "foo"; } 456 + | ^ + 2| + + error: evaluation aborted with the following error message: 'foo' diff --git a/tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp b/tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp new file mode 100644 index 000000000..0069285fb --- /dev/null +++ b/tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp @@ -0,0 +1,38 @@ +error: + … while calling the 'foldl'' builtin + + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:2:1: + + 1| # Tests that the result of applying op is forced even if the value is never used + 2| builtins.foldl' + | ^ + 3| (_: f: f null) + + … while calling anonymous lambda + + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:3:7: + + 2| builtins.foldl' + 3| (_: f: f null) + | ^ + 4| null + + … from call site + + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:3:10: + + 2| builtins.foldl' + 3| (_: f: f null) + | ^ + 4| null + + … while calling anonymous lambda + + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:5:6: + + 4| null + 5| [ (_: throw "Not the final value, but is still forced!") (_: 23) ] + | ^ + 6| + + error: Not the final value, but is still forced! diff --git a/tests/lang/eval-fail-fromTOML-timestamps.err.exp b/tests/lang/eval-fail-fromTOML-timestamps.err.exp new file mode 100644 index 000000000..f6bd19f5a --- /dev/null +++ b/tests/lang/eval-fail-fromTOML-timestamps.err.exp @@ -0,0 +1,12 @@ +error: + … while calling the 'fromTOML' builtin + + at /pwd/lang/eval-fail-fromTOML-timestamps.nix:1:1: + + 1| builtins.fromTOML '' + | ^ + 2| key = "value" + + error: while parsing a TOML string: Dates and times are not supported + + at «none»:0: (source not available) diff --git a/tests/lang/eval-fail-hashfile-missing.err.exp b/tests/lang/eval-fail-hashfile-missing.err.exp new file mode 100644 index 000000000..8e77dec1e --- /dev/null +++ b/tests/lang/eval-fail-hashfile-missing.err.exp @@ -0,0 +1,19 @@ +error: + … while calling the 'toString' builtin + + at /pwd/lang/eval-fail-hashfile-missing.nix:4:3: + + 3| in + 4| toString (builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"])) + | ^ + 5| + + … while evaluating the first argument passed to builtins.toString + + at «none»:0: (source not available) + + … while calling the 'hashFile' builtin + + at «none»:0: (source not available) + + error: opening file '/pwd/lang/this-file-is-definitely-not-there-7392097': No such file or directory diff --git a/tests/lang/eval-fail-list.err.exp b/tests/lang/eval-fail-list.err.exp new file mode 100644 index 000000000..24d682118 --- /dev/null +++ b/tests/lang/eval-fail-list.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating one of the elements to concatenate + + at /pwd/lang/eval-fail-list.nix:1:2: + + 1| 8++1 + | ^ + 2| + + error: value is an integer while a list was expected diff --git a/tests/lang/eval-fail-list.nix b/tests/lang/eval-fail-list.nix new file mode 100644 index 000000000..fa749f2f7 --- /dev/null +++ b/tests/lang/eval-fail-list.nix @@ -0,0 +1 @@ +8++1 diff --git a/tests/lang/eval-fail-missing-arg.err.exp b/tests/lang/eval-fail-missing-arg.err.exp new file mode 100644 index 000000000..61fabf0d5 --- /dev/null +++ b/tests/lang/eval-fail-missing-arg.err.exp @@ -0,0 +1,16 @@ +error: + … from call site + + at /pwd/lang/eval-fail-missing-arg.nix:1:1: + + 1| ({x, y, z}: x + y + z) {x = "foo"; z = "bar";} + | ^ + 2| + + error: function 'anonymous lambda' called without required argument 'y' + + at /pwd/lang/eval-fail-missing-arg.nix:1:2: + + 1| ({x, y, z}: x + y + z) {x = "foo"; z = "bar";} + | ^ + 2| diff --git a/tests/lang/eval-fail-nonexist-path.err.exp b/tests/lang/eval-fail-nonexist-path.err.exp new file mode 100644 index 000000000..c8fe39d12 --- /dev/null +++ b/tests/lang/eval-fail-nonexist-path.err.exp @@ -0,0 +1 @@ +error: operation 'addToStoreFromDump' is not supported by store 'dummy' diff --git a/tests/lang/eval-fail-path-slash.err.exp b/tests/lang/eval-fail-path-slash.err.exp new file mode 100644 index 000000000..f0011c97f --- /dev/null +++ b/tests/lang/eval-fail-path-slash.err.exp @@ -0,0 +1,8 @@ +error: path has a trailing slash + + at /pwd/lang/eval-fail-path-slash.nix:6:12: + + 5| # and https://nixos.org/nix-dev/2016-June/020829.html + 6| /nix/store/ + | ^ + 7| diff --git a/tests/lang/eval-fail-recursion.err.exp b/tests/lang/eval-fail-recursion.err.exp new file mode 100644 index 000000000..af64133cb --- /dev/null +++ b/tests/lang/eval-fail-recursion.err.exp @@ -0,0 +1,16 @@ +error: + … in the right operand of the update (//) operator + + at /pwd/lang/eval-fail-recursion.nix:1:12: + + 1| let a = {} // a; in a.foo + | ^ + 2| + + error: infinite recursion encountered + + at /pwd/lang/eval-fail-recursion.nix:1:15: + + 1| let a = {} // a; in a.foo + | ^ + 2| diff --git a/tests/lang/eval-fail-recursion.nix b/tests/lang/eval-fail-recursion.nix new file mode 100644 index 000000000..075b5ed06 --- /dev/null +++ b/tests/lang/eval-fail-recursion.nix @@ -0,0 +1 @@ +let a = {} // a; in a.foo diff --git a/tests/lang/eval-fail-remove.err.exp b/tests/lang/eval-fail-remove.err.exp new file mode 100644 index 000000000..e82cdac98 --- /dev/null +++ b/tests/lang/eval-fail-remove.err.exp @@ -0,0 +1,19 @@ +error: + … while evaluating the attribute 'body' + + at /pwd/lang/eval-fail-remove.nix:4:3: + + 3| + 4| body = (removeAttrs attrs ["x"]).x; + | ^ + 5| } + + error: attribute 'x' missing + + at /pwd/lang/eval-fail-remove.nix:4:10: + + 3| + 4| body = (removeAttrs attrs ["x"]).x; + | ^ + 5| } + Did you mean y? diff --git a/tests/lang/eval-fail-scope-5.err.exp b/tests/lang/eval-fail-scope-5.err.exp new file mode 100644 index 000000000..22b6166f8 --- /dev/null +++ b/tests/lang/eval-fail-scope-5.err.exp @@ -0,0 +1,36 @@ +error: + … while evaluating the attribute 'body' + + at /pwd/lang/eval-fail-scope-5.nix:8:3: + + 7| + 8| body = f {}; + | ^ + 9| + + … from call site + + at /pwd/lang/eval-fail-scope-5.nix:8:10: + + 7| + 8| body = f {}; + | ^ + 9| + + … while calling 'f' + + at /pwd/lang/eval-fail-scope-5.nix:6:7: + + 5| + 6| f = {x ? y, y ? x}: x + y; + | ^ + 7| + + error: infinite recursion encountered + + at /pwd/lang/eval-fail-scope-5.nix:6:12: + + 5| + 6| f = {x ? y, y ? x}: x + y; + | ^ + 7| diff --git a/tests/lang/eval-fail-seq.err.exp b/tests/lang/eval-fail-seq.err.exp new file mode 100644 index 000000000..33a7e9491 --- /dev/null +++ b/tests/lang/eval-fail-seq.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'seq' builtin + + at /pwd/lang/eval-fail-seq.nix:1:1: + + 1| builtins.seq (abort "foo") 2 + | ^ + 2| + + … while calling the 'abort' builtin + + at /pwd/lang/eval-fail-seq.nix:1:15: + + 1| builtins.seq (abort "foo") 2 + | ^ + 2| + + error: evaluation aborted with the following error message: 'foo' diff --git a/tests/lang/eval-fail-set-override.err.exp b/tests/lang/eval-fail-set-override.err.exp new file mode 100644 index 000000000..beb29d678 --- /dev/null +++ b/tests/lang/eval-fail-set-override.err.exp @@ -0,0 +1,6 @@ +error: + … while evaluating the `__overrides` attribute + + at «none»:0: (source not available) + + error: value is an integer while a set was expected diff --git a/tests/lang/eval-fail-set-override.nix b/tests/lang/eval-fail-set-override.nix new file mode 100644 index 000000000..03551c186 --- /dev/null +++ b/tests/lang/eval-fail-set-override.nix @@ -0,0 +1 @@ +rec { __overrides = 1; } diff --git a/tests/lang/eval-fail-set.err.exp b/tests/lang/eval-fail-set.err.exp new file mode 100644 index 000000000..0d0140508 --- /dev/null +++ b/tests/lang/eval-fail-set.err.exp @@ -0,0 +1,7 @@ +error: undefined variable 'x' + + at /pwd/lang/eval-fail-set.nix:1:3: + + 1| 8.x + | ^ + 2| diff --git a/tests/lang/eval-fail-set.nix b/tests/lang/eval-fail-set.nix new file mode 100644 index 000000000..c6b7980b6 --- /dev/null +++ b/tests/lang/eval-fail-set.nix @@ -0,0 +1 @@ +8.x diff --git a/tests/lang/eval-fail-substring.err.exp b/tests/lang/eval-fail-substring.err.exp new file mode 100644 index 000000000..dc26a00bd --- /dev/null +++ b/tests/lang/eval-fail-substring.err.exp @@ -0,0 +1,12 @@ +error: + … while calling the 'substring' builtin + + at /pwd/lang/eval-fail-substring.nix:1:1: + + 1| builtins.substring (builtins.sub 0 1) 1 "x" + | ^ + 2| + + error: negative start position in 'substring' + + at «none»:0: (source not available) diff --git a/tests/lang/eval-fail-to-path.err.exp b/tests/lang/eval-fail-to-path.err.exp new file mode 100644 index 000000000..43ed2bdfc --- /dev/null +++ b/tests/lang/eval-fail-to-path.err.exp @@ -0,0 +1,14 @@ +error: + … while calling the 'toPath' builtin + + at /pwd/lang/eval-fail-to-path.nix:1:1: + + 1| builtins.toPath "foo/bar" + | ^ + 2| + + … while evaluating the first argument passed to builtins.toPath + + at «none»:0: (source not available) + + error: string 'foo/bar' doesn't represent an absolute path diff --git a/tests/lang/eval-fail-undeclared-arg.err.exp b/tests/lang/eval-fail-undeclared-arg.err.exp new file mode 100644 index 000000000..30db743c7 --- /dev/null +++ b/tests/lang/eval-fail-undeclared-arg.err.exp @@ -0,0 +1,17 @@ +error: + … from call site + + at /pwd/lang/eval-fail-undeclared-arg.nix:1:1: + + 1| ({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";} + | ^ + 2| + + error: function 'anonymous lambda' called with unexpected argument 'y' + + at /pwd/lang/eval-fail-undeclared-arg.nix:1:2: + + 1| ({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";} + | ^ + 2| + Did you mean one of x or z? diff --git a/tests/lang/eval-okay-fromjson.nix b/tests/lang/eval-okay-fromjson.nix index e1c0f86cc..4c526b9ae 100644 --- a/tests/lang/eval-okay-fromjson.nix +++ b/tests/lang/eval-okay-fromjson.nix @@ -11,9 +11,12 @@ builtins.fromJSON "Width": 200, "Height": 250 }, + "Animated" : false, + "IDs": [116, 943, 234, 38793, true ,false,null, -100], + "Escapes": "\"\\\/\t\n\r\t", "Subtitle" : false, - "Latitude": 46.2051, - "Longitude": 6.0723 + "Latitude": 37.7668, + "Longitude": -122.3959 } } '' @@ -28,8 +31,11 @@ builtins.fromJSON Width = 200; Height = 250; }; + Animated = false; + IDs = [ 116 943 234 38793 true false null (0-100) ]; + Escapes = "\"\\\/\t\n\r\t"; # supported in JSON but not Nix: \b\f Subtitle = false; - Latitude = 46.2051; - Longitude = 6.0723; + Latitude = 37.7668; + Longitude = -122.3959; }; } diff --git a/tests/lang/eval-okay-overrides.nix b/tests/lang/eval-okay-overrides.nix index 358742b36..719bdc9c0 100644 --- a/tests/lang/eval-okay-overrides.nix +++ b/tests/lang/eval-okay-overrides.nix @@ -1,6 +1,6 @@ let - overrides = { a = 2; }; + overrides = { a = 2; b = 3; }; in (rec { __overrides = overrides; diff --git a/tests/lang/eval-okay-print.err.exp b/tests/lang/eval-okay-print.err.exp new file mode 100644 index 000000000..3fc99be3e --- /dev/null +++ b/tests/lang/eval-okay-print.err.exp @@ -0,0 +1 @@ +trace: [ ] diff --git a/tests/lang/eval-okay-print.exp b/tests/lang/eval-okay-print.exp new file mode 100644 index 000000000..0d960fb70 --- /dev/null +++ b/tests/lang/eval-okay-print.exp @@ -0,0 +1 @@ +[ null [ [ «repeated» ] ] ] diff --git a/tests/lang/eval-okay-print.nix b/tests/lang/eval-okay-print.nix new file mode 100644 index 000000000..d36ba4da3 --- /dev/null +++ b/tests/lang/eval-okay-print.nix @@ -0,0 +1 @@ +with builtins; trace [(1+1)] [ null toString (deepSeq "x") (a: a) (let x=[x]; in x) ] diff --git a/tests/lang/eval-okay-search-path.flags b/tests/lang/eval-okay-search-path.flags index a28e68210..dfad1c611 100644 --- a/tests/lang/eval-okay-search-path.flags +++ b/tests/lang/eval-okay-search-path.flags @@ -1 +1 @@ --I lang/dir1 -I lang/dir2 -I dir5=lang/dir3 \ No newline at end of file +-I lang/dir1 -I lang/dir2 -I dir5=lang/dir3 diff --git a/tests/lang/framework.sh b/tests/lang/framework.sh new file mode 100644 index 000000000..d6499e0e9 --- /dev/null +++ b/tests/lang/framework.sh @@ -0,0 +1,33 @@ +# Golden test support +# +# Test that the output of the given test matches what is expected. If +# `_NIX_TEST_ACCEPT` is non-empty also update the expected output so +# that next time the test succeeds. +function diffAndAcceptInner() { + local -r testName=$1 + local -r got="$2" + local -r expected="$3" + + # Absence of expected file indicates empty output expected. + if test -e "$expected"; then + local -r expectedOrEmpty="$expected" + else + local -r expectedOrEmpty=lang/empty.exp + fi + + # Diff so we get a nice message + if ! diff "$got" "$expectedOrEmpty"; then + echo "FAIL: evaluation result of $testName not as expected" + badDiff=1 + fi + + # Update expected if `_NIX_TEST_ACCEPT` is non-empty. + if test -n "${_NIX_TEST_ACCEPT-}"; then + cp "$got" "$expected" + # Delete empty expected files to avoid bloating the repo with + # empty files. + if ! test -s "$expected"; then + rm "$expected" + fi + fi +} diff --git a/tests/lang/parse-fail-dup-attrs-1.err.exp b/tests/lang/parse-fail-dup-attrs-1.err.exp new file mode 100644 index 000000000..4fe6b7a1f --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-1.err.exp @@ -0,0 +1,7 @@ +error: attribute 'x' already defined at «stdin»:1:3 + + at «stdin»:3:3: + + 2| y = 456; + 3| x = 789; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-2.err.exp b/tests/lang/parse-fail-dup-attrs-2.err.exp new file mode 100644 index 000000000..3aba2891f --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-2.err.exp @@ -0,0 +1,7 @@ +error: attribute 'x' already defined at «stdin»:9:5 + + at «stdin»:10:17: + + 9| x = 789; + 10| inherit (as) x; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-3.err.exp b/tests/lang/parse-fail-dup-attrs-3.err.exp new file mode 100644 index 000000000..3aba2891f --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-3.err.exp @@ -0,0 +1,7 @@ +error: attribute 'x' already defined at «stdin»:9:5 + + at «stdin»:10:17: + + 9| x = 789; + 10| inherit (as) x; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-4.err.exp b/tests/lang/parse-fail-dup-attrs-4.err.exp new file mode 100644 index 000000000..ff68446a1 --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-4.err.exp @@ -0,0 +1,7 @@ +error: attribute 'services.ssh.port' already defined at «stdin»:2:3 + + at «stdin»:3:3: + + 2| services.ssh.port = 22; + 3| services.ssh.port = 23; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-6.err.exp b/tests/lang/parse-fail-dup-attrs-6.err.exp new file mode 100644 index 000000000..74823fc25 --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-6.err.exp @@ -0,0 +1 @@ +error: attribute ‘services.ssh’ at (string):3:3 already defined at (string):2:3 diff --git a/tests/lang/parse-fail-dup-attrs-7.err.exp b/tests/lang/parse-fail-dup-attrs-7.err.exp new file mode 100644 index 000000000..512a499ca --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-7.err.exp @@ -0,0 +1,7 @@ +error: attribute 'x' already defined at «stdin»:6:12 + + at «stdin»:7:12: + + 6| inherit x; + 7| inherit x; + | ^ diff --git a/tests/lang/parse-fail-dup-formals.err.exp b/tests/lang/parse-fail-dup-formals.err.exp new file mode 100644 index 000000000..1d566fb33 --- /dev/null +++ b/tests/lang/parse-fail-dup-formals.err.exp @@ -0,0 +1,6 @@ +error: duplicate formal function argument 'x' + + at «stdin»:1:8: + + 1| {x, y, x}: x + | ^ diff --git a/tests/lang/parse-fail-eof-in-string.err.exp b/tests/lang/parse-fail-eof-in-string.err.exp new file mode 100644 index 000000000..f9fa72312 --- /dev/null +++ b/tests/lang/parse-fail-eof-in-string.err.exp @@ -0,0 +1,7 @@ +error: syntax error, unexpected end of file, expecting '"' + + at «stdin»:3:5: + + 2| # Note that this file must not end with a newline. + 3| a 1"$ + | ^ diff --git a/tests/lang/parse-fail-mixed-nested-attrs1.err.exp b/tests/lang/parse-fail-mixed-nested-attrs1.err.exp new file mode 100644 index 000000000..32f776795 --- /dev/null +++ b/tests/lang/parse-fail-mixed-nested-attrs1.err.exp @@ -0,0 +1,8 @@ +error: attribute 'z' already defined at «stdin»:3:16 + + at «stdin»:2:3: + + 1| { + 2| x.z = 3; + | ^ + 3| x = { y = 3; z = 3; }; diff --git a/tests/lang/parse-fail-mixed-nested-attrs2.err.exp b/tests/lang/parse-fail-mixed-nested-attrs2.err.exp new file mode 100644 index 000000000..0437cd50c --- /dev/null +++ b/tests/lang/parse-fail-mixed-nested-attrs2.err.exp @@ -0,0 +1,8 @@ +error: attribute 'y' already defined at «stdin»:3:9 + + at «stdin»:2:3: + + 1| { + 2| x.y.y = 3; + | ^ + 3| x = { y.y= 3; z = 3; }; diff --git a/tests/lang/parse-fail-patterns-1.err.exp b/tests/lang/parse-fail-patterns-1.err.exp new file mode 100644 index 000000000..634a04aaa --- /dev/null +++ b/tests/lang/parse-fail-patterns-1.err.exp @@ -0,0 +1,7 @@ +error: duplicate formal function argument 'args' + + at «stdin»:1:1: + + 1| args@{args, x, y, z}: x + | ^ + 2| diff --git a/tests/lang/parse-fail-regression-20060610.err.exp b/tests/lang/parse-fail-regression-20060610.err.exp new file mode 100644 index 000000000..167d01e85 --- /dev/null +++ b/tests/lang/parse-fail-regression-20060610.err.exp @@ -0,0 +1,8 @@ +error: undefined variable 'gcc' + + at «stdin»:8:12: + + 7| + 8| body = ({ + | ^ + 9| inherit gcc; diff --git a/tests/lang/parse-fail-undef-var-2.err.exp b/tests/lang/parse-fail-undef-var-2.err.exp new file mode 100644 index 000000000..77c96bbd2 --- /dev/null +++ b/tests/lang/parse-fail-undef-var-2.err.exp @@ -0,0 +1,7 @@ +error: syntax error, unexpected ':', expecting '}' + + at «stdin»:3:13: + + 2| + 3| f = {x, y : + | ^ diff --git a/tests/lang/parse-fail-undef-var.err.exp b/tests/lang/parse-fail-undef-var.err.exp new file mode 100644 index 000000000..48e88747f --- /dev/null +++ b/tests/lang/parse-fail-undef-var.err.exp @@ -0,0 +1,7 @@ +error: undefined variable 'y' + + at «stdin»:1:4: + + 1| x: y + | ^ + 2| diff --git a/tests/lang/parse-fail-utf8.err.exp b/tests/lang/parse-fail-utf8.err.exp new file mode 100644 index 000000000..6087479a3 --- /dev/null +++ b/tests/lang/parse-fail-utf8.err.exp @@ -0,0 +1,6 @@ +error: syntax error, unexpected invalid token, expecting end of file + + at «stdin»:1:5: + + 1| 123 + | ^ diff --git a/tests/lang/parse-okay-1.exp b/tests/lang/parse-okay-1.exp new file mode 100644 index 000000000..d5ab5f18a --- /dev/null +++ b/tests/lang/parse-okay-1.exp @@ -0,0 +1 @@ +({ x, y, z }: ((x + y) + z)) diff --git a/tests/lang/parse-okay-crlf.exp b/tests/lang/parse-okay-crlf.exp new file mode 100644 index 000000000..4213609fc --- /dev/null +++ b/tests/lang/parse-okay-crlf.exp @@ -0,0 +1 @@ +rec { foo = "multi\nline\n string\n test\r"; x = y; y = 123; z = 456; } diff --git a/tests/lang/parse-okay-dup-attrs-5.exp b/tests/lang/parse-okay-dup-attrs-5.exp new file mode 100644 index 000000000..88b0b036f --- /dev/null +++ b/tests/lang/parse-okay-dup-attrs-5.exp @@ -0,0 +1 @@ +{ services = { ssh = { enable = true; port = 23; }; }; } diff --git a/tests/lang/parse-okay-dup-attrs-6.exp b/tests/lang/parse-okay-dup-attrs-6.exp new file mode 100644 index 000000000..88b0b036f --- /dev/null +++ b/tests/lang/parse-okay-dup-attrs-6.exp @@ -0,0 +1 @@ +{ services = { ssh = { enable = true; port = 23; }; }; } diff --git a/tests/lang/parse-okay-mixed-nested-attrs-1.exp b/tests/lang/parse-okay-mixed-nested-attrs-1.exp new file mode 100644 index 000000000..89c66f760 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-1.exp @@ -0,0 +1 @@ +{ x = { q = 3; y = 3; z = 3; }; } diff --git a/tests/lang/parse-okay-mixed-nested-attrs-2.exp b/tests/lang/parse-okay-mixed-nested-attrs-2.exp new file mode 100644 index 000000000..89c66f760 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-2.exp @@ -0,0 +1 @@ +{ x = { q = 3; y = 3; z = 3; }; } diff --git a/tests/lang/parse-okay-mixed-nested-attrs-3.exp b/tests/lang/parse-okay-mixed-nested-attrs-3.exp new file mode 100644 index 000000000..b89a59734 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-3.exp @@ -0,0 +1 @@ +{ services = { httpd = { enable = true; }; ssh = { enable = true; port = 123; }; }; } diff --git a/tests/lang/parse-okay-regression-20041027.exp b/tests/lang/parse-okay-regression-20041027.exp new file mode 100644 index 000000000..9df7219e4 --- /dev/null +++ b/tests/lang/parse-okay-regression-20041027.exp @@ -0,0 +1 @@ +({ fetchurl, stdenv }: ((stdenv).mkDerivation { name = "libXi-6.0.1"; src = (fetchurl { md5 = "7e935a42428d63a387b3c048be0f2756"; url = "http://freedesktop.org/~xlibs/release/libXi-6.0.1.tar.bz2"; }); })) diff --git a/tests/lang/parse-okay-regression-751.exp b/tests/lang/parse-okay-regression-751.exp new file mode 100644 index 000000000..e2ed886fe --- /dev/null +++ b/tests/lang/parse-okay-regression-751.exp @@ -0,0 +1 @@ +(let const = (a: "const"); in ((const { x = "q"; }))) diff --git a/tests/lang/parse-okay-subversion.exp b/tests/lang/parse-okay-subversion.exp new file mode 100644 index 000000000..4168ee8bf --- /dev/null +++ b/tests/lang/parse-okay-subversion.exp @@ -0,0 +1 @@ +({ fetchurl, localServer ? false, httpServer ? false, sslSupport ? false, pythonBindings ? false, javaSwigBindings ? false, javahlBindings ? false, stdenv, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null }: assert (expat != null); assert (localServer -> (db4 != null)); assert (httpServer -> ((httpd != null) && ((httpd).expat == expat))); assert (sslSupport -> ((openssl != null) && (httpServer -> ((httpd).openssl == openssl)))); assert (pythonBindings -> ((swig != null) && (swig).pythonSupport)); assert (javaSwigBindings -> ((swig != null) && (swig).javaSupport)); assert (javahlBindings -> (j2sdk != null)); ((stdenv).mkDerivation { builder = /foo/bar; db4 = (if localServer then db4 else null); inherit expat ; inherit httpServer ; httpd = (if httpServer then httpd else null); j2sdk = (if javaSwigBindings then (swig).j2sdk else (if javahlBindings then j2sdk else null)); inherit javaSwigBindings ; inherit javahlBindings ; inherit localServer ; name = "subversion-1.1.1"; openssl = (if sslSupport then openssl else null); patches = (if javahlBindings then [ (/javahl.patch) ] else [ ]); python = (if pythonBindings then (swig).python else null); inherit pythonBindings ; src = (fetchurl { md5 = "a180c3fe91680389c210c99def54d9e0"; url = "http://subversion.tigris.org/tarballs/subversion-1.1.1.tar.bz2"; }); inherit sslSupport ; swig = (if (pythonBindings || javaSwigBindings) then swig else null); })) diff --git a/tests/lang/parse-okay-url.exp b/tests/lang/parse-okay-url.exp new file mode 100644 index 000000000..e5f0829b0 --- /dev/null +++ b/tests/lang/parse-okay-url.exp @@ -0,0 +1 @@ +[ ("x:x") ("https://svn.cs.uu.nl:12443/repos/trace/trunk") ("http://www2.mplayerhq.hu/MPlayer/releases/fonts/font-arial-iso-8859-1.tar.bz2") ("http://losser.st-lab.cs.uu.nl/~armijn/.nix/gcc-3.3.4-static-nix.tar.gz") ("http://fpdownload.macromedia.com/get/shockwave/flash/english/linux/7.0r25/install_flash_player_7_linux.tar.gz") ("https://ftp5.gwdg.de/pub/linux/archlinux/extra/os/x86_64/unzip-6.0-14-x86_64.pkg.tar.zst") ("ftp://ftp.gtk.org/pub/gtk/v1.2/gtk+-1.2.10.tar.gz") ] diff --git a/tests/local.mk b/tests/local.mk index 88848926b..bb80c5317 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -20,6 +20,7 @@ nix_tests = \ remote-store.sh \ legacy-ssh-store.sh \ lang.sh \ + lang-test-infra.sh \ experimental-features.sh \ fetchMercurial.sh \ gc-auto.sh \ diff --git a/tests/misc.sh b/tests/misc.sh index 60d58310e..af96d20bd 100644 --- a/tests/misc.sh +++ b/tests/misc.sh @@ -24,3 +24,9 @@ eval_stdin_res=$(echo 'let a = {} // a; in a.foo' | nix-instantiate --eval -E - echo $eval_stdin_res | grep "at «stdin»:1:15:" echo $eval_stdin_res | grep "infinite recursion encountered" +# Attribute path errors +expectStderr 1 nix-instantiate --eval -E '{}' -A '"x' | grepQuiet "missing closing quote in selection path" +expectStderr 1 nix-instantiate --eval -E '[]' -A 'x' | grepQuiet "should be a set" +expectStderr 1 nix-instantiate --eval -E '{}' -A '1' | grepQuiet "should be a list" +expectStderr 1 nix-instantiate --eval -E '{}' -A '.' | grepQuiet "empty attribute name" +expectStderr 1 nix-instantiate --eval -E '[]' -A '1' | grepQuiet "out of range" diff --git a/tests/restricted.sh b/tests/restricted.sh index 776893a56..17f310a4b 100644 --- a/tests/restricted.sh +++ b/tests/restricted.sh @@ -49,3 +49,5 @@ output="$(nix eval --raw --restrict-eval -I "$traverseDir" \ 2>&1 || :)" echo "$output" | grep "is forbidden" echo "$output" | grepInverse -F restricted-secret + +expectStderr 1 nix-instantiate --restrict-eval true ./dependencies.nix | grepQuiet "forbidden in restricted mode" From 0309f6b5b8ca354e1a43d320928d3d546ce7f878 Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Wed, 12 Jul 2023 12:32:57 +0100 Subject: [PATCH 105/909] Update src/libstore/globals.hh Co-authored-by: Valentin Gagarin --- src/libstore/globals.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 81fa154bb..d8fda9707 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -530,8 +530,8 @@ public: Nix will attempt to drop supplementary groups when building with sandboxing. However this can fail under some circumstances. - For example, if the user lacks the CAP_SETGID capability. - Search setgroups(2) for EPERM to find more detailed information on this. + For example, if the user lacks the `CAP_SETGID` capability. + Search `setgroups(2)` for `EPERM` to find more detailed information on this. If you encounter such a failure, setting this option to `false` will let you ignore it and continue. But before doing so, you should consider the security implications carefully. From a2acd23466108a1a10ed3b6b874c558bd95e7645 Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Wed, 12 Jul 2023 12:33:05 +0100 Subject: [PATCH 106/909] Update src/libstore/globals.hh Co-authored-by: Valentin Gagarin --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d8fda9707..d4b8fb1f9 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -538,7 +538,7 @@ public: Not dropping supplementary groups means the build sandbox will be less restricted than intended. This option defaults to `true` when the user is root - (since root usually has permissions to call setgroups) + (since `root` usually has permissions to call setgroups) and `false` otherwise. )"}; From 2c3fb0eb33d205d1937b7ed801bdb36bb301d1a8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 12 Jul 2023 22:22:44 -0400 Subject: [PATCH 107/909] Move `BuiltPath` to its own header/C++ file in libcmd It is less important, and used less widely, than `DerivedPath`. --- src/libcmd/built-path.cc | 67 ++++++++++++++++++++++++++++++++++++ src/libcmd/built-path.hh | 47 +++++++++++++++++++++++++ src/libcmd/installables.hh | 1 + src/libstore/derived-path.cc | 56 ------------------------------ src/libstore/derived-path.hh | 41 ---------------------- 5 files changed, 115 insertions(+), 97 deletions(-) create mode 100644 src/libcmd/built-path.cc create mode 100644 src/libcmd/built-path.hh diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc new file mode 100644 index 000000000..db9c440e3 --- /dev/null +++ b/src/libcmd/built-path.cc @@ -0,0 +1,67 @@ +#include "built-path.hh" +#include "derivations.hh" +#include "store-api.hh" + +#include + +#include + +namespace nix { + +nlohmann::json BuiltPath::Built::toJSON(ref store) const { + nlohmann::json res; + res["drvPath"] = store->printStorePath(drvPath); + for (const auto& [output, path] : outputs) { + res["outputs"][output] = store->printStorePath(path); + } + return res; +} + +StorePathSet BuiltPath::outPaths() const +{ + return std::visit( + overloaded{ + [](const BuiltPath::Opaque & p) { return StorePathSet{p.path}; }, + [](const BuiltPath::Built & b) { + StorePathSet res; + for (auto & [_, path] : b.outputs) + res.insert(path); + return res; + }, + }, raw() + ); +} + +RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const +{ + RealisedPath::Set res; + std::visit( + overloaded{ + [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, + [&](const BuiltPath::Built & p) { + auto drvHashes = + staticOutputHashes(store, store.readDerivation(p.drvPath)); + for (auto& [outputName, outputPath] : p.outputs) { + if (experimentalFeatureSettings.isEnabled( + Xp::CaDerivations)) { + auto drvOutput = get(drvHashes, outputName); + if (!drvOutput) + throw Error( + "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", + store.printStorePath(p.drvPath), outputName); + auto thisRealisation = store.queryRealisation( + DrvOutput{*drvOutput, outputName}); + assert(thisRealisation); // We’ve built it, so we must + // have the realisation + res.insert(*thisRealisation); + } else { + res.insert(outputPath); + } + } + }, + }, + raw()); + return res; +} + +} diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh new file mode 100644 index 000000000..c563a46e9 --- /dev/null +++ b/src/libcmd/built-path.hh @@ -0,0 +1,47 @@ +#include "derived-path.hh" + +namespace nix { + +/** + * A built derived path with hints in the form of optional concrete output paths. + * + * See 'BuiltPath' for more an explanation. + */ +struct BuiltPathBuilt { + StorePath drvPath; + std::map outputs; + + nlohmann::json toJSON(ref store) const; + static BuiltPathBuilt parse(const Store & store, std::string_view); + + GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs); +}; + +using _BuiltPathRaw = std::variant< + DerivedPath::Opaque, + BuiltPathBuilt +>; + +/** + * A built path. Similar to a DerivedPath, but enriched with the corresponding + * output path(s). + */ +struct BuiltPath : _BuiltPathRaw { + using Raw = _BuiltPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = BuiltPathBuilt; + + inline const Raw & raw() const { + return static_cast(*this); + } + + StorePathSet outPaths() const; + RealisedPath::Set toRealisedPaths(Store & store) const; + +}; + +typedef std::vector BuiltPaths; + +} diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 42d6c7c7c..b0dc0dc02 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -5,6 +5,7 @@ #include "path.hh" #include "outputs-spec.hh" #include "derived-path.hh" +#include "built-path.hh" #include "store-api.hh" #include "build-result.hh" diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 9a2ffda39..52d073f81 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -1,5 +1,4 @@ #include "derived-path.hh" -#include "derivations.hh" #include "store-api.hh" #include @@ -30,30 +29,6 @@ nlohmann::json DerivedPath::Built::toJSON(ref store) const { return res; } -nlohmann::json BuiltPath::Built::toJSON(ref store) const { - nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); - for (const auto& [output, path] : outputs) { - res["outputs"][output] = store->printStorePath(path); - } - return res; -} - -StorePathSet BuiltPath::outPaths() const -{ - return std::visit( - overloaded{ - [](const BuiltPath::Opaque & p) { return StorePathSet{p.path}; }, - [](const BuiltPath::Built & b) { - StorePathSet res; - for (auto & [_, path] : b.outputs) - res.insert(path); - return res; - }, - }, raw() - ); -} - std::string DerivedPath::Opaque::to_string(const Store & store) const { return store.printStorePath(path); @@ -121,35 +96,4 @@ DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s) return parseWith(store, s, "!"); } -RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const -{ - RealisedPath::Set res; - std::visit( - overloaded{ - [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, - [&](const BuiltPath::Built & p) { - auto drvHashes = - staticOutputHashes(store, store.readDerivation(p.drvPath)); - for (auto& [outputName, outputPath] : p.outputs) { - if (experimentalFeatureSettings.isEnabled( - Xp::CaDerivations)) { - auto drvOutput = get(drvHashes, outputName); - if (!drvOutput) - throw Error( - "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", - store.printStorePath(p.drvPath), outputName); - auto thisRealisation = store.queryRealisation( - DrvOutput{*drvOutput, outputName}); - assert(thisRealisation); // We’ve built it, so we must - // have the realisation - res.insert(*thisRealisation); - } else { - res.insert(outputPath); - } - } - }, - }, - raw()); - return res; -} } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 5f7acbebc..6ea80c92e 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -109,47 +109,6 @@ struct DerivedPath : _DerivedPathRaw { static DerivedPath parseLegacy(const Store & store, std::string_view); }; -/** - * A built derived path with hints in the form of optional concrete output paths. - * - * See 'BuiltPath' for more an explanation. - */ -struct BuiltPathBuilt { - StorePath drvPath; - std::map outputs; - - nlohmann::json toJSON(ref store) const; - static BuiltPathBuilt parse(const Store & store, std::string_view); - - GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs); -}; - -using _BuiltPathRaw = std::variant< - DerivedPath::Opaque, - BuiltPathBuilt ->; - -/** - * A built path. Similar to a DerivedPath, but enriched with the corresponding - * output path(s). - */ -struct BuiltPath : _BuiltPathRaw { - using Raw = _BuiltPathRaw; - using Raw::Raw; - - using Opaque = DerivedPathOpaque; - using Built = BuiltPathBuilt; - - inline const Raw & raw() const { - return static_cast(*this); - } - - StorePathSet outPaths() const; - RealisedPath::Set toRealisedPaths(Store & store) const; - -}; - typedef std::vector DerivedPaths; -typedef std::vector BuiltPaths; } From caabc4f64889d5a4c47d6102b3aa1d3c80bbc107 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 12 Jul 2023 23:33:43 -0400 Subject: [PATCH 108/909] Feature gate `DownstreamPlaceholder::unknownCaOutput` This is a part of CA derivations that we forgot to put behind the experimental feature. This was caught by @fricklerhandwerk in https://github.com/NixOS/nix/pull/8369#discussion_r1258133719 --- src/libexpr/eval.cc | 5 ++-- src/libexpr/eval.hh | 5 +++- src/libexpr/primops.cc | 25 ++++++++++---------- src/libexpr/tests/derived-path.cc | 9 ++++++- src/libstore/derivations.cc | 8 ++++--- src/libstore/downstream-placeholder.cc | 4 +++- src/libstore/downstream-placeholder.hh | 5 +++- src/libstore/tests/downstream-placeholder.cc | 16 +++++++++---- src/nix/app.cc | 11 +++++---- 9 files changed, 57 insertions(+), 31 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index be1bdb806..7071815d4 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1068,14 +1068,15 @@ void EvalState::mkOutputString( Value & value, const StorePath & drvPath, const std::string outputName, - std::optional optOutputPath) + std::optional optOutputPath, + const ExperimentalFeatureSettings & xpSettings) { value.mkString( optOutputPath ? store->printStorePath(*std::move(optOutputPath)) /* Downstream we would substitute this for an actual path once we build the floating CA derivation */ - : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName).render(), + : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName, xpSettings).render(), NixStringContext { NixStringContextElem::Built { .drvPath = drvPath, diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 277e77ad5..7f11ce71d 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -689,12 +689,15 @@ public: * be passed if and only if output store object is input-addressed. * Will be printed to form string if passed, otherwise a placeholder * will be used (see `DownstreamPlaceholder`). + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ void mkOutputString( Value & value, const StorePath & drvPath, const std::string outputName, - std::optional optOutputPath); + std::optional optOutputPath, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8a61e57cc..5dab06f26 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -83,22 +83,21 @@ StringMap EvalState::realiseContext(const NixStringContext & context) for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d }); store->buildPaths(buildReqs); - /* Get all the output paths corresponding to the placeholders we had */ for (auto & drv : drvs) { auto outputs = resolveDerivedPath(*store, drv); for (auto & [outputName, outputPath] : outputs) { - res.insert_or_assign( - DownstreamPlaceholder::unknownCaOutput(drv.drvPath, outputName).render(), - store->printStorePath(outputPath) - ); - } - } - - /* Add the output of this derivations to the allowed - paths. */ - if (allowedPaths) { - for (auto & [_placeholder, outputPath] : res) { - allowPath(store->toRealPath(outputPath)); + /* Add the output of this derivations to the allowed + paths. */ + if (allowedPaths) { + allowPath(outputPath); + } + /* Get all the output paths corresponding to the placeholders we had */ + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + res.insert_or_assign( + DownstreamPlaceholder::unknownCaOutput(drv.drvPath, outputName).render(), + store->printStorePath(outputPath) + ); + } } } diff --git a/src/libexpr/tests/derived-path.cc b/src/libexpr/tests/derived-path.cc index 8210efef2..c713fe28a 100644 --- a/src/libexpr/tests/derived-path.cc +++ b/src/libexpr/tests/derived-path.cc @@ -37,8 +37,15 @@ RC_GTEST_FIXTURE_PROP( prop_built_path_placeholder_round_trip, (const StorePath & drvPath, const StorePathName & outputName)) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "ca-derivations"); + auto * v = state.allocValue(); - state.mkOutputString(*v, drvPath, outputName.name, std::nullopt); + state.mkOutputString(*v, drvPath, outputName.name, std::nullopt, mockXpSettings); auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, ""); DerivedPath::Built b { .drvPath = drvPath, diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 6f63685d4..234d70ec5 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -878,9 +878,11 @@ std::optional Derivation::tryResolve( for (auto & [inputDrv, inputOutputs] : inputDrvs) { for (auto & outputName : inputOutputs) { if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) { - inputRewrites.emplace( - DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(), - store.printStorePath(*actualPath)); + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + inputRewrites.emplace( + DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(), + store.printStorePath(*actualPath)); + } resolved.inputSrcs.insert(*actualPath); } else { warn("output '%s' of input '%s' missing, aborting the resolving", diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index 1752738f2..d623c05e2 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -11,8 +11,10 @@ std::string DownstreamPlaceholder::render() const DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput( const StorePath & drvPath, - std::string_view outputName) + std::string_view outputName, + const ExperimentalFeatureSettings & xpSettings) { + xpSettings.require(Xp::CaDerivations); auto drvNameWithExtension = drvPath.name(); auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4); auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName); diff --git a/src/libstore/downstream-placeholder.hh b/src/libstore/downstream-placeholder.hh index f0c0dee77..97f77e6b8 100644 --- a/src/libstore/downstream-placeholder.hh +++ b/src/libstore/downstream-placeholder.hh @@ -52,10 +52,13 @@ public: * * The derivation itself is known (we have a store path for it), but * the output doesn't yet have a known store path. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DownstreamPlaceholder unknownCaOutput( const StorePath & drvPath, - std::string_view outputName); + std::string_view outputName, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** * Create a placehold for the output of an unknown derivation. diff --git a/src/libstore/tests/downstream-placeholder.cc b/src/libstore/tests/downstream-placeholder.cc index ec3e1000f..fd29530ac 100644 --- a/src/libstore/tests/downstream-placeholder.cc +++ b/src/libstore/tests/downstream-placeholder.cc @@ -5,17 +5,24 @@ namespace nix { TEST(DownstreamPlaceholder, unknownCaOutput) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "ca-derivations"); + ASSERT_EQ( DownstreamPlaceholder::unknownCaOutput( StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" }, - "out").render(), + "out", + mockXpSettings).render(), "/0c6rn30q4frawknapgwq386zq358m8r6msvywcvc89n6m5p2dgbz"); } TEST(DownstreamPlaceholder, unknownDerivation) { /** - * We set these in tests rather than the regular globals so we don't have - * to worry about race conditions if the tests run concurrently. + * Same reason as above */ ExperimentalFeatureSettings mockXpSettings; mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); @@ -24,7 +31,8 @@ TEST(DownstreamPlaceholder, unknownDerivation) { DownstreamPlaceholder::unknownDerivation( DownstreamPlaceholder::unknownCaOutput( StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv.drv" }, - "out"), + "out", + mockXpSettings), "out", mockXpSettings).render(), "/0gn6agqxjyyalf0dpihgyf49xq5hqxgw100f0wydnj6yqrhqsb3w"); diff --git a/src/nix/app.cc b/src/nix/app.cc index e678b54f0..e0f68b4fc 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -22,11 +22,12 @@ StringPairs resolveRewrites( StringPairs res; for (auto & dep : dependencies) if (auto drvDep = std::get_if(&dep.path)) - for (auto & [ outputName, outputPath ] : drvDep->outputs) - res.emplace( - DownstreamPlaceholder::unknownCaOutput(drvDep->drvPath, outputName).render(), - store.printStorePath(outputPath) - ); + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) + for (auto & [ outputName, outputPath ] : drvDep->outputs) + res.emplace( + DownstreamPlaceholder::unknownCaOutput(drvDep->drvPath, outputName).render(), + store.printStorePath(outputPath) + ); return res; } From e072e18475bc461e522f34d63cc74fc5fbf1640e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 13 Jul 2023 08:03:42 -0400 Subject: [PATCH 109/909] Fix race condition in the language tests When we pipe to `>(...)` like that, we unfortunately don't wait for the process to finish. Better to just substitute the file. Also, use the "unified" diff output that people (including myself) are more familiar with, thanks to Git. --- tests/lang.sh | 5 +++-- tests/lang/framework.sh | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/lang.sh b/tests/lang.sh index bfb942d3d..75dbbc38e 100755 --- a/tests/lang.sh +++ b/tests/lang.sh @@ -52,9 +52,10 @@ for i in lang/parse-okay-*.nix; do i=$(basename "$i" .nix) if expect 0 nix-instantiate --parse - < "lang/$i.nix" \ - 1> >(sed "s!$(pwd)!/pwd!g" > "lang/$i.out") \ - 2> >(sed "s!$(pwd)!/pwd!g" > "lang/$i.err") + 1> "lang/$i.out" \ + 2> "lang/$i.err" then + sed "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" diffAndAccept "$i" out exp diffAndAccept "$i" err err.exp else diff --git a/tests/lang/framework.sh b/tests/lang/framework.sh index d6499e0e9..516bff8ad 100644 --- a/tests/lang/framework.sh +++ b/tests/lang/framework.sh @@ -16,7 +16,7 @@ function diffAndAcceptInner() { fi # Diff so we get a nice message - if ! diff "$got" "$expectedOrEmpty"; then + if ! diff --unified "$got" "$expectedOrEmpty"; then echo "FAIL: evaluation result of $testName not as expected" badDiff=1 fi From 9e64f243406ff2b3fbe06adbc762cf965e6825a8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 13 Jul 2023 15:06:34 -0400 Subject: [PATCH 110/909] Revert "Check _NIX_TEST_NO_SANDBOX when setting _canUseSandbox." This reverts commit c1d39de1fbceebbb6b46abc4a21fb8e34423df99. --- tests/common/vars-and-functions.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common/vars-and-functions.sh.in b/tests/common/vars-and-functions.sh.in index ad0623871..dc7ce13cc 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/common/vars-and-functions.sh.in @@ -141,7 +141,7 @@ restartDaemon() { startDaemon } -if [[ -z "${_NIX_TEST_NO_SANDBOX:-}" ]] && [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then +if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then _canUseSandbox=1 fi From 84c4e6f0acfc902955d580aa090aa52f20ee82ad Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 13 Jul 2023 15:06:50 -0400 Subject: [PATCH 111/909] Revert "Skip build-remote-trustless unless sandbox is supported." This reverts commit 41412dc4ae0fec2d08335c19724276d99e0c6056. --- tests/build-remote-trustless-should-fail-0.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/build-remote-trustless-should-fail-0.sh b/tests/build-remote-trustless-should-fail-0.sh index b14101f83..fad1def59 100644 --- a/tests/build-remote-trustless-should-fail-0.sh +++ b/tests/build-remote-trustless-should-fail-0.sh @@ -2,7 +2,6 @@ source common.sh enableFeatures "daemon-trust-override" -requireSandboxSupport restartDaemon [[ $busybox =~ busybox ]] || skipTest "no busybox" From 1a13757880479921966adeca839cf182b3ffcbd0 Mon Sep 17 00:00:00 2001 From: cidkidnix Date: Thu, 13 Jul 2023 14:17:22 -0500 Subject: [PATCH 112/909] Add comment regarding the unset of NIX_STORE_DIR in build-remote.sh and supplementary-groups.sh --- tests/build-remote.sh | 1 + tests/supplementary-groups.sh | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/tests/build-remote.sh b/tests/build-remote.sh index 78e12b477..d2a2132c1 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -1,6 +1,7 @@ requireSandboxSupport [[ $busybox =~ busybox ]] || skipTest "no busybox" +# Avoid store dir being inside sandbox build-dir unset NIX_STORE_DIR unset NIX_STATE_DIR diff --git a/tests/supplementary-groups.sh b/tests/supplementary-groups.sh index 47c6ef605..474ba7503 100644 --- a/tests/supplementary-groups.sh +++ b/tests/supplementary-groups.sh @@ -5,6 +5,10 @@ requireSandboxSupport if ! command -p -v unshare; then skipTest "Need unshare"; fi needLocalStore "The test uses --store always so we would just be bypassing the daemon" +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + unshare --mount --map-root-user bash < Date: Thu, 13 Jul 2023 14:23:24 -0500 Subject: [PATCH 113/909] move unset NIX_STORE_DIR in supplementary-groups.sh to inside the unshare --- tests/supplementary-groups.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/supplementary-groups.sh b/tests/supplementary-groups.sh index 474ba7503..d18fb2414 100644 --- a/tests/supplementary-groups.sh +++ b/tests/supplementary-groups.sh @@ -5,13 +5,13 @@ requireSandboxSupport if ! command -p -v unshare; then skipTest "Need unshare"; fi needLocalStore "The test uses --store always so we would just be bypassing the daemon" -# Avoid store dir being inside sandbox build-dir -unset NIX_STORE_DIR -unset NIX_STATE_DIR - unshare --mount --map-root-user bash < Date: Thu, 13 Jul 2023 13:17:17 -0400 Subject: [PATCH 114/909] Test nested sandboxing, and make nicer error We were bedeviled by sandboxing issues when working on the layered store. The problem ended up being that when we have nested nix builds, and the inner store is inside the build dir (e.g. store is `/build/nix-test/$name/store`, build dir is `/build`) bind mounts clobber each other and store paths cannot be found. After thoroughly cleaning up `local-derivation-goal.cc`, we might be able to make that work. But that is a lot of work. For now, we just fail earlier with a proper error message. Finally, test this: nested sandboxing without the problematic store dir should work, and with should fail with the expected error message. Co-authored-by: Dylan Green <67574902+cidkidnix@users.noreply.github.com> Co-authored-by: Robert Hensing --- src/libstore/build/local-derivation-goal.cc | 4 +++ tests/local.mk | 3 ++- tests/nested-sandboxing.sh | 11 ++++++++ tests/nested-sandboxing/command.sh | 29 +++++++++++++++++++++ tests/nested-sandboxing/runner.nix | 24 +++++++++++++++++ 5 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 tests/nested-sandboxing.sh create mode 100644 tests/nested-sandboxing/command.sh create mode 100644 tests/nested-sandboxing/runner.nix diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index ee66ee500..e22a522a2 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -594,6 +594,10 @@ void LocalDerivationGoal::startBuilder() else dirsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; } + if (hasPrefix(worker.store.storeDir, tmpDirInSandbox)) + { + throw Error("`sandbox-build-dir` must not contain the storeDir"); + } dirsInChroot[tmpDirInSandbox] = tmpDir; /* Add the closure of store paths to the chroot. */ diff --git a/tests/local.mk b/tests/local.mk index bb80c5317..173bc84b3 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -138,7 +138,8 @@ nix_tests = \ path-from-hash-part.sh \ test-libstoreconsumer.sh \ toString-path.sh \ - read-only-store.sh + read-only-store.sh \ + nested-sandboxing.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh diff --git a/tests/nested-sandboxing.sh b/tests/nested-sandboxing.sh new file mode 100644 index 000000000..d9fa788aa --- /dev/null +++ b/tests/nested-sandboxing.sh @@ -0,0 +1,11 @@ +source common.sh +# This test is run by `tests/nested-sandboxing/runner.nix` in an extra layer of sandboxing. +[[ -d /nix/store ]] || skipTest "running this test without Nix's deps being drawn from /nix/store is not yet supported" + +requireSandboxSupport + +source ./nested-sandboxing/command.sh + +expectStderr 100 runNixBuild badStoreUrl 2 | grepQuiet '`sandbox-build-dir` must not contain' + +runNixBuild goodStoreUrl 5 diff --git a/tests/nested-sandboxing/command.sh b/tests/nested-sandboxing/command.sh new file mode 100644 index 000000000..69366486c --- /dev/null +++ b/tests/nested-sandboxing/command.sh @@ -0,0 +1,29 @@ +export NIX_BIN_DIR=$(dirname $(type -p nix)) +# TODO Get Nix and its closure more flexibly +export EXTRA_SANDBOX="/nix/store $(dirname $NIX_BIN_DIR)" + +badStoreUrl () { + local altitude=$1 + echo $TEST_ROOT/store-$altitude +} + +goodStoreUrl () { + local altitude=$1 + echo $("badStoreUrl" "$altitude")?store=/foo-$altitude +} + +# The non-standard sandbox-build-dir helps ensure that we get the same behavior +# whether this test is being run in a derivation as part of the nix build or +# being manually run by a developer outside a derivation +runNixBuild () { + local storeFun=$1 + local altitude=$2 + nix-build \ + --no-substitute --no-out-link \ + --store "$("$storeFun" "$altitude")" \ + --extra-sandbox-paths "$EXTRA_SANDBOX" \ + ./nested-sandboxing/runner.nix \ + --arg altitude "$((altitude - 1))" \ + --argstr storeFun "$storeFun" \ + --sandbox-build-dir /build-non-standard +} diff --git a/tests/nested-sandboxing/runner.nix b/tests/nested-sandboxing/runner.nix new file mode 100644 index 000000000..9a5822c88 --- /dev/null +++ b/tests/nested-sandboxing/runner.nix @@ -0,0 +1,24 @@ +{ altitude, storeFun }: + +with import ../config.nix; + +mkDerivation { + name = "nested-sandboxing"; + busybox = builtins.getEnv "busybox"; + EXTRA_SANDBOX = builtins.getEnv "EXTRA_SANDBOX"; + buildCommand = if altitude == 0 then '' + echo Deep enough! > $out + '' else '' + cp -r ${../common} ./common + cp ${../common.sh} ./common.sh + cp ${../config.nix} ./config.nix + cp -r ${./.} ./nested-sandboxing + + export PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH + + source common.sh + source ./nested-sandboxing/command.sh + + runNixBuild ${storeFun} ${toString altitude} >> $out + ''; +} From a5c88f860987bd5dec8c96efed1e6c9d8ce7a669 Mon Sep 17 00:00:00 2001 From: Sinan Mohd <69694713+sinanmohd@users.noreply.github.com> Date: Sun, 16 Jul 2023 09:25:11 +0000 Subject: [PATCH 115/909] Nix Reference Manual: keep nix expressions uptodate with nixpkgs (#8703) --- doc/manual/src/language/constructs.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md index c53eb8889..2482f2d17 100644 --- a/doc/manual/src/language/constructs.md +++ b/doc/manual/src/language/constructs.md @@ -92,10 +92,10 @@ In this fragment from `all-packages.nix`, ```nix graphviz = (import ../tools/graphics/graphviz) { inherit fetchurl stdenv libpng libjpeg expat x11 yacc; - inherit (xlibs) libXaw; + inherit (xorg) libXaw; }; -xlibs = { +xorg = { libX11 = ...; libXaw = ...; ... @@ -109,7 +109,7 @@ libjpg = ...; the set used in the function call to the function defined in `../tools/graphics/graphviz` inherits a number of variables from the surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` -(the X Athena Widgets) from the `xlibs` (X11 client-side libraries) set. +(the X Athena Widgets) from the `xorg` set. Summarizing the fragment From 259e328de81a91cf824efb0603f57fcde94ad3ff Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 9 Jul 2023 22:24:51 -0400 Subject: [PATCH 116/909] Introduce notion of a test group, use for CA tests Grouping our tests should make it easier to understand the intent than one long poorly-arranged list. It also is convenient for running just the tests for a specific component when working on that component. We need at least one test group so this isn't dead code; I decided to collect the tests for the `ca-derivations` and `dynamic-derivations` experimental features in groups. Do ```bash make ca.test-group -jN ``` and ```bash make dyn-drv.test-group -jN ``` to try running just them. I originally did this as part of #8397 for being able to just the local overlay store alone. I am PRing it separately now so we can separate general infra from new features. Co-authored-by: Valentin Gagarin --- Makefile | 2 ++ doc/manual/src/contributing/testing.md | 29 ++++++++++++++++++++++++++ mk/lib.mk | 20 +++++++++++++++++- mk/tests.mk | 8 +++++-- tests/ca/local.mk | 27 ++++++++++++++++++++++++ tests/dyn-drv/local.mk | 11 ++++++++++ tests/local.mk | 29 +++----------------------- 7 files changed, 97 insertions(+), 29 deletions(-) create mode 100644 tests/ca/local.mk create mode 100644 tests/dyn-drv/local.mk diff --git a/Makefile b/Makefile index c6220482a..31b54b93d 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,8 @@ makefiles += \ src/libstore/tests/local.mk \ src/libexpr/tests/local.mk \ tests/local.mk \ + tests/ca/local.mk \ + tests/dyn-drv/local.mk \ tests/test-libstoreconsumer/local.mk \ tests/plugins/local.mk else diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index a5253997d..c3c82e3c0 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -14,6 +14,8 @@ You can run the whole testsuite with `make check`, or the tests for a specific c The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. Each test is a bash script. +### Running the whole test suite + The whole test suite can be run with: ```shell-session @@ -23,6 +25,33 @@ ran test tests/bar.sh... [PASS] ... ``` +### Grouping tests + +Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite. +Each test group is in a subdirectory of `tests`. +For example, `tests/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs. + +That test group can be run like this: + +```shell-session +$ make ca.test-group -j50 +ran test tests/ca/nix-run.sh... [PASS] +ran test tests/ca/import-derivation.sh... [PASS] +... +``` + +The test group is defined in Make like this: +```makefile +$(test-group-name)-tests := \ + $(d)/test0.sh \ + $(d)/test1.sh \ + ... + +install-tests-groups += $(test-group-name) +``` + +### Running individual tests + Individual tests can be run with `make`: ```shell-session diff --git a/mk/lib.mk b/mk/lib.mk index 34fa624d8..e86a7f1a4 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -10,6 +10,7 @@ bin-scripts := noinst-scripts := man-pages := install-tests := +install-tests-groups := ifdef HOST_OS HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS))) @@ -121,7 +122,16 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) -$(foreach test, $(install-tests), $(eval $(call run-install-test,$(test)))) +$(foreach test, $(install-tests), \ + $(eval $(call run-install-test,$(test))) \ + $(eval installcheck: $(test).test)) +$(foreach test-group, $(install-tests-groups), \ + $(eval $(call run-install-test-group,$(test-group))) \ + $(eval installcheck: $(test-group).test-group) \ + $(foreach test, $($(test-group)-tests), \ + $(eval $(call run-install-test,$(test))) \ + $(eval $(test-group).test-group: $(test).test))) + $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file)))))) @@ -151,6 +161,14 @@ ifdef libs-list @echo "The following libraries can be built:" @echo "" @for i in $(libs-list); do echo " $$i"; done +endif +ifdef install-tests-groups + @echo "" + @echo "The following groups of functional tests can be run:" + @echo "" + @for i in $(install-tests-groups); do echo " $$i.test-group"; done + @echo "" + @echo "(installcheck includes tests in test groups too.)" endif @echo "" @echo "The following variables control the build:" diff --git a/mk/tests.mk b/mk/tests.mk index 3ebbd86e3..ec8128bdf 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -4,8 +4,6 @@ test-deps = define run-install-test - installcheck: $1.test - .PHONY: $1.test $1.test: $1 $(test-deps) @env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null @@ -16,6 +14,12 @@ define run-install-test endef +define run-install-test-group + + .PHONY: $1.test-group + +endef + .PHONY: check installcheck print-top-help += \ diff --git a/tests/ca/local.mk b/tests/ca/local.mk new file mode 100644 index 000000000..d15312708 --- /dev/null +++ b/tests/ca/local.mk @@ -0,0 +1,27 @@ +ca-tests := \ + $(d)/build-with-garbage-path.sh \ + $(d)/build.sh \ + $(d)/concurrent-builds.sh \ + $(d)/derivation-json.sh \ + $(d)/duplicate-realisation-in-closure.sh \ + $(d)/gc.sh \ + $(d)/import-derivation.sh \ + $(d)/new-build-cmd.sh \ + $(d)/nix-copy.sh \ + $(d)/nix-run.sh \ + $(d)/nix-shell.sh \ + $(d)/post-hook.sh \ + $(d)/recursive.sh \ + $(d)/repl.sh \ + $(d)/selfref-gc.sh \ + $(d)/signatures.sh \ + $(d)/substitute.sh \ + $(d)/why-depends.sh + +install-tests-groups += ca + +clean-files += \ + $(d)/config.nix + +test-deps += \ + tests/ca/config.nix diff --git a/tests/dyn-drv/local.mk b/tests/dyn-drv/local.mk new file mode 100644 index 000000000..f065a5627 --- /dev/null +++ b/tests/dyn-drv/local.mk @@ -0,0 +1,11 @@ +dyn-drv-tests := \ + $(d)/text-hashed-output.sh \ + $(d)/recursive-mod-json.sh + +install-tests-groups += dyn-drv + +clean-files += \ + $(d)/config.nix + +test-deps += \ + tests/dyn-drv/config.nix diff --git a/tests/local.mk b/tests/local.mk index df20f3dd7..2afe91220 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -14,7 +14,6 @@ nix_tests = \ flakes/absolute-paths.sh \ flakes/build-paths.sh \ flakes/flake-in-submodule.sh \ - ca/gc.sh \ gc.sh \ nix-collect-garbage-d.sh \ remote-store.sh \ @@ -28,8 +27,6 @@ nix_tests = \ user-envs-migration.sh \ binary-cache.sh \ multiple-outputs.sh \ - ca/build.sh \ - ca/new-build-cmd.sh \ nix-build.sh \ gc-concurrent.sh \ repair.sh \ @@ -47,24 +44,17 @@ nix_tests = \ referrers.sh \ optimise-store.sh \ substitute-with-invalid-ca.sh \ - ca/concurrent-builds.sh \ signing.sh \ - ca/build-with-garbage-path.sh \ hash.sh \ gc-non-blocking.sh \ check.sh \ - ca/substitute.sh \ nix-shell.sh \ - ca/signatures.sh \ - ca/nix-shell.sh \ - ca/nix-copy.sh \ check-refs.sh \ build-remote-input-addressed.sh \ secure-drv-outputs.sh \ restricted.sh \ fetchGitSubmodules.sh \ flakes/search-root.sh \ - ca/duplicate-realisation-in-closure.sh \ readfile-context.sh \ nix-channel.sh \ recursive.sh \ @@ -80,10 +70,7 @@ nix_tests = \ nar-access.sh \ pure-eval.sh \ eval.sh \ - ca/post-hook.sh \ repl.sh \ - ca/repl.sh \ - ca/recursive.sh \ binary-cache-build-remote.sh \ search.sh \ logging.sh \ @@ -109,13 +96,8 @@ nix_tests = \ fmt.sh \ eval-store.sh \ why-depends.sh \ - ca/why-depends.sh \ derivation-json.sh \ - ca/derivation-json.sh \ import-derivation.sh \ - ca/import-derivation.sh \ - dyn-drv/text-hashed-output.sh \ - dyn-drv/recursive-mod-json.sh \ nix_path.sh \ case-hack.sh \ placeholders.sh \ @@ -124,8 +106,7 @@ nix_tests = \ build.sh \ build-delete.sh \ output-normalization.sh \ - ca/nix-run.sh \ - selfref-gc.sh ca/selfref-gc.sh \ + selfref-gc.sh \ db-migration.sh \ bash-profile.sh \ pass-as-file.sh \ @@ -150,16 +131,12 @@ install-tests += $(foreach x, $(nix_tests), $(d)/$(x)) clean-files += \ $(d)/common/vars-and-functions.sh \ - $(d)/config.nix \ - $(d)/ca/config.nix \ - $(d)/dyn-drv/config.nix + $(d)/config.nix test-deps += \ tests/common/vars-and-functions.sh \ tests/config.nix \ - tests/ca/config.nix \ - tests/test-libstoreconsumer/test-libstoreconsumer \ - tests/dyn-drv/config.nix + tests/test-libstoreconsumer/test-libstoreconsumer ifeq ($(BUILD_SHARED_LIBS), 1) test-deps += tests/plugins/libplugintest.$(SO_EXT) From ee72ede389a7a8b9ac2d0f908b2524950d72f303 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sun, 5 Mar 2023 03:59:17 +0100 Subject: [PATCH 117/909] remove the Channels section this is a how-to guide which should not be in the reference manual. it also refers to `nix-env`, which should not be the first thing readers of the reference manual encounter, as it behaves very differently in spirit from the rest of Nix. slightly reword the documentation to be more concise and informative. --- doc/manual/src/SUMMARY.md.in | 1 - doc/manual/src/command-ref/nix-channel.md | 70 +++++++++++-------- .../package-management/basic-package-mgmt.md | 2 +- doc/manual/src/package-management/channels.md | 50 ------------- 4 files changed, 41 insertions(+), 82 deletions(-) delete mode 100644 doc/manual/src/package-management/channels.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 1bd8fa774..f2e69178f 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -21,7 +21,6 @@ - [Profiles](package-management/profiles.md) - [Garbage Collection](package-management/garbage-collection.md) - [Garbage Collector Roots](package-management/garbage-collector-roots.md) - - [Channels](package-management/channels.md) - [Sharing Packages Between Machines](package-management/sharing-packages.md) - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) - [Copying Closures via SSH](package-management/copy-closure.md) diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md index 025f758e7..8c0dcdfaa 100644 --- a/doc/manual/src/command-ref/nix-channel.md +++ b/doc/manual/src/command-ref/nix-channel.md @@ -8,36 +8,41 @@ # Description -A Nix channel is a mechanism that allows you to automatically stay -up-to-date with a set of pre-built Nix expressions. A Nix channel is -just a URL that points to a place containing a set of Nix expressions. +Channels are a mechanism for referencing remote Nix expressions and conveniently retrieving their latest version. +For the list of official channels, visit . -To see the list of official NixOS channels, visit -. +> **Note** +> +> The state of a subscribed channel is external to the Nix expressions relying on it. +> This may limit reproducibility. +> +> Dependencies on other Nix expressions can be declared explicitly with: +> - [`fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl), [`fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball), or [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) in Nix expressions +> - the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) in command line invocations This command has the following operations: - `--add` *url* \[*name*\]\ - Adds a channel named *name* with URL *url* to the list of subscribed - channels. If *name* is omitted, it defaults to the last component of - *url*, with the suffixes `-stable` or `-unstable` removed. + Add a channel *name* located at *url* to the list of subscribed channels. + If *name* is omitted, default to the last component of *url*, with the suffixes `-stable` or `-unstable` removed. + + > **Note** + > + > `--add` does not automatically perform an update. + > Use `--update` explicitly. A channel URL must point to a directory containing a file `nixexprs.tar.gz`. At the top level, that tarball must contain a single directory with a `default.nix` file that serves as the channel’s entry point. - `--remove` *name*\ - Removes the channel named *name* from the list of subscribed - channels. + Remove the channel *name* from the list of subscribed channels. - `--list`\ - Prints the names and URLs of all subscribed channels on standard - output. + Print the names and URLs of all subscribed channels on standard output. - `--update` \[*names*…\]\ - Downloads the Nix expressions of all subscribed channels (or only - those included in *names* if specified) and makes them the default - for `nix-env` operations (by symlinking them from the directory - `~/.nix-defexpr`). + Download the Nix expressions of subscribed channels and create a new generation. + Update all channels if none is specified, and only those included in *names* otherwise. - `--list-generations`\ Prints a list of all the current existing generations for the @@ -49,13 +54,8 @@ This command has the following operations: ``` - `--rollback` \[*generation*\]\ - Reverts the previous call to `nix-channel - --update`. Optionally, you can specify a specific channel generation - number to restore. - -Note that `--add` does not automatically perform an update. - -The list of subscribed channels is stored in `~/.nix-channels`. + Revert channels to the state before the last call to `nix-channel --update`. + Optionally, you can specify a specific channel *generation* number to restore. {{#include ./opt-common.md}} @@ -69,23 +69,33 @@ The list of subscribed channels is stored in `~/.nix-channels`. # Examples -To subscribe to the Nixpkgs channel and install the GNU Hello package: +Subscribe to the Nixpkgs channel and run `hello` from the GNU Hello package: ```console $ nix-channel --add https://nixos.org/channels/nixpkgs-unstable +$ nix-channel --list +nixpkgs https://nixos.org/channels/nixpkgs $ nix-channel --update -$ nix-env --install --attr nixpkgs.hello +$ nix-shell -p hello --run hello +hello ``` -You can revert channel updates using `--rollback`: +Revert channel updates using `--rollback`: ```console -$ nix-instantiate --eval --expr '(import {}).lib.version' -"14.04.527.0e935f1" +$ nix-instantiate --eval '' --attr lib.version +"22.11pre296212.530a53dcbc9" $ nix-channel --rollback switching from generation 483 to 482 -$ nix-instantiate --eval --expr '(import {}).lib.version' -"14.04.526.dbadfad" +$ nix-instantiate --eval '' --attr lib.version +"22.11pre281526.d0419badfad" +``` + +Remove a channel: + +```console +$ nix-channel --remove nixpkgs +$ nix-channel --list ``` diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md index 6b86e763e..07b92fb76 100644 --- a/doc/manual/src/package-management/basic-package-mgmt.md +++ b/doc/manual/src/package-management/basic-package-mgmt.md @@ -25,7 +25,7 @@ or completely new ones.) You can manually download the latest version of Nixpkgs from . However, it’s much more -convenient to use the Nixpkgs [*channel*](channels.md), since it makes +convenient to use the Nixpkgs [*channel*](../command-ref/nix-channel.md), since it makes it easy to stay up to date with new versions of Nixpkgs. Nixpkgs is automatically added to your list of “subscribed” channels when you install Nix. If this is not the case for some reason, you can add it diff --git a/doc/manual/src/package-management/channels.md b/doc/manual/src/package-management/channels.md deleted file mode 100644 index 8e4da180b..000000000 --- a/doc/manual/src/package-management/channels.md +++ /dev/null @@ -1,50 +0,0 @@ -# Channels - -If you want to stay up to date with a set of packages, it’s not very -convenient to manually download the latest set of Nix expressions for -those packages and upgrade using `nix-env`. Fortunately, there’s a -better way: *Nix channels*. - -A Nix channel is just a URL that points to a place that contains a set -of Nix expressions and a manifest. Using the command -[`nix-channel`](../command-ref/nix-channel.md) you can automatically -stay up to date with whatever is available at that URL. - -To see the list of official NixOS channels, visit -. - -You can “subscribe” to a channel using `nix-channel --add`, e.g., - -```console -$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable -``` - -subscribes you to a channel that always contains that latest version of -the Nix Packages collection. (Subscribing really just means that the URL -is added to the file `~/.nix-channels`, where it is read by subsequent -calls to `nix-channel ---update`.) You can “unsubscribe” using `nix-channel ---remove`: - -```console -$ nix-channel --remove nixpkgs -``` - -To obtain the latest Nix expressions available in a channel, do - -```console -$ nix-channel --update -``` - -This downloads and unpacks the Nix expressions in every channel -(downloaded from `url/nixexprs.tar.bz2`). It also makes the union of -each channel’s Nix expressions available by default to `nix-env` -operations (via the symlink `~/.nix-defexpr/channels`). Consequently, -you can then say - -```console -$ nix-env --upgrade -``` - -to upgrade all packages in your profile to the latest versions available -in the subscribed channels. From cd0e39bd899b51d0b1f139117b1fd6c1280caa70 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 21 Jun 2023 11:54:23 +0200 Subject: [PATCH 118/909] remove redundant information from channel profile description --- doc/manual/src/command-ref/files/channels.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/manual/src/command-ref/files/channels.md b/doc/manual/src/command-ref/files/channels.md index 7b1f27128..691468cf8 100644 --- a/doc/manual/src/command-ref/files/channels.md +++ b/doc/manual/src/command-ref/files/channels.md @@ -1,11 +1,10 @@ ## Channels -A directory containing symlinks to Nix channels, managed by [`nix-channel`]: +[`nix-channel`] uses a [profile](@docroot@/command-ref/files/profiles.md) to store channels: - `$XDG_STATE_HOME/nix/profiles/channels` for regular users - `$NIX_STATE_DIR/profiles/per-user/root/channels` for `root` -[`nix-channel`] uses a [profile](@docroot@/command-ref/files/profiles.md) to store channels. This profile contains symlinks to the contents of those channels. ## Subscribed channels From 4bab5a62081a792530de8c4ae2cd163c1bb581a5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 09:42:53 +0200 Subject: [PATCH 119/909] revert channel files overview --- doc/manual/src/command-ref/files/channels.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/files/channels.md b/doc/manual/src/command-ref/files/channels.md index 691468cf8..7b1f27128 100644 --- a/doc/manual/src/command-ref/files/channels.md +++ b/doc/manual/src/command-ref/files/channels.md @@ -1,10 +1,11 @@ ## Channels -[`nix-channel`] uses a [profile](@docroot@/command-ref/files/profiles.md) to store channels: +A directory containing symlinks to Nix channels, managed by [`nix-channel`]: - `$XDG_STATE_HOME/nix/profiles/channels` for regular users - `$NIX_STATE_DIR/profiles/per-user/root/channels` for `root` +[`nix-channel`] uses a [profile](@docroot@/command-ref/files/profiles.md) to store channels. This profile contains symlinks to the contents of those channels. ## Subscribed channels From e14c8a359efcacd267064b9c7d28adb0f41a168e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 10:25:05 +0200 Subject: [PATCH 120/909] list moving parts of channels --- doc/manual/src/command-ref/nix-channel.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md index 8c0dcdfaa..cebbc7b00 100644 --- a/doc/manual/src/command-ref/nix-channel.md +++ b/doc/manual/src/command-ref/nix-channel.md @@ -9,7 +9,12 @@ # Description Channels are a mechanism for referencing remote Nix expressions and conveniently retrieving their latest version. -For the list of official channels, visit . + +The moving parts of channels are: +- The official channels listed at +- The user-specific list of [subscribed channels](#subscribed-channels) +- The [downloaded channel contents](#channels) +- The [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path), set with the [`-I` option](#opt-i) or the [`NIX_PATH` environment variable](#env-NIX_PATH) > **Note** > From 68b7bb1a062c6471b678d5488a71324e0486d405 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 11 May 2023 15:51:48 +0200 Subject: [PATCH 121/909] add information on the system type string --- configure.ac | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index bb3f92e4d..26df39d54 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,12 @@ AC_CONFIG_AUX_DIR(config) AC_PROG_SED -# Construct a Nix system name (like "i686-linux"). +# Construct a Nix system name (like "i686-linux"): +# https://www.gnu.org/software/autoconf/manual/html_node/Canonicalizing.html#index-AC_005fCANONICAL_005fHOST-1 +# The inital value is produced by the `config.guess` script: +# https://git.savannah.gnu.org/cgit/config.git/tree/config.guess +# It has the following form, which is not documented anywhere: +# --[][-] AC_CANONICAL_HOST AC_MSG_CHECKING([for the canonical Nix system name]) From 0751c1bfc6375ffbe94e90cfcfdb987e8b286f2e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 31 May 2023 03:21:11 +0200 Subject: [PATCH 122/909] one line per sentence for easier review --- src/libstore/globals.hh | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d4b8fb1f9..76afeb5f5 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -193,18 +193,12 @@ public: Setting thisSystem{ this, SYSTEM, "system", R"( - This option specifies the canonical Nix system name of the current - installation, such as `i686-linux` or `x86_64-darwin`. Nix can only - build derivations whose `system` attribute equals the value - specified here. In general, it never makes sense to modify this - value from its default, since you can use it to ‘lie’ about the - platform you are building on (e.g., perform a Mac OS build on a - Linux machine; the result would obviously be wrong). It only makes - sense if the Nix binaries can run on multiple platforms, e.g., - ‘universal binaries’ that run on `x86_64-linux` and `i686-linux`. + This option specifies the canonical Nix system name of the current installation, such as `i686-linux` or `x86_64-darwin`. + Nix can only build derivations whose `system` attribute equals the value specified here. + In general, it never makes sense to modify this value from its default, since you can use it to ‘lie’ about the platform you are building on (e.g., perform a Mac OS build on a Linux machine; the result would obviously be wrong). + It only makes sense if the Nix binaries can run on multiple platforms, e.g., ‘universal binaries’ that run on `x86_64-linux` and `i686-linux`. - It defaults to the canonical Nix system name detected by `configure` - at build time. + It defaults to the canonical Nix system name detected by `configure` at build time. )"}; Setting maxSilentTime{ From c8a42039eae4c046297517b67a3ce43685d3e756 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 31 May 2023 03:48:05 +0200 Subject: [PATCH 123/909] move docs of the current system to the system setting add information what happens when Nix itself is cross-compiled --- configure.ac | 2 ++ src/libstore/globals.hh | 45 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 26df39d54..cc7009859 100644 --- a/configure.ac +++ b/configure.ac @@ -11,6 +11,8 @@ AC_PROG_SED # https://git.savannah.gnu.org/cgit/config.git/tree/config.guess # It has the following form, which is not documented anywhere: # --[][-] +# If `./configure` is passed any of the `--host`, `--build`, `--target` options, the value comes from `config.sub` instead: +# https://git.savannah.gnu.org/cgit/config.git/tree/config.sub AC_CANONICAL_HOST AC_MSG_CHECKING([for the canonical Nix system name]) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 76afeb5f5..1fb5927d0 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -193,12 +193,49 @@ public: Setting thisSystem{ this, SYSTEM, "system", R"( - This option specifies the canonical Nix system name of the current installation, such as `i686-linux` or `x86_64-darwin`. - Nix can only build derivations whose `system` attribute equals the value specified here. - In general, it never makes sense to modify this value from its default, since you can use it to ‘lie’ about the platform you are building on (e.g., perform a Mac OS build on a Linux machine; the result would obviously be wrong). + The system type of the current Nix installation. + + Nix can only build [derivations](@docroot@/language/derivations.md) whose `system` attribute equals the value specified here. + In general, it never makes sense to modify this value, since you can use it to ‘lie’ about the system you are building on (e.g., perform a macOS build on a Linux machine; the result would obviously be wrong). It only makes sense if the Nix binaries can run on multiple platforms, e.g., ‘universal binaries’ that run on `x86_64-linux` and `i686-linux`. - It defaults to the canonical Nix system name detected by `configure` at build time. + The default value is set when Nix itself is compiled for the system it will run on. + The following system types are widely used, as Nix is actively supported on these platforms: + + - `x86_64-linux` + - `x86_64-darwin` + - `i686-linux` + - `aarch64-linux` + - `aarch64-darwin` + + The concrete value is based on the output of [`config.guess`](https://git.savannah.gnu.org/cgit/config.git/tree/config.guess): + + ``` + --[][-] + ``` + + When Nix is built such that `./configure` is passed any of the `--host`, `--build`, `--target` options, the value is based on the output of [`config.sub`](https://git.savannah.gnu.org/cgit/config.git/tree/config.sub): + + ``` + -[-]- + ``` + + Nix only uses the CPU and OS identifiers: + + ``` + -[-] + ``` + + For historic reasons and backwards-compatibility, some CPU and OS identifiers are transformed as follows: + + | `config.guess` | Nix | + |----------------------------|---------------------| + | `amd64` | `x86_64` | + | `i*86` | `i686` | + | `arm6` | `arm6l` | + | `arm7` | `arm7l` | + | `linux-gnu*` | `linux` | + | `linux-musl*` | `linux` | )"}; Setting maxSilentTime{ From 3763c7bb5ef5a7a1ddcf47169c9733edb7989e98 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 20 Jun 2023 13:42:23 +0200 Subject: [PATCH 124/909] shorten `system` setting description --- src/libstore/globals.hh | 38 ++++++-------------------------------- 1 file changed, 6 insertions(+), 32 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 1fb5927d0..442d72911 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -194,12 +194,12 @@ public: this, SYSTEM, "system", R"( The system type of the current Nix installation. - - Nix can only build [derivations](@docroot@/language/derivations.md) whose `system` attribute equals the value specified here. - In general, it never makes sense to modify this value, since you can use it to ‘lie’ about the system you are building on (e.g., perform a macOS build on a Linux machine; the result would obviously be wrong). - It only makes sense if the Nix binaries can run on multiple platforms, e.g., ‘universal binaries’ that run on `x86_64-linux` and `i686-linux`. + Nix will only build [derivations](@docroot@/language/derivations.md) whose `system` attribute equals the value specified here. The default value is set when Nix itself is compiled for the system it will run on. + In general, it never makes sense to modify this value. + While you can force Nix to run a Darwin-specific `builder` executable on a Linux machine, the result would obviously be wrong. + The following system types are widely used, as Nix is actively supported on these platforms: - `x86_64-linux` @@ -207,35 +207,9 @@ public: - `i686-linux` - `aarch64-linux` - `aarch64-darwin` + - `armv6l-linux` + - `armv7l-linux` - The concrete value is based on the output of [`config.guess`](https://git.savannah.gnu.org/cgit/config.git/tree/config.guess): - - ``` - --[][-] - ``` - - When Nix is built such that `./configure` is passed any of the `--host`, `--build`, `--target` options, the value is based on the output of [`config.sub`](https://git.savannah.gnu.org/cgit/config.git/tree/config.sub): - - ``` - -[-]- - ``` - - Nix only uses the CPU and OS identifiers: - - ``` - -[-] - ``` - - For historic reasons and backwards-compatibility, some CPU and OS identifiers are transformed as follows: - - | `config.guess` | Nix | - |----------------------------|---------------------| - | `amd64` | `x86_64` | - | `i*86` | `i686` | - | `arm6` | `arm6l` | - | `arm7` | `arm7l` | - | `linux-gnu*` | `linux` | - | `linux-musl*` | `linux` | )"}; Setting maxSilentTime{ From 4944e37ec04cc8c62f35aa9b1f4e97ee8f233eb8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 20 Jun 2023 14:10:30 +0200 Subject: [PATCH 125/909] expand on the system type in hacking guide --- configure.ac | 8 +-- doc/manual/src/contributing/hacking.md | 69 +++++++++++++++++++------- 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/configure.ac b/configure.ac index cc7009859..6d78237f0 100644 --- a/configure.ac +++ b/configure.ac @@ -7,12 +7,12 @@ AC_PROG_SED # Construct a Nix system name (like "i686-linux"): # https://www.gnu.org/software/autoconf/manual/html_node/Canonicalizing.html#index-AC_005fCANONICAL_005fHOST-1 -# The inital value is produced by the `config.guess` script: -# https://git.savannah.gnu.org/cgit/config.git/tree/config.guess +# The inital value is produced by the `config/config.guess` script: +# upstream: https://git.savannah.gnu.org/cgit/config.git/tree/config.guess # It has the following form, which is not documented anywhere: # --[][-] -# If `./configure` is passed any of the `--host`, `--build`, `--target` options, the value comes from `config.sub` instead: -# https://git.savannah.gnu.org/cgit/config.git/tree/config.sub +# If `./configure` is passed any of the `--host`, `--build`, `--target` options, the value comes from `config/config.sub` instead: +# upstream: https://git.savannah.gnu.org/cgit/config.git/tree/config.sub AC_CANONICAL_HOST AC_MSG_CHECKING([for the canonical Nix system name]) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 7b2440971..c7cb0d980 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -110,41 +110,72 @@ You can also build Nix for one of the [supported platforms](#platforms). ## Platforms -As specified in [`flake.nix`], Nix can be built for various platforms: - -- `aarch64-linux` -- `i686-linux` -- `x86_64-darwin` -- `x86_64-linux` +Nix can be built for various platforms, as specified in [`flake.nix`]: [`flake.nix`]: https://github.com/nixos/nix/blob/master/flake.nix +- `x86_64-linux` +- `x86_64-darwin` +- `i686-linux` +- `aarch64-linux` +- `aarch64-darwin` +- `armv6l-linux` +- `armv7l-linux` + In order to build Nix for a different platform than the one you're currently -on, you need to have some way for your system Nix to build code for that -platform. Common solutions include [remote builders] and [binfmt emulation] +on, you need a way for your current Nix installation to build code for that +platform. Common solutions include [remote builders] and [binary format emulation] (only supported on NixOS). [remote builders]: ../advanced-topics/distributed-builds.md [binfmt emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems -These solutions let Nix perform builds as if you're on the native platform, so -executing the build is as simple as - -```console -$ nix build .#packages.aarch64-linux.default -``` - -for flake-enabled Nix, or +Given such a setup, executing the build only requires selecting the respective output attribute. +For example, to compile for `aarch64-linux`: ```console $ nix-build --attr packages.aarch64-linux.default ``` -for classic Nix. +or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled: -You can use any of the other supported platforms in place of `aarch64-linux`. +```console +$ nix build .#packages.aarch64-linux.default +``` -Cross-compiled builds are available for ARMv6 and ARMv7, and Nix on unsupported platforms can be bootstrapped by adding more `crossSystems` in `flake.nix`. +Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`). +Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms. + +## System type + +Nix uses a string with he following format to identify the *system type* or *platform* it runs on: + +``` +-[-] +``` + +It is set when Nix is compiled for the given system, and based on the output of [`config.guess`](https://github.com/nixos/nix/blob/master/config/config.guess) ([upstream](https://git.savannah.gnu.org/cgit/config.git/tree/config.guess)): + +``` +--[][-] +``` + +When Nix is built such that `./configure` is passed any of the `--host`, `--build`, `--target` options, the value is based on the output of [`config.sub`](https://github.com/nixos/nix/blob/master/config/config.sub) ([upstream](https://git.savannah.gnu.org/cgit/config.git/tree/config.sub)): + +``` +-[-]- +``` + +For historic reasons and backward-compatibility, some CPU and OS identifiers are transformed as follows in [`configure.ac`](https://github.com/nixos/nix/blob/master/config/config.sub): + +| `config.guess` | Nix | +|----------------------------|---------------------| +| `amd64` | `x86_64` | +| `i*86` | `i686` | +| `arm6` | `arm6l` | +| `arm7` | `arm7l` | +| `linux-gnu*` | `linux` | +| `linux-musl*` | `linux` | ## Compilation environments From 32de11923e932b5bbda8ce4877542fb299de9b03 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 22 Jun 2023 13:50:00 +0200 Subject: [PATCH 126/909] add cross-links --- src/libstore/globals.hh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 442d72911..760655302 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -197,10 +197,7 @@ public: Nix will only build [derivations](@docroot@/language/derivations.md) whose `system` attribute equals the value specified here. The default value is set when Nix itself is compiled for the system it will run on. - In general, it never makes sense to modify this value. - While you can force Nix to run a Darwin-specific `builder` executable on a Linux machine, the result would obviously be wrong. - - The following system types are widely used, as Nix is actively supported on these platforms: + The following system types are widely used, as [Nix is actively supported on these platforms](@docroot@/contributing/hacking.md#platforms): - `x86_64-linux` - `x86_64-darwin` @@ -210,6 +207,10 @@ public: - `armv6l-linux` - `armv7l-linux` + In general, it never makes sense to modify this value when building derivations. + While you can force Nix to run a Darwin-specific `builder` executable on a Linux machine, the result would obviously be wrong. + + This value is available in the Nix language as [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem). )"}; Setting maxSilentTime{ From c8f04e2024d3e66f620ae38258fa86eb6103ce05 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 13 Jul 2023 22:00:16 +0200 Subject: [PATCH 127/909] note that naming convention is from Autotools Co-authored-by: Robert Hensing --- doc/manual/src/contributing/hacking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index c7cb0d980..ff4690f1e 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -166,7 +166,7 @@ When Nix is built such that `./configure` is passed any of the `--host`, `--buil -[-]- ``` -For historic reasons and backward-compatibility, some CPU and OS identifiers are transformed as follows in [`configure.ac`](https://github.com/nixos/nix/blob/master/config/config.sub): +For historic reasons and backward-compatibility, some CPU and OS identifiers are translated from the GNU Autotools naming convention in [`configure.ac`](https://github.com/nixos/nix/blob/master/config/config.sub) as follows: | `config.guess` | Nix | |----------------------------|---------------------| From 1a220bed9361be2c360f83eb42025f9765c96f34 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 18 Jul 2023 23:21:43 +0200 Subject: [PATCH 128/909] do not mention output attributes Co-authored-by: Robert Hensing --- doc/manual/src/contributing/hacking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index ff4690f1e..b8ea977d7 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -130,7 +130,7 @@ platform. Common solutions include [remote builders] and [binary format emulatio [remote builders]: ../advanced-topics/distributed-builds.md [binfmt emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems -Given such a setup, executing the build only requires selecting the respective output attribute. +Given such a setup, executing the build only requires selecting the respective attribute. For example, to compile for `aarch64-linux`: ```console From aba32def7390432855401b815ff21915aba3eb81 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 18 Jul 2023 23:26:36 +0200 Subject: [PATCH 129/909] fix wording Co-authored-by: Robert Hensing --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 760655302..63ac54e97 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -207,7 +207,7 @@ public: - `armv6l-linux` - `armv7l-linux` - In general, it never makes sense to modify this value when building derivations. + In general, you do not have to modify this setting. While you can force Nix to run a Darwin-specific `builder` executable on a Linux machine, the result would obviously be wrong. This value is available in the Nix language as [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem). From fcadac0a02b30d1d876c804edf7f2745d880a10c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 10:33:41 +0200 Subject: [PATCH 130/909] mention `extra-platforms` --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 63ac54e97..ad02bfb42 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -194,7 +194,7 @@ public: this, SYSTEM, "system", R"( The system type of the current Nix installation. - Nix will only build [derivations](@docroot@/language/derivations.md) whose `system` attribute equals the value specified here. + Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals the value specified here or in [`extra-platforms`](@docroot@/command-ref/conf-file.html#conf-extra-platforms). The default value is set when Nix itself is compiled for the system it will run on. The following system types are widely used, as [Nix is actively supported on these platforms](@docroot@/contributing/hacking.md#platforms): From 0779005f4945d373b81c933a6b7390c8bc7ad6cb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 10:57:37 +0200 Subject: [PATCH 131/909] expand on the `extra-platforms` option --- src/libstore/globals.hh | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ad02bfb42..723d18d74 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -194,7 +194,7 @@ public: this, SYSTEM, "system", R"( The system type of the current Nix installation. - Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals the value specified here or in [`extra-platforms`](@docroot@/command-ref/conf-file.html#conf-extra-platforms). + Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in [`extra-platforms`](#conf-extra-platforms). The default value is set when Nix itself is compiled for the system it will run on. The following system types are widely used, as [Nix is actively supported on these platforms](@docroot@/contributing/hacking.md#platforms): @@ -676,18 +676,20 @@ public: getDefaultExtraPlatforms(), "extra-platforms", R"( - Platforms other than the native one which this machine is capable of - building for. This can be useful for supporting additional - architectures on compatible machines: i686-linux can be built on - x86\_64-linux machines (and the default for this setting reflects - this); armv7 is backwards-compatible with armv6 and armv5tel; some - aarch64 machines can also natively run 32-bit ARM code; and - qemu-user may be used to support non-native platforms (though this - may be slow and buggy). Most values for this are not enabled by - default because build systems will often misdetect the target - platform and generate incompatible code, so you may wish to - cross-check the results of using this option against proper - natively-built versions of your derivations. + System types of executables that can be run on this machine. + + Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in the [`system` option](#conf-system). + + Setting this can be useful to build derivations locally on compatible machines: + - `i686-linux` executables can be run on `x86_64-linux` machines (set by default) + - `x86_64-darwin` executables can be run on macOS `aarch64-darwin` with Rosetta 2 (set by default where applicable) + - `armv6` and `armv5tel` executables can be run on `armv7` + - some `aarch64` machines can also natively run 32-bit ARM code + - `qemu-user` may be used to support non-native platforms (though this + may be slow and buggy) + + Build systems will usually detect the target platform to be the current physical system and therefore produce machine code incompatible with what may be intended in the derivation. + You should design your derivation's `builder` accordingly and cross-check the results when using this option against natively-built versions of your derivation. )", {}, false}; Setting systemFeatures{ From 6c3cd429a6aabe8673af99d6bc0653d376dd69b8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 11:01:48 +0200 Subject: [PATCH 132/909] fix broken links --- doc/manual/redirects.js | 2 +- src/libstore/globals.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index dcdb5d6e9..b43622ed6 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -281,7 +281,7 @@ const redirects = { "chap-introduction": "introduction.html", "ch-basic-package-mgmt": "package-management/basic-package-mgmt.html", "ssec-binary-cache-substituter": "package-management/binary-cache-substituter.html", - "sec-channels": "package-management/channels.html", + "sec-channels": "command-ref/nix-channel.html", "ssec-copy-closure": "package-management/copy-closure.html", "sec-garbage-collection": "package-management/garbage-collection.html", "ssec-gc-roots": "package-management/garbage-collector-roots.html", diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d4b8fb1f9..2d8cac656 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1010,7 +1010,7 @@ public: | `~/.nix-defexpr` | `$XDG_STATE_HOME/nix/defexpr` | | `~/.nix-channels` | `$XDG_STATE_HOME/nix/channels` | - If you already have Nix installed and are using [profiles](@docroot@/package-management/profiles.md) or [channels](@docroot@/package-management/channels.md), you should migrate manually when you enable this option. + If you already have Nix installed and are using [profiles](@docroot@/package-management/profiles.md) or [channels](@docroot@/command-ref/nix-channel.md), you should migrate manually when you enable this option. If `$XDG_STATE_HOME` is not set, use `$HOME/.local/state/nix` instead of `$XDG_STATE_HOME/nix`. This can be achieved with the following shell commands: From feb01b22ed2954232455b5c8346cc5ed09c21f81 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 17 Jul 2023 11:28:12 +0200 Subject: [PATCH 133/909] add links to store API documentation --- src/libstore/store-api.hh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 14a862eef..43aa00637 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -136,19 +136,22 @@ struct StoreConfig : public Config Setting priority{this, 0, "priority", R"( - Priority of this store when used as a substituter. A lower value means a higher priority. + Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + A lower value means a higher priority. )"}; Setting wantMassQuery{this, false, "want-mass-query", R"( - Whether this store (when used as a substituter) can be - queried efficiently for path validity. + Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). )"}; Setting systemFeatures{this, getDefaultSystemFeatures(), "system-features", - "Optional features that the system this store builds on implements (like \"kvm\")."}; + R"( + Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. + Example: `"kvm"` + )" }; }; class Store : public std::enable_shared_from_this, public virtual StoreConfig From 493ddf617f0b2c406dc35026b6d481a69cf1d934 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 17 Jul 2023 11:28:27 +0200 Subject: [PATCH 134/909] reformat `system-features` setting documentation --- src/libstore/globals.hh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d4b8fb1f9..879588375 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -689,19 +689,21 @@ public: getDefaultSystemFeatures(), "system-features", R"( - A set of system “features” supported by this machine, e.g. `kvm`. - Derivations can express a dependency on such features through the - derivation attribute `requiredSystemFeatures`. For example, the - attribute + A set of system “features” supported by this machine. + Derivations can express a dependency on such features through the derivation attribute `requiredSystemFeatures`. + + For example, the attribute requiredSystemFeatures = [ "kvm" ]; - ensures that the derivation can only be built on a machine with the - `kvm` feature. + ensures that the derivation can only be built on a machine with the `kvm` feature. - This setting by default includes `kvm` if `/dev/kvm` is accessible, - and the pseudo-features `nixos-test`, `benchmark` and `big-parallel` - that are used in Nixpkgs to route builds to specific machines. + This setting by default includes + - `kvm` if `/dev/kvm` is accessible + - historical pseudo-features for backwards compatibility, used in Nixpkgs to route Hydra builds to specific machines + - `nixos-test` + - `benchmark` + - `big-parallel` )", {}, false}; Setting substituters{ From 5f37ebcf83c856b08a63778e07bfc5c84ea4a5ec Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 11:57:20 +0200 Subject: [PATCH 135/909] document all special system features and their behavior --- .../src/language/advanced-attributes.md | 12 ++++++ doc/manual/src/release-notes/rl-2.12.md | 14 +------ src/libstore/globals.hh | 39 +++++++++++++------ 3 files changed, 41 insertions(+), 24 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 307971434..2961f9dcc 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -345,3 +345,15 @@ Derivations can declare some infrequently used optional attributes. This is useful, for example, when generating self-contained filesystem images with their own embedded Nix store: hashes found inside such an image refer to the embedded store and not to the host's Nix store. + +- [`requiredSystemFeatures`]{#adv-attr-requiredSystemFeatures}\ + + If a derivation has the `requiredSystemFeatures` attribute, then Nix will only build it on a machine that has the corresponding features set in its [`system-features` configuration](@docroot@/command-ref/conf-file.md#conf-system-features). + + For example, setting + + ```nix + requiredSystemFeatures = [ "kvm" ]; + ``` + + ensures that the derivation can only be built on a machine with the `kvm` feature. diff --git a/doc/manual/src/release-notes/rl-2.12.md b/doc/manual/src/release-notes/rl-2.12.md index e2045d7bf..57d092e01 100644 --- a/doc/manual/src/release-notes/rl-2.12.md +++ b/doc/manual/src/release-notes/rl-2.12.md @@ -2,20 +2,8 @@ * On Linux, Nix can now run builds in a user namespace where they run as root (UID 0) and have 65,536 UIDs available. - - This is primarily useful for running containers such as `systemd-nspawn` - inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn]. - [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. - - A build can enable this by setting the derivation attribute: - - ``` - requiredSystemFeatures = [ "uid-range" ]; - ``` - - The `uid-range` [system feature] requires the [`auto-allocate-uids`] - setting to be enabled. + This can be used by requiring `uid-range` [system feature] in derivations. [system feature]: ../command-ref/conf-file.md#conf-system-features diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 879588375..9f852dc7b 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -337,7 +337,7 @@ public: users in `build-users-group`. UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS. - )"}; + )", {}, true, Xp::AutoAllocateUids}; Setting startId{this, #if __linux__ @@ -690,20 +690,37 @@ public: "system-features", R"( A set of system “features” supported by this machine. - Derivations can express a dependency on such features through the derivation attribute `requiredSystemFeatures`. - For example, the attribute + This complements the [`system`](#conf-system) and [`extra-platforms`](#conf-extra-platforms) configuration options and the corresponding [`system`](@docroot@/language/derivations.md#attr-system) attribute on derivations. - requiredSystemFeatures = [ "kvm" ]; + Derivations can require system features in the derivation attribute [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.md#adv-attr-requiredSystemFeatures). - ensures that the derivation can only be built on a machine with the `kvm` feature. + System features are generally user-defined, but the following have special treatment: - This setting by default includes - - `kvm` if `/dev/kvm` is accessible - - historical pseudo-features for backwards compatibility, used in Nixpkgs to route Hydra builds to specific machines - - `nixos-test` - - `benchmark` - - `big-parallel` + - `kvm` + + Set by default if `/dev/kvm` is accessible. + + - `nixos-test`, `benchmark`, `big-parallel` + + These historical pseudo-features are always enabled for backwards compatibility, used in Nixpkgs to route Hydra builds to specific machines. + + - `ca-derivations` + + Set by default if the [`ca-derivations` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) is enabled. + + - `recursive-nix` + + Set by default if the [`recursive-nix` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-recursive-nix) is enabled. + + - `uid-range` + + On Linux, Nix can run builds in a user namespace where they run as root (UID 0) and have 65,536 UIDs available. + This is primarily useful for running containers such as `systemd-nspawn` inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn]. + + [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. + + Set by default on Linux if the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting is enabled. )", {}, false}; Setting substituters{ From 2fa90e5824635998b445b6862bb49469e8fc75de Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 13:59:18 +0200 Subject: [PATCH 136/909] add more details on CA derivations --- doc/manual/src/language/advanced-attributes.md | 2 ++ src/libstore/globals.hh | 16 +++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 2961f9dcc..5292a2ffc 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -229,6 +229,8 @@ Derivations can declare some infrequently used optional attributes. [`outputHashAlgo`](#adv-attr-outputHashAlgo) like for *fixed-output derivations* (see above). + It also implicitly requires that the machine to build the derivation must have the `ca-derivations` [system feature](@docroot@/command-ref/conf-file.md#conf-system-features). + - [`passAsFile`]{#adv-attr-passAsFile}\ A list of names of attributes that should be passed via files rather than environment variables. For example, if you have diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 9f852dc7b..c08a1a943 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -693,25 +693,27 @@ public: This complements the [`system`](#conf-system) and [`extra-platforms`](#conf-extra-platforms) configuration options and the corresponding [`system`](@docroot@/language/derivations.md#attr-system) attribute on derivations. - Derivations can require system features in the derivation attribute [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.md#adv-attr-requiredSystemFeatures). + A derivation can require system features in the [`requiredSystemFeatures` attribute](@docroot@/language/advanced-attributes.md#adv-attr-requiredSystemFeatures), and the machine to build the derivation must have them. - System features are generally user-defined, but the following have special treatment: + System features are user-defined, but Nix sets the following defaults: - `kvm` - Set by default if `/dev/kvm` is accessible. + Included by default if `/dev/kvm` is accessible. - `nixos-test`, `benchmark`, `big-parallel` - These historical pseudo-features are always enabled for backwards compatibility, used in Nixpkgs to route Hydra builds to specific machines. + These historical pseudo-features are always enabled for backwards compatibility, as they are used in Nixpkgs to route Hydra builds to specific machines. - `ca-derivations` - Set by default if the [`ca-derivations` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) is enabled. + Included by default if the [`ca-derivations` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) is enabled. + + This system feature is implicitly required by derivations with the [`__contentAddressed` attribute](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed). - `recursive-nix` - Set by default if the [`recursive-nix` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-recursive-nix) is enabled. + Included by default if the [`recursive-nix` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-recursive-nix) is enabled. - `uid-range` @@ -720,7 +722,7 @@ public: [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. - Set by default on Linux if the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting is enabled. + Included by default on Linux if the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting is enabled. )", {}, false}; Setting substituters{ From b0173716f6b27b4fb307ac9ded544e46e712ad22 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 15:07:07 +0200 Subject: [PATCH 137/909] clarify wording on args@ default handling (#8596) * clarify wording on args@ default handling Most importantly use shorter sentences and emphasize the key point that defaults aren't taken into account Co-authored-by: Robert Hensing Co-authored-by: John Ericson --- doc/manual/src/language/constructs.md | 45 +++++++++++++++++---------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md index 2482f2d17..a3590f55d 100644 --- a/doc/manual/src/language/constructs.md +++ b/doc/manual/src/language/constructs.md @@ -208,30 +208,41 @@ three kinds of patterns: ```nix { x, y, z, ... } @ args: z + y + x + args.a ``` - - Here `args` is bound to the entire argument, which is further - matched against the pattern `{ x, y, z, - ... }`. `@`-pattern makes mainly sense with an ellipsis(`...`) as + + Here `args` is bound to the argument *as passed*, which is further + matched against the pattern `{ x, y, z, ... }`. + The `@`-pattern makes mainly sense with an ellipsis(`...`) as you can access attribute names as `a`, using `args.a`, which was given as an additional attribute to the function. - + > **Warning** - > - > The `args@` expression is bound to the argument passed to the - > function which means that attributes with defaults that aren't - > explicitly specified in the function call won't cause an - > evaluation error, but won't exist in `args`. - > + > + > `args@` binds the name `args` to the attribute set that is passed to the function. + > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. + > > For instance - > + > > ```nix > let - > function = args@{ a ? 23, ... }: args; + > f = args@{ a ? 23, ... }: [ a args ]; > in - > function {} - > ```` - > - > will evaluate to an empty attribute set. + > f {} + > ``` + > + > is equivalent to + > + > ```nix + > let + > f = args @ { ... }: [ (args.a or 23) args ]; + > in + > f {} + > ``` + > + > and both expressions will evaluate to: + > + > ```nix + > [ 23 {} ] + > ``` Note that functions do not have names. If you want to give them a name, you can bind them to an attribute, e.g., From 0e4f6dfcf7303bfe3c5d8af6863d57b3e70d370a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 20 Jul 2023 10:27:38 +0200 Subject: [PATCH 138/909] revert anchor prefix for builtin constants the original change broke many pre-existing anchor links. also change formatting of the constants listing slightly: - the type should not be part of the anchor - add highlight to the "impure only" note --- doc/manual/generate-builtin-constants.nix | 8 +++++--- src/libexpr/eval.hh | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/manual/generate-builtin-constants.nix b/doc/manual/generate-builtin-constants.nix index 3fc1fae42..8af80a02c 100644 --- a/doc/manual/generate-builtin-constants.nix +++ b/doc/manual/generate-builtin-constants.nix @@ -10,12 +10,14 @@ let type' = optionalString (type != null) " (${type})"; impureNotice = optionalString impure-only '' - Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). + > **Note** + > + > Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). ''; in squash '' -
- ${name}${type'} +
+ ${name}${type'}
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 277e77ad5..46fa96d05 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -806,7 +806,7 @@ struct EvalSettings : Config List of directories to be searched for `<...>` file references In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtin-constants-nixPath). + [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). )"}; Setting restrictEval{ From 85d0eb63165e7d7f441fe3dd94bb548a40502e52 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 20 Jul 2023 17:58:14 +0200 Subject: [PATCH 139/909] fix broken links (#8722) --- doc/manual/src/contributing/hacking.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index b8ea977d7..4b0a3a3e5 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -128,7 +128,7 @@ platform. Common solutions include [remote builders] and [binary format emulatio (only supported on NixOS). [remote builders]: ../advanced-topics/distributed-builds.md -[binfmt emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems +[binary format emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems Given such a setup, executing the build only requires selecting the respective attribute. For example, to compile for `aarch64-linux`: @@ -166,7 +166,7 @@ When Nix is built such that `./configure` is passed any of the `--host`, `--buil -[-]- ``` -For historic reasons and backward-compatibility, some CPU and OS identifiers are translated from the GNU Autotools naming convention in [`configure.ac`](https://github.com/nixos/nix/blob/master/config/config.sub) as follows: +For historic reasons and backward-compatibility, some CPU and OS identifiers are translated from the GNU Autotools naming convention in [`configure.ac`](https://github.com/nixos/nix/blob/master/configure.ac) as follows: | `config.guess` | Nix | |----------------------------|---------------------| From 7b30293d389ea75ee24ec1f2a4a4187f175757ab Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 19 Jul 2023 15:50:25 -0400 Subject: [PATCH 140/909] Tighten `#include`s: `DerivedPath` doesn't care about `Realisation` --- src/libcmd/built-path.hh | 1 + src/libstore/derived-path.hh | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index c563a46e9..744e8090b 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -1,4 +1,5 @@ #include "derived-path.hh" +#include "realisation.hh" namespace nix { diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 6ea80c92e..7a4261ce0 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -3,7 +3,6 @@ #include "util.hh" #include "path.hh" -#include "realisation.hh" #include "outputs-spec.hh" #include "comparator.hh" From f62543fe1cc544a5684af327f63a1aeb1bdeba94 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 20 Jul 2023 09:58:25 -0400 Subject: [PATCH 141/909] Remove unneeded copy It appeared in 8eb73a87245acf9d93dc401831b629981864fa58 (by me!) without justification. --- src/libstore/local-store.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e69460e6c..892ff2295 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1022,9 +1022,8 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) std::map> -LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) +LocalStore::queryPartialDerivationOutputMap(const StorePath & path) { - auto path = path_; auto outputs = retrySQLite>>([&]() { auto state(_state.lock()); std::map> outputs; From 6bc98c7fba9f783414692fcef41d90ed80928b6c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 19 Jul 2023 14:52:35 -0400 Subject: [PATCH 142/909] Give `queryPartialDerivationOutputMap` an `evalStore` parameter This makes it more useful. In general, the derivation will be in one store, and the realisation info is in another. This also helps us avoid duplication. See how `resolveDerivedPath` is now simpler because it uses `queryPartialDerivationOutputMap`. In #8369 we get more flavors of derived path, and need more code to resolve them all, and this problem only gets worse. The fact that we need a new method to deal with the multiple dispatch is unfortunate, but this generally relates to the fact that `Store` is a sub-par interface, too bulky/unwieldy and conflating separate concerns. Solving that is out of scope of this PR. This is part of the RFC 92 work. See tracking issue #6316 --- src/libstore/build/local-derivation-goal.cc | 6 ++- src/libstore/local-store.cc | 19 +------- src/libstore/local-store.hh | 2 +- src/libstore/misc.cc | 53 +++++++++------------ src/libstore/realisation.hh | 9 +++- src/libstore/remote-store.cc | 33 ++++++++----- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 34 +++++++++++-- src/libstore/store-api.hh | 15 +++++- 9 files changed, 103 insertions(+), 70 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 2935b9da9..0d982bbff 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1251,11 +1251,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo void queryReferrers(const StorePath & path, StorePathSet & referrers) override { } - std::map> queryPartialDerivationOutputMap(const StorePath & path) override + std::map> queryPartialDerivationOutputMap( + const StorePath & path, + Store * evalStore = nullptr) override { if (!goal.isAllowed(path)) throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); - return next->queryPartialDerivationOutputMap(path); + return next->queryPartialDerivationOutputMap(path, evalStore); } std::optional queryPathFromHashPart(const std::string & hashPart) override diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 892ff2295..010e52019 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1022,9 +1022,9 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) std::map> -LocalStore::queryPartialDerivationOutputMap(const StorePath & path) +LocalStore::queryStaticPartialDerivationOutputMap(const StorePath & path) { - auto outputs = retrySQLite>>([&]() { + return retrySQLite>>([&]() { auto state(_state.lock()); std::map> outputs; uint64_t drvId; @@ -1036,21 +1036,6 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath & path) return outputs; }); - - if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) - return outputs; - - auto drv = readInvalidDerivation(path); - auto drvHashes = staticOutputHashes(*this, drv); - for (auto& [outputName, hash] : drvHashes) { - auto realisation = queryRealisation(DrvOutput{hash, outputName}); - if (realisation) - outputs.insert_or_assign(outputName, realisation->outPath); - else - outputs.insert({outputName, std::nullopt}); - } - - return outputs; } std::optional LocalStore::queryPathFromHashPart(const std::string & hashPart) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 8a3b0b43f..d20e9cc4f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -165,7 +165,7 @@ public: StorePathSet queryValidDerivers(const StorePath & path) override; - std::map> queryPartialDerivationOutputMap(const StorePath & path) override; + std::map> queryStaticPartialDerivationOutputMap(const StorePath & path) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 50336c779..14160dc8b 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -310,43 +310,34 @@ std::map drvOutputReferences( OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) { - auto & evalStore = evalStore_ ? *evalStore_ : store; + auto outputsOpt_ = store.queryPartialDerivationOutputMap(bfd.drvPath, evalStore_); - OutputPathMap outputs; - auto drv = evalStore.readDerivation(bfd.drvPath); - auto outputHashes = staticOutputHashes(store, drv); - auto drvOutputs = drv.outputsAndOptPaths(store); - auto outputNames = std::visit(overloaded { + auto outputsOpt = std::visit(overloaded { [&](const OutputsSpec::All &) { - StringSet names; - for (auto & [outputName, _] : drv.outputs) - names.insert(outputName); - return names; + // Keep all outputs + return std::move(outputsOpt_); }, [&](const OutputsSpec::Names & names) { - return static_cast>(names); + // Get just those mentioned by name + std::map> outputsOpt; + for (auto & output : names) { + auto * pOutputPathOpt = get(outputsOpt_, output); + if (!pOutputPathOpt) + throw Error( + "the derivation '%s' doesn't have an output named '%s'", + store.printStorePath(bfd.drvPath), output); + outputsOpt.insert_or_assign(output, std::move(*pOutputPathOpt)); + } + return outputsOpt; }, }, bfd.outputs.raw()); - for (auto & output : outputNames) { - auto outputHash = get(outputHashes, output); - if (!outputHash) - throw Error( - "the derivation '%s' doesn't have an output named '%s'", - store.printStorePath(bfd.drvPath), output); - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - DrvOutput outputId { *outputHash, output }; - auto realisation = store.queryRealisation(outputId); - if (!realisation) - throw MissingRealisation(outputId); - outputs.insert_or_assign(output, realisation->outPath); - } else { - // If ca-derivations isn't enabled, assume that - // the output path is statically known. - auto drvOutput = get(drvOutputs, output); - assert(drvOutput); - assert(drvOutput->second); - outputs.insert_or_assign(output, *drvOutput->second); - } + + OutputPathMap outputs; + for (auto & [outputName, outputPathOpt] : outputsOpt) { + if (!outputPathOpt) + throw MissingRealisation(store.printStorePath(bfd.drvPath), outputName); + auto & outputPath = *outputPathOpt; + outputs.insert_or_assign(outputName, outputPath); } return outputs; } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 2a093c128..0548b30c1 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -5,6 +5,7 @@ #include "hash.hh" #include "path.hh" +#include "derived-path.hh" #include #include "comparator.hh" #include "crypto.hh" @@ -143,9 +144,13 @@ class MissingRealisation : public Error { public: MissingRealisation(DrvOutput & outputId) - : Error( "cannot operate on an output of the " + : MissingRealisation(outputId.outputName, outputId.strHash()) + {} + MissingRealisation(std::string_view drv, std::string outputName) + : Error( "cannot operate on output '%s' of the " "unbuilt derivation '%s'", - outputId.to_string()) + outputName, + drv) {} }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1e2104e1f..bfe2258a4 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -378,27 +378,36 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) } -std::map> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path) +std::map> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore_) { if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { - auto conn(getConnection()); - conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path); - conn.processStderr(); - return WorkerProto::Serialise>>::read(*this, *conn); + if (!evalStore_) { + auto conn(getConnection()); + conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path); + conn.processStderr(); + return WorkerProto::Serialise>>::read(*this, *conn); + } else { + auto & evalStore = *evalStore_; + auto outputs = evalStore.queryStaticPartialDerivationOutputMap(path); + // union with the first branch overriding the statically-known ones + // when non-`std::nullopt`. + for (auto && [outputName, optPath] : queryPartialDerivationOutputMap(path, nullptr)) { + if (optPath) + outputs.insert_or_assign(std::move(outputName), std::move(optPath)); + else + outputs.insert({std::move(outputName), std::nullopt}); + } + return outputs; + } } else { + auto & evalStore = evalStore_ ? *evalStore_ : *this; // Fallback for old daemon versions. // For floating-CA derivations (and their co-dependencies) this is an // under-approximation as it only returns the paths that can be inferred // from the derivation itself (and not the ones that are known because // the have been built), but as old stores don't handle floating-CA // derivations this shouldn't matter - auto derivation = readDerivation(path); - auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this); - std::map> ret; - for (auto & [outputName, outputAndPath] : outputsWithOptPaths) { - ret.emplace(outputName, outputAndPath.second); - } - return ret; + return evalStore.queryStaticPartialDerivationOutputMap(path); } } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index cb7a71acf..b12f5437f 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -63,7 +63,7 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; - std::map> queryPartialDerivationOutputMap(const StorePath & path) override; + std::map> queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5bee1af9f..a581edcc0 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -496,22 +496,50 @@ bool Store::PathInfoCacheValue::isKnownNow() return std::chrono::steady_clock::now() < time_point + ttl; } -std::map> Store::queryPartialDerivationOutputMap(const StorePath & path) +std::map> Store::queryStaticPartialDerivationOutputMap(const StorePath & path) { std::map> outputs; auto drv = readInvalidDerivation(path); - for (auto& [outputName, output] : drv.outputsAndOptPaths(*this)) { + for (auto & [outputName, output] : drv.outputsAndOptPaths(*this)) { outputs.emplace(outputName, output.second); } return outputs; } +std::map> Store::queryPartialDerivationOutputMap( + const StorePath & path, + Store * evalStore_) +{ + auto & evalStore = evalStore_ ? *evalStore_ : *this; + + auto outputs = evalStore.queryStaticPartialDerivationOutputMap(path); + + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) + return outputs; + + auto drv = evalStore.readInvalidDerivation(path); + auto drvHashes = staticOutputHashes(*this, drv); + for (auto & [outputName, hash] : drvHashes) { + auto realisation = queryRealisation(DrvOutput{hash, outputName}); + if (realisation) { + outputs.insert_or_assign(outputName, realisation->outPath); + } else { + // queryStaticPartialDerivationOutputMap is not guaranteed + // to return std::nullopt for outputs which are not + // statically known. + outputs.insert({outputName, std::nullopt}); + } + } + + return outputs; +} + OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { auto resp = queryPartialDerivationOutputMap(path); OutputPathMap result; for (auto & [outName, optOutPath] : resp) { if (!optOutPath) - throw Error("output '%s' of derivation '%s' has no store path mapped to it", outName, printStorePath(path)); + throw MissingRealisation(printStorePath(path), outName); result.insert_or_assign(outName, *optOutPath); } return result; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 14a862eef..7b412d2dd 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -425,7 +425,20 @@ public: * derivation. All outputs are mentioned so ones mising the mapping * are mapped to `std::nullopt`. */ - virtual std::map> queryPartialDerivationOutputMap(const StorePath & path); + virtual std::map> queryPartialDerivationOutputMap( + const StorePath & path, + Store * evalStore = nullptr); + + /** + * Like `queryPartialDerivationOutputMap` but only considers + * statically known output paths (i.e. those that can be gotten from + * the derivation itself. + * + * Just a helper function for implementing + * `queryPartialDerivationOutputMap`. + */ + virtual std::map> queryStaticPartialDerivationOutputMap( + const StorePath & path); /** * Query the mapping outputName=>outputPath for the given derivation. From 570a1a3ad773d93aa2128a8fba49f98e1e115d5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Sat, 8 Jul 2023 12:28:13 +0200 Subject: [PATCH 143/909] parser: merge nested dynamic attributes Fixes https://github.com/NixOS/nix/issues/7115 --- doc/manual/src/release-notes/rl-next.md | 19 +++++++++++++++++++ src/libexpr/parser.y | 1 + .../lang/eval-fail-dup-dynamic-attrs.err.exp | 8 ++++++++ tests/lang/eval-fail-dup-dynamic-attrs.nix | 4 ++++ tests/lang/eval-okay-merge-dynamic-attrs.exp | 1 + tests/lang/eval-okay-merge-dynamic-attrs.nix | 13 +++++++++++++ 6 files changed, 46 insertions(+) create mode 100644 tests/lang/eval-fail-dup-dynamic-attrs.err.exp create mode 100644 tests/lang/eval-fail-dup-dynamic-attrs.nix create mode 100644 tests/lang/eval-okay-merge-dynamic-attrs.exp create mode 100644 tests/lang/eval-okay-merge-dynamic-attrs.nix diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 139d07188..160245a31 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -6,3 +6,22 @@ - Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths. Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths. + +- Nested dynamic attributes are now merged correctly by the parser. For example: + + ```nix + { + nested = { foo = 1; }; + nested = { ${"ba" + "r"} = 2; }; + } + ``` + + This used to silently discard `nested.bar`, but now behaves as one would expect and evaluates to: + + ```nix + { nested = { bar = 2; foo = 1; }; } + ``` + + Note that the feature of merging multiple attribute set declarations is of questionable value. + It allows writing expressions that are very hard to read, for instance when there are many lines of code between two declarations of the same attribute. + This has been around for a long time and is therefore supported for backwards compatibility, but should not be relied upon. diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 0a1ad9967..217c17382 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -137,6 +137,7 @@ static void addAttr(ExprAttrs * attrs, AttrPath && attrPath, dupAttr(state, ad.first, j2->second.pos, ad.second.pos); jAttrs->attrs.emplace(ad.first, ad.second); } + jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end()); } else { dupAttr(state, attrPath, pos, j->second.pos); } diff --git a/tests/lang/eval-fail-dup-dynamic-attrs.err.exp b/tests/lang/eval-fail-dup-dynamic-attrs.err.exp new file mode 100644 index 000000000..e01f8e6d0 --- /dev/null +++ b/tests/lang/eval-fail-dup-dynamic-attrs.err.exp @@ -0,0 +1,8 @@ +error: dynamic attribute 'b' already defined at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:11 + + at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:3:11: + + 2| set = { "${"" + "b"}" = 1; }; + 3| set = { "${"b" + ""}" = 2; }; + | ^ + 4| } diff --git a/tests/lang/eval-fail-dup-dynamic-attrs.nix b/tests/lang/eval-fail-dup-dynamic-attrs.nix new file mode 100644 index 000000000..7ea17f6c8 --- /dev/null +++ b/tests/lang/eval-fail-dup-dynamic-attrs.nix @@ -0,0 +1,4 @@ +{ + set = { "${"" + "b"}" = 1; }; + set = { "${"b" + ""}" = 2; }; +} diff --git a/tests/lang/eval-okay-merge-dynamic-attrs.exp b/tests/lang/eval-okay-merge-dynamic-attrs.exp new file mode 100644 index 000000000..157d677ce --- /dev/null +++ b/tests/lang/eval-okay-merge-dynamic-attrs.exp @@ -0,0 +1 @@ +{ set1 = { a = 1; b = 2; }; set2 = { a = 1; b = 2; }; set3 = { a = 1; b = 2; }; set4 = { a = 1; b = 2; }; } diff --git a/tests/lang/eval-okay-merge-dynamic-attrs.nix b/tests/lang/eval-okay-merge-dynamic-attrs.nix new file mode 100644 index 000000000..f459a554f --- /dev/null +++ b/tests/lang/eval-okay-merge-dynamic-attrs.nix @@ -0,0 +1,13 @@ +{ + set1 = { a = 1; }; + set1 = { "${"b" + ""}" = 2; }; + + set2 = { "${"b" + ""}" = 2; }; + set2 = { a = 1; }; + + set3.a = 1; + set3."${"b" + ""}" = 2; + + set4."${"b" + ""}" = 2; + set4.a = 1; +} From 0a30b072779aa605b92f1961fe94218ccd183669 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Apr 2023 12:17:35 -0400 Subject: [PATCH 144/909] Move `Store::Params` typedef to `StoreConfig::Params` This is because `StoreConfig` also uses it. --- src/libstore/store-api.hh | 6 ++---- src/libstore/uds-remote-store.hh | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7b412d2dd..3758c730f 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -99,6 +99,8 @@ typedef std::map> StorePathCAMap; struct StoreConfig : public Config { + typedef std::map Params; + using Config::Config; StoreConfig() = delete; @@ -153,10 +155,6 @@ struct StoreConfig : public Config class Store : public std::enable_shared_from_this, public virtual StoreConfig { -public: - - typedef std::map Params; - protected: struct PathInfoCacheValue { diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 2bd6517fa..b9d9a39b5 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -9,7 +9,7 @@ namespace nix { struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig { - UDSRemoteStoreConfig(const Store::Params & params) + UDSRemoteStoreConfig(const Params & params) : StoreConfig(params) , LocalFSStoreConfig(params) , RemoteStoreConfig(params) From 13269ba93b7453def7084b00eb4a34ad787a7c45 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Apr 2023 12:14:15 -0400 Subject: [PATCH 145/909] Make `RemoteStore::ConnectionHandle` part of class and expose Will need to do subclass-specific implementations in the next commit. This isn't because there will be multiple variations of the daemon protocol (whew!) but because different clients pick and choose different parts to use. --- src/libstore/remote-store-connection.hh | 31 +++++++++++++++ src/libstore/remote-store.cc | 52 +++++++------------------ src/libstore/remote-store.hh | 4 +- 3 files changed, 47 insertions(+), 40 deletions(-) diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh index d32d91a60..ce4740a9c 100644 --- a/src/libstore/remote-store-connection.hh +++ b/src/libstore/remote-store-connection.hh @@ -1,5 +1,6 @@ #include "remote-store.hh" #include "worker-protocol.hh" +#include "pool.hh" namespace nix { @@ -94,4 +95,34 @@ struct RemoteStore::Connection std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); }; +/** + * A wrapper around Pool::Handle that marks + * the connection as bad (causing it to be closed) if a non-daemon + * exception is thrown before the handle is closed. Such an exception + * causes a deviation from the expected protocol and therefore a + * desynchronization between the client and daemon. + */ +struct RemoteStore::ConnectionHandle +{ + Pool::Handle handle; + bool daemonException = false; + + ConnectionHandle(Pool::Handle && handle) + : handle(std::move(handle)) + { } + + ConnectionHandle(ConnectionHandle && h) + : handle(std::move(h.handle)) + { } + + ~ConnectionHandle(); + + RemoteStore::Connection & operator * () { return *handle; } + RemoteStore::Connection * operator -> () { return &*handle; } + + void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); + + void withFramedSink(std::function fun); +}; + } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index bfe2258a4..926ccd9d1 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -159,49 +159,25 @@ void RemoteStore::setOptions(Connection & conn) } -/* A wrapper around Pool::Handle that marks - the connection as bad (causing it to be closed) if a non-daemon - exception is thrown before the handle is closed. Such an exception - causes a deviation from the expected protocol and therefore a - desynchronization between the client and daemon. */ -struct ConnectionHandle +RemoteStore::ConnectionHandle::~ConnectionHandle() { - Pool::Handle handle; - bool daemonException = false; - - ConnectionHandle(Pool::Handle && handle) - : handle(std::move(handle)) - { } - - ConnectionHandle(ConnectionHandle && h) - : handle(std::move(h.handle)) - { } - - ~ConnectionHandle() - { - if (!daemonException && std::uncaught_exceptions()) { - handle.markBad(); - debug("closing daemon connection because of an exception"); - } + if (!daemonException && std::uncaught_exceptions()) { + handle.markBad(); + debug("closing daemon connection because of an exception"); } +} - RemoteStore::Connection * operator -> () { return &*handle; } - RemoteStore::Connection & operator * () { return *handle; } - - void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true) - { - auto ex = handle->processStderr(sink, source, flush); - if (ex) { - daemonException = true; - std::rethrow_exception(ex); - } +void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, bool flush) +{ + auto ex = handle->processStderr(sink, source, flush); + if (ex) { + daemonException = true; + std::rethrow_exception(ex); } - - void withFramedSink(std::function fun); -}; +} -ConnectionHandle RemoteStore::getConnection() +RemoteStore::ConnectionHandle RemoteStore::getConnection() { return ConnectionHandle(connections->get()); } @@ -1099,7 +1075,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } -void ConnectionHandle::withFramedSink(std::function fun) +void RemoteStore::ConnectionHandle::withFramedSink(std::function fun) { (*this)->to.flush(); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index b12f5437f..68ec29e6c 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -17,7 +17,6 @@ class Pid; struct FdSink; struct FdSource; template class Pool; -struct ConnectionHandle; struct RemoteStoreConfig : virtual StoreConfig { @@ -182,6 +181,8 @@ protected: void setOptions() override; + struct ConnectionHandle; + ConnectionHandle getConnection(); friend struct ConnectionHandle; @@ -199,5 +200,4 @@ private: std::shared_ptr evalStore); }; - } From 60d8dd7aeaf7fc022a1b207012c94180f6732b45 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 23 Mar 2023 10:06:45 -0400 Subject: [PATCH 146/909] Clean up store hierarchy with `IndirectRootStore` See the API doc comments for details. --- src/libstore/build/local-derivation-goal.cc | 4 +- src/libstore/daemon.cc | 5 ++- src/libstore/gc-store.hh | 35 +++++++++++---- src/libstore/gc.cc | 3 +- src/libstore/indirect-root-store.hh | 48 +++++++++++++++++++++ src/libstore/local-fs-store.hh | 16 ++++++- src/libstore/local-store.hh | 13 ++++-- src/libstore/remote-store.cc | 9 ---- src/libstore/remote-store.hh | 2 - src/libstore/ssh-store.cc | 12 ++++-- src/libstore/uds-remote-store.cc | 10 +++++ src/libstore/uds-remote-store.hh | 16 ++++++- 12 files changed, 136 insertions(+), 37 deletions(-) create mode 100644 src/libstore/indirect-root-store.hh diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 214f3d49e..b7a27490c 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1,5 +1,5 @@ #include "local-derivation-goal.hh" -#include "gc-store.hh" +#include "indirect-root-store.hh" #include "hook-instance.hh" #include "worker.hh" #include "builtins.hh" @@ -1200,7 +1200,7 @@ struct RestrictedStoreConfig : virtual LocalFSStoreConfig /* A wrapper around LocalStore that only allows building/querying of paths that are in the input closures of the build or were added via recursive Nix calls. */ -struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore, public virtual GcStore +struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual IndirectRootStore, public virtual GcStore { ref next; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index ad3dee1a2..8cbf6f044 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -7,6 +7,7 @@ #include "store-cast.hh" #include "gc-store.hh" #include "log-store.hh" +#include "indirect-root-store.hh" #include "path-with-outputs.hh" #include "finally.hh" #include "archive.hh" @@ -675,8 +676,8 @@ static void performOp(TunnelLogger * logger, ref store, Path path = absPath(readString(from)); logger->startWork(); - auto & gcStore = require(*store); - gcStore.addIndirectRoot(path); + auto & indirectRootStore = require(*store); + indirectRootStore.addIndirectRoot(path); logger->stopWork(); to << 1; diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index 2c26c65c4..ab1059fb1 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -71,19 +71,36 @@ struct GCResults }; +/** + * Mix-in class for \ref Store "stores" which expose a notion of garbage + * collection. + * + * Garbage collection will allow deleting paths which are not + * transitively "rooted". + * + * The notion of GC roots actually not part of this class. + * + * - The base `Store` class has `Store::addTempRoot()` because for a store + * that doesn't support garbage collection at all, a temporary GC root is + * safely implementable as no-op. + * + * @todo actually this is not so good because stores are *views*. + * Some views have only a no-op temp roots even though others to the + * same store allow triggering GC. For instance one can't add a root + * over ssh, but that doesn't prevent someone from gc-ing that store + * accesed via SSH locally). + * + * - The derived `LocalFSStore` class has `LocalFSStore::addPermRoot`, + * which is not part of this class because it relies on the notion of + * an ambient file system. There are stores (`ssh-ng://`, for one), + * that *do* support garbage collection but *don't* expose any file + * system, and `LocalFSStore::addPermRoot` thus does not make sense + * for them. + */ struct GcStore : public virtual Store { inline static std::string operationName = "Garbage collection"; - /** - * Add an indirect root, which is merely a symlink to `path` from - * `/nix/var/nix/gcroots/auto/`. `path` is supposed - * to be a symlink to a store path. The garbage collector will - * automatically remove the indirect root when it finds that - * `path` has disappeared. - */ - virtual void addIndirectRoot(const Path & path) = 0; - /** * Find the roots of the garbage collector. Each root is a pair * `(link, storepath)` where `link` is the path of the symlink diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 20720fb99..26c87391c 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,7 +1,6 @@ #include "derivations.hh" #include "globals.hh" #include "local-store.hh" -#include "local-fs-store.hh" #include "finally.hh" #include @@ -50,7 +49,7 @@ void LocalStore::addIndirectRoot(const Path & path) } -Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot) +Path IndirectRootStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot) { Path gcRoot(canonPath(_gcRoot)); diff --git a/src/libstore/indirect-root-store.hh b/src/libstore/indirect-root-store.hh new file mode 100644 index 000000000..59e45af45 --- /dev/null +++ b/src/libstore/indirect-root-store.hh @@ -0,0 +1,48 @@ +#pragma once +///@file + +#include "local-fs-store.hh" + +namespace nix { + +/** + * Mix-in class for implementing permanent roots as a pair of a direct + * (strong) reference and indirect weak reference to the first + * reference. + * + * See methods for details on the operations it represents. + */ +struct IndirectRootStore : public virtual LocalFSStore +{ + inline static std::string operationName = "Indirect GC roots registration"; + + /** + * Implementation of `LocalFSStore::addPermRoot` where the permanent + * root is a pair of + * + * - The user-facing symlink which all implementations must create + * + * - An additional weak reference known as the "indirect root" that + * points to that symlink. + * + * The garbage collector will automatically remove the indirect root + * when it finds that the symlink has disappeared. + * + * The implementation of this method is concrete, but it delegates + * to `addIndirectRoot()` which is abstract. + */ + Path addPermRoot(const StorePath & storePath, const Path & gcRoot) override final; + + /** + * Add an indirect root, which is a weak reference to the + * user-facing symlink created by `addPermRoot()`. + * + * @param path user-facing and user-controlled symlink to a store + * path. + * + * The form this weak-reference takes is implementation-specific. + */ + virtual void addIndirectRoot(const Path & path) = 0; +}; + +} diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 2ee2ef0c8..488109501 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -40,6 +40,7 @@ class LocalFSStore : public virtual LocalFSStoreConfig, public virtual LogStore { public: + inline static std::string operationName = "Local Filesystem Store"; const static std::string drvsLogDir; @@ -49,9 +50,20 @@ public: ref getFSAccessor() override; /** - * Register a permanent GC root. + * Creates symlink from the `gcRoot` to the `storePath` and + * registers the `gcRoot` as a permanent GC root. The `gcRoot` + * symlink lives outside the store and is created and owned by the + * user. + * + * @param gcRoot The location of the symlink. + * + * @param storePath The store object being rooted. The symlink will + * point to `toRealPath(store.printStorePath(storePath))`. + * + * How the permanent GC root corresponding to this symlink is + * managed is implementation-specific. */ - Path addPermRoot(const StorePath & storePath, const Path & gcRoot); + virtual Path addPermRoot(const StorePath & storePath, const Path & gcRoot) = 0; virtual Path getRealStoreDir() { return realStoreDir; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 2bdd46fa2..4125cacf4 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -5,8 +5,7 @@ #include "pathlocks.hh" #include "store-api.hh" -#include "local-fs-store.hh" -#include "gc-store.hh" +#include "indirect-root-store.hh" #include "sync.hh" #include "util.hh" @@ -68,7 +67,9 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig std::string doc() override; }; -class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore +class LocalStore : public virtual LocalStoreConfig + , public virtual IndirectRootStore + , public virtual GcStore { private: @@ -209,6 +210,12 @@ private: public: + /** + * Implementation of IndirectRootStore::addIndirectRoot(). + * + * The weak reference merely is a symlink to `path' from + * /nix/var/nix/gcroots/auto/. + */ void addIndirectRoot(const Path & path) override; private: diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 926ccd9d1..21258daec 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -822,15 +822,6 @@ void RemoteStore::addTempRoot(const StorePath & path) } -void RemoteStore::addIndirectRoot(const Path & path) -{ - auto conn(getConnection()); - conn->to << WorkerProto::Op::AddIndirectRoot << path; - conn.processStderr(); - readInt(conn->from); -} - - Roots RemoteStore::findRoots(bool censor) { auto conn(getConnection()); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 68ec29e6c..a1ae82a0f 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -126,8 +126,6 @@ public: void addTempRoot(const StorePath & path) override; - void addIndirectRoot(const Path & path) override; - Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 0200076c0..9c6c42ef4 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,5 +1,6 @@ #include "ssh-store-config.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "remote-store.hh" #include "remote-store-connection.hh" #include "remote-fs-accessor.hh" @@ -61,7 +62,7 @@ public: std::optional getBuildLogExact(const StorePath & path) override { unsupported("getBuildLogExact"); } -private: +protected: struct Connection : RemoteStore::Connection { @@ -93,9 +94,12 @@ private: ref SSHStore::openConnection() { auto conn = make_ref(); - conn->sshConn = master.startCommand( - fmt("%s --stdio", remoteProgram) - + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); + + std::string command = remoteProgram + " --stdio"; + if (remoteStore.get() != "") + command += " --store " + shellEscape(remoteStore.get()); + + conn->sshConn = master.startCommand(command); conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); return conn; diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 69dae2da5..99589f8b2 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,4 +1,5 @@ #include "uds-remote-store.hh" +#include "worker-protocol.hh" #include #include @@ -77,6 +78,15 @@ ref UDSRemoteStore::openConnection() } +void UDSRemoteStore::addIndirectRoot(const Path & path) +{ + auto conn(getConnection()); + conn->to << WorkerProto::Op::AddIndirectRoot << path; + conn.processStderr(); + readInt(conn->from); +} + + static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index b9d9a39b5..cdb28a001 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -3,7 +3,7 @@ #include "remote-store.hh" #include "remote-store-connection.hh" -#include "local-fs-store.hh" +#include "indirect-root-store.hh" namespace nix { @@ -21,7 +21,9 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon std::string doc() override; }; -class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore +class UDSRemoteStore : public virtual UDSRemoteStoreConfig + , public virtual IndirectRootStore + , public virtual RemoteStore { public: @@ -39,6 +41,16 @@ public: void narFromPath(const StorePath & path, Sink & sink) override { LocalFSStore::narFromPath(path, sink); } + /** + * Implementation of `IndirectRootStore::addIndirectRoot()` which + * delegates to the remote store. + * + * The idea is that the client makes the direct symlink, so it is + * owned managed by the client's user account, and the server makes + * the indirect symlink. + */ + void addIndirectRoot(const Path & path) override; + private: struct Connection : RemoteStore::Connection From c51be0345eda59b93dc1bda09d1c125f3f84d716 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Jul 2023 17:19:31 +0200 Subject: [PATCH 147/909] Release notes --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/release-notes/rl-2.17.md | 29 +++++++++++++++++++++++++ doc/manual/src/release-notes/rl-next.md | 26 ---------------------- 3 files changed, 30 insertions(+), 26 deletions(-) create mode 100644 doc/manual/src/release-notes/rl-2.17.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index f2e69178f..6c599abcf 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -109,6 +109,7 @@ - [C++ style guide](contributing/cxx.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md) - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md) - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md) diff --git a/doc/manual/src/release-notes/rl-2.17.md b/doc/manual/src/release-notes/rl-2.17.md new file mode 100644 index 000000000..125a93cfd --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.17.md @@ -0,0 +1,29 @@ +# Release 2.17 (2023-07-24) + +* [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand. + +* The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure evaluation mode](../command-ref/conf-file.md#conf-pure-eval), as those are not impure. + +* Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths. + Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths. + +* Nested dynamic attributes are now merged correctly by the parser. For example: + + ```nix + { + nested = { foo = 1; }; + nested = { ${"ba" + "r"} = 2; }; + } + ``` + + This used to silently discard `nested.bar`, but now behaves as one would expect and evaluates to: + + ```nix + { nested = { bar = 2; foo = 1; }; } + ``` + + Note that the feature of merging multiple attribute set declarations is of questionable value. + It allows writing expressions that are very hard to read, for instance when there are many lines of code between two declarations of the same attribute. + This has been around for a long time and is therefore supported for backwards compatibility, but should not be relied upon. + +* Tarball flakes can now redirect to an "immutable" URL that will be recorded in lock files. This allows the use of "mutable" tarball URLs like `https://example.org/hello/latest.tar.gz` in flakes. See the [tarball fetcher](../protocols/tarball-fetcher.md) for details. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 160245a31..c869b5e2f 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,27 +1 @@ # Release X.Y (202?-??-??) - -- [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand - -* The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure evaluation mode](../command-ref/conf-file.md#conf-pure-eval), as those are not impure. - -- Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths. - Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths. - -- Nested dynamic attributes are now merged correctly by the parser. For example: - - ```nix - { - nested = { foo = 1; }; - nested = { ${"ba" + "r"} = 2; }; - } - ``` - - This used to silently discard `nested.bar`, but now behaves as one would expect and evaluates to: - - ```nix - { nested = { bar = 2; foo = 1; }; } - ``` - - Note that the feature of merging multiple attribute set declarations is of questionable value. - It allows writing expressions that are very hard to read, for instance when there are many lines of code between two declarations of the same attribute. - This has been around for a long time and is therefore supported for backwards compatibility, but should not be relied upon. From 0c275558e75b62beb53e8ada2c116643c7ecb0ba Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Jul 2023 21:30:33 +0200 Subject: [PATCH 148/909] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index d76bd2ba3..cf8690732 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.17.0 +2.18.0 From 1b756e300f98afe6a583f8bed3acd687d4a8abf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Tue, 25 Jul 2023 15:51:15 +0200 Subject: [PATCH 149/909] doc: clarify release notes about nested attribute merges --- doc/manual/src/release-notes/rl-2.17.md | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/release-notes/rl-2.17.md b/doc/manual/src/release-notes/rl-2.17.md index 125a93cfd..0b861aecc 100644 --- a/doc/manual/src/release-notes/rl-2.17.md +++ b/doc/manual/src/release-notes/rl-2.17.md @@ -11,8 +11,12 @@ ```nix { - nested = { foo = 1; }; - nested = { ${"ba" + "r"} = 2; }; + nested = { + foo = 1; + }; + nested = { + ${"ba" + "r"} = 2; + }; } ``` @@ -22,8 +26,17 @@ { nested = { bar = 2; foo = 1; }; } ``` - Note that the feature of merging multiple attribute set declarations is of questionable value. + Note that the feature of merging multiple *full declarations* of attribute sets like `nested` in the example is of questionable value. It allows writing expressions that are very hard to read, for instance when there are many lines of code between two declarations of the same attribute. This has been around for a long time and is therefore supported for backwards compatibility, but should not be relied upon. + Instead, consider using the *nested attribute path* syntax: + + ```nix + { + nested.foo = 1; + nested.${"ba" + "r"} = 2; + } + ``` + * Tarball flakes can now redirect to an "immutable" URL that will be recorded in lock files. This allows the use of "mutable" tarball URLs like `https://example.org/hello/latest.tar.gz` in flakes. See the [tarball fetcher](../protocols/tarball-fetcher.md) for details. From 2d1d81114d72ace89ce08cd3bc93f4eb27a2975d Mon Sep 17 00:00:00 2001 From: Alex Ameen Date: Tue, 25 Jul 2023 12:43:33 -0500 Subject: [PATCH 150/909] Add `parseFlakeRef` and `flakeRefToString` builtins (#8670) Over the last year or so I've run into several use cases where I need to parse and/or serialize URLs for use by `builtins.fetchTree` or `builtins.getFlake`, largely in order to produce _lockfile-like_ files for lang2nix frameworks or tools which use `nix` internally to drive builds. I've gone through the painstaking process of emulating `nix::FlakeRef::fromAttrs` and `nix::parseFlakeRef` several times with mixed success; but these are difficult to create and even harder to maintain if I hope to stay aligned with changes to the real parser/serializer. I understand why adding new `builtins` isn't something we want to do flagrantly. I'm recommending this addition simply because I keep encountering use cases where I need to parse/serialize these URIs in `nix` expressions, and I want a reliable solution. Co-authored-by: Eelco Dolstra Co-authored-by: John Ericson --- doc/manual/src/release-notes/rl-next.md | 7 ++ src/libexpr/flake/flake.cc | 95 ++++++++++++++++++++ tests/lang/eval-okay-flake-ref-to-string.exp | 1 + tests/lang/eval-okay-flake-ref-to-string.nix | 7 ++ tests/lang/eval-okay-parse-flake-ref.exp | 1 + tests/lang/eval-okay-parse-flake-ref.nix | 1 + 6 files changed, 112 insertions(+) create mode 100644 tests/lang/eval-okay-flake-ref-to-string.exp create mode 100644 tests/lang/eval-okay-flake-ref-to-string.nix create mode 100644 tests/lang/eval-okay-parse-flake-ref.exp create mode 100644 tests/lang/eval-okay-parse-flake-ref.nix diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index c869b5e2f..065364453 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1 +1,8 @@ # Release X.Y (202?-??-??) + +- Two new builtin functions, + [`builtins.parseFlakeRef`](@docroot@/language/builtins.md#builtins-parseFlakeRef) + and + [`builtins.flakeRefToString`](@docroot@/language/builtins.md#builtins-flakeRefToString), + have been added. + These functions are useful for converting between flake references encoded as attribute sets and URLs. diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 5aa44d6a1..9112becff 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -793,6 +793,101 @@ static RegisterPrimOp r2({ .experimentalFeature = Xp::Flakes, }); +static void prim_parseFlakeRef( + EvalState & state, + const PosIdx pos, + Value * * args, + Value & v) +{ + std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, + "while evaluating the argument passed to builtins.parseFlakeRef")); + auto attrs = parseFlakeRef(flakeRefS, {}, true).toAttrs(); + auto binds = state.buildBindings(attrs.size()); + for (const auto & [key, value] : attrs) { + auto s = state.symbols.create(key); + auto & vv = binds.alloc(s); + std::visit(overloaded { + [&vv](const std::string & value) { vv.mkString(value); }, + [&vv](const uint64_t & value) { vv.mkInt(value); }, + [&vv](const Explicit & value) { vv.mkBool(value.t); } + }, value); + } + v.mkAttrs(binds); +} + +static RegisterPrimOp r3({ + .name = "__parseFlakeRef", + .args = {"flake-ref"}, + .doc = R"( + Parse a flake reference, and return its exploded form. + + For example: + ```nix + builtins.parseFlakeRef "github:NixOS/nixpkgs/23.05?dir=lib" + ``` + evaluates to: + ```nix + { dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; } + ``` + )", + .fun = prim_parseFlakeRef, + .experimentalFeature = Xp::Flakes, +}); + + +static void prim_flakeRefToString( + EvalState & state, + const PosIdx pos, + Value * * args, + Value & v) +{ + state.forceAttrs(*args[0], noPos, + "while evaluating the argument passed to builtins.flakeRefToString"); + fetchers::Attrs attrs; + for (const auto & attr : *args[0]->attrs) { + auto t = attr.value->type(); + if (t == nInt) { + attrs.emplace(state.symbols[attr.name], + (uint64_t) attr.value->integer); + } else if (t == nBool) { + attrs.emplace(state.symbols[attr.name], + Explicit { attr.value->boolean }); + } else if (t == nString) { + attrs.emplace(state.symbols[attr.name], + std::string(attr.value->str())); + } else { + state.error( + "flake reference attribute sets may only contain integers, Booleans, " + "and strings, but attribute '%s' is %s", + state.symbols[attr.name], + showType(*attr.value)).debugThrow(); + } + } + auto flakeRef = FlakeRef::fromAttrs(attrs); + v.mkString(flakeRef.to_string()); +} + +static RegisterPrimOp r4({ + .name = "__flakeRefToString", + .args = {"attrs"}, + .doc = R"( + Convert a flake reference from attribute set format to URL format. + + For example: + ```nix + builtins.flakeRefToString { + dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; + } + ``` + evaluates to + ```nix + "github:NixOS/nixpkgs/23.05?dir=lib" + ``` + )", + .fun = prim_flakeRefToString, + .experimentalFeature = Xp::Flakes, +}); + } Fingerprint LockedFlake::getFingerprint() const diff --git a/tests/lang/eval-okay-flake-ref-to-string.exp b/tests/lang/eval-okay-flake-ref-to-string.exp new file mode 100644 index 000000000..110f8442d --- /dev/null +++ b/tests/lang/eval-okay-flake-ref-to-string.exp @@ -0,0 +1 @@ +"github:NixOS/nixpkgs/23.05?dir=lib" diff --git a/tests/lang/eval-okay-flake-ref-to-string.nix b/tests/lang/eval-okay-flake-ref-to-string.nix new file mode 100644 index 000000000..dbb4e5b2a --- /dev/null +++ b/tests/lang/eval-okay-flake-ref-to-string.nix @@ -0,0 +1,7 @@ +builtins.flakeRefToString { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + ref = "23.05"; + dir = "lib"; +} diff --git a/tests/lang/eval-okay-parse-flake-ref.exp b/tests/lang/eval-okay-parse-flake-ref.exp new file mode 100644 index 000000000..fc17ba085 --- /dev/null +++ b/tests/lang/eval-okay-parse-flake-ref.exp @@ -0,0 +1 @@ +{ dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; } diff --git a/tests/lang/eval-okay-parse-flake-ref.nix b/tests/lang/eval-okay-parse-flake-ref.nix new file mode 100644 index 000000000..db4ed2742 --- /dev/null +++ b/tests/lang/eval-okay-parse-flake-ref.nix @@ -0,0 +1 @@ + builtins.parseFlakeRef "github:NixOS/nixpkgs/23.05?dir=lib" From 33d58a90c26f9a62bb20863b767c0a505e6f997a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 30 Jun 2023 01:29:11 +0200 Subject: [PATCH 151/909] toJSON: Add attribute path to trace --- doc/manual/src/release-notes/rl-next.md | 2 + src/libexpr/value-to-json.cc | 22 ++++++++-- tests/lang/eval-fail-toJSON.err.exp | 57 +++++++++++++++++++++++++ tests/lang/eval-fail-toJSON.nix | 10 +++++ 4 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 tests/lang/eval-fail-toJSON.err.exp create mode 100644 tests/lang/eval-fail-toJSON.nix diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 065364453..df30ce83d 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -6,3 +6,5 @@ [`builtins.flakeRefToString`](@docroot@/language/builtins.md#builtins-flakeRefToString), have been added. These functions are useful for converting between flake references encoded as attribute sets and URLs. + +- [`builtins.toJSON`](@docroot@/language/builtins.md#builtins-parseFlakeRef) now prints [--show-trace](@docroot@/command-ref/conf-file.html#conf-show-trace) items for the path in which it finds an evaluation error. diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 4996a5bde..ac3986c87 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -43,6 +43,7 @@ json printValueAsJSON(EvalState & state, bool strict, break; case nNull: + // already initialized as null break; case nAttrs: { @@ -59,7 +60,13 @@ json printValueAsJSON(EvalState & state, bool strict, names.emplace(state.symbols[j.name]); for (auto & j : names) { Attr & a(*v.attrs->find(state.symbols.create(j))); - out[j] = printValueAsJSON(state, strict, *a.value, a.pos, context, copyToStore); + try { + out[j] = printValueAsJSON(state, strict, *a.value, a.pos, context, copyToStore); + } catch (Error & e) { + e.addTrace(state.positions[a.pos], + hintfmt("while evaluating attribute '%1%'", j)); + throw; + } } } else return printValueAsJSON(state, strict, *i->value, i->pos, context, copyToStore); @@ -68,8 +75,17 @@ json printValueAsJSON(EvalState & state, bool strict, case nList: { out = json::array(); - for (auto elem : v.listItems()) - out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore)); + int i = 0; + for (auto elem : v.listItems()) { + try { + out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore)); + } catch (Error & e) { + e.addTrace({}, + hintfmt("while evaluating list element at index %1%", i)); + throw; + } + i++; + } break; } diff --git a/tests/lang/eval-fail-toJSON.err.exp b/tests/lang/eval-fail-toJSON.err.exp new file mode 100644 index 000000000..4e618c203 --- /dev/null +++ b/tests/lang/eval-fail-toJSON.err.exp @@ -0,0 +1,57 @@ +error: + … while calling the 'toJSON' builtin + + at /pwd/lang/eval-fail-toJSON.nix:1:1: + + 1| builtins.toJSON { + | ^ + 2| a.b = [ + + … while evaluating attribute 'a' + + at /pwd/lang/eval-fail-toJSON.nix:2:3: + + 1| builtins.toJSON { + 2| a.b = [ + | ^ + 3| true + + … while evaluating attribute 'b' + + at /pwd/lang/eval-fail-toJSON.nix:2:3: + + 1| builtins.toJSON { + 2| a.b = [ + | ^ + 3| true + + … while evaluating list element at index 3 + + … while evaluating attribute 'c' + + at /pwd/lang/eval-fail-toJSON.nix:7:7: + + 6| { + 7| c.d = throw "hah no"; + | ^ + 8| } + + … while evaluating attribute 'd' + + at /pwd/lang/eval-fail-toJSON.nix:7:7: + + 6| { + 7| c.d = throw "hah no"; + | ^ + 8| } + + … while calling the 'throw' builtin + + at /pwd/lang/eval-fail-toJSON.nix:7:13: + + 6| { + 7| c.d = throw "hah no"; + | ^ + 8| } + + error: hah no diff --git a/tests/lang/eval-fail-toJSON.nix b/tests/lang/eval-fail-toJSON.nix new file mode 100644 index 000000000..8112e1c1f --- /dev/null +++ b/tests/lang/eval-fail-toJSON.nix @@ -0,0 +1,10 @@ +builtins.toJSON { + a.b = [ + true + false + "it's a bird" + { + c.d = throw "hah no"; + } + ]; +} From 1570e80219df92461ede2a672f8997a013364f5c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 31 Jul 2023 09:19:19 -0400 Subject: [PATCH 152/909] Move evaluator settings (type and global) to separate file/header --- src/libcmd/common-eval-args.cc | 1 + src/libcmd/installables.cc | 1 + src/libcmd/repl.cc | 1 + src/libexpr/eval-settings.cc | 95 ++++++++++++++++++++++++++ src/libexpr/eval-settings.hh | 98 +++++++++++++++++++++++++++ src/libexpr/eval.cc | 89 +----------------------- src/libexpr/eval.hh | 92 ------------------------- src/libexpr/flake/flake.cc | 1 + src/libexpr/parser.y | 1 + src/libexpr/primops.cc | 1 + src/libexpr/primops/fetchMercurial.cc | 1 + src/libexpr/primops/fetchTree.cc | 1 + src/nix/flake.cc | 1 + src/nix/main.cc | 1 + src/nix/repl.cc | 1 + src/nix/search.cc | 1 + src/nix/upgrade-nix.cc | 1 + 17 files changed, 207 insertions(+), 180 deletions(-) create mode 100644 src/libexpr/eval-settings.cc create mode 100644 src/libexpr/eval-settings.hh diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 3df2c71a5..e36bda52f 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,3 +1,4 @@ +#include "eval-settings.hh" #include "common-eval-args.hh" #include "shared.hh" #include "filetransfer.hh" diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 10b077fb5..9d593a01f 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -11,6 +11,7 @@ #include "derivations.hh" #include "eval-inline.hh" #include "eval.hh" +#include "eval-settings.hh" #include "get-drvs.hh" #include "store-api.hh" #include "shared.hh" diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index f9e9c2bf8..d15162e76 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -26,6 +26,7 @@ extern "C" { #include "eval.hh" #include "eval-cache.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "attr-path.hh" #include "store-api.hh" #include "log-store.hh" diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc new file mode 100644 index 000000000..422aaf8d5 --- /dev/null +++ b/src/libexpr/eval-settings.cc @@ -0,0 +1,95 @@ +#include "globals.hh" +#include "profiles.hh" +#include "eval.hh" +#include "eval-settings.hh" + +namespace nix { + +/* Very hacky way to parse $NIX_PATH, which is colon-separated, but + can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ +static Strings parseNixPath(const std::string & s) +{ + Strings res; + + auto p = s.begin(); + + while (p != s.end()) { + auto start = p; + auto start2 = p; + + while (p != s.end() && *p != ':') { + if (*p == '=') start2 = p + 1; + ++p; + } + + if (p == s.end()) { + if (p != start) res.push_back(std::string(start, p)); + break; + } + + if (*p == ':') { + auto prefix = std::string(start2, s.end()); + if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) { + ++p; + while (p != s.end() && *p != ':') ++p; + } + res.push_back(std::string(start, p)); + if (p == s.end()) break; + } + + ++p; + } + + return res; +} + +EvalSettings::EvalSettings() +{ + auto var = getEnv("NIX_PATH"); + if (var) nixPath = parseNixPath(*var); +} + +Strings EvalSettings::getDefaultNixPath() +{ + Strings res; + auto add = [&](const Path & p, const std::string & s = std::string()) { + if (pathAccessible(p)) { + if (s.empty()) { + res.push_back(p); + } else { + res.push_back(s + "=" + p); + } + } + }; + + if (!evalSettings.restrictEval && !evalSettings.pureEval) { + add(settings.useXDGBaseDirectories ? getStateDir() + "/nix/defexpr/channels" : getHome() + "/.nix-defexpr/channels"); + add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); + add(rootChannelsDir()); + } + + return res; +} + +bool EvalSettings::isPseudoUrl(std::string_view s) +{ + if (s.compare(0, 8, "channel:") == 0) return true; + size_t pos = s.find("://"); + if (pos == std::string::npos) return false; + std::string scheme(s, 0, pos); + return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; +} + +std::string EvalSettings::resolvePseudoUrl(std::string_view url) +{ + if (hasPrefix(url, "channel:")) + return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz"; + else + return std::string(url); +} + +EvalSettings evalSettings; + +static GlobalConfig::Register rEvalSettings(&evalSettings); + +} diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh new file mode 100644 index 000000000..043af6cab --- /dev/null +++ b/src/libexpr/eval-settings.hh @@ -0,0 +1,98 @@ +#pragma once +#include "config.hh" + +namespace nix { + +struct EvalSettings : Config +{ + EvalSettings(); + + static Strings getDefaultNixPath(); + + static bool isPseudoUrl(std::string_view s); + + static std::string resolvePseudoUrl(std::string_view url); + + Setting enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", + "Whether builtin functions that allow executing native code should be enabled."}; + + Setting nixPath{ + this, getDefaultNixPath(), "nix-path", + R"( + List of directories to be searched for `<...>` file references + + In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of + [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). + )"}; + + Setting restrictEval{ + this, false, "restrict-eval", + R"( + If set to `true`, the Nix evaluator will not allow access to any + files outside of the Nix search path (as set via the `NIX_PATH` + environment variable or the `-I` option), or to URIs outside of + [`allowed-uris`](../command-ref/conf-file.md#conf-allowed-uris). + The default is `false`. + )"}; + + Setting pureEval{this, false, "pure-eval", + R"( + Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state: + + - Restrict file system and network access to files specified by cryptographic hash + - Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) + )" + }; + + Setting enableImportFromDerivation{ + this, true, "allow-import-from-derivation", + R"( + By default, Nix allows you to `import` from a derivation, allowing + building at evaluation time. With this option set to false, Nix will + throw an error when evaluating an expression that uses this feature, + allowing users to ensure their evaluation will not require any + builds to take place. + )"}; + + Setting allowedUris{this, {}, "allowed-uris", + R"( + A list of URI prefixes to which access is allowed in restricted + evaluation mode. For example, when set to + `https://github.com/NixOS`, builtin functions such as `fetchGit` are + allowed to access `https://github.com/NixOS/patchelf.git`. + )"}; + + Setting traceFunctionCalls{this, false, "trace-function-calls", + R"( + If set to `true`, the Nix evaluator will trace every function call. + Nix will print a log message at the "vomit" level for every function + entrance and function exit. + + function-trace entered undefined position at 1565795816999559622 + function-trace exited undefined position at 1565795816999581277 + function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 + function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 + + The `undefined position` means the function call is a builtin. + + Use the `contrib/stack-collapse.py` script distributed with the Nix + source code to convert the trace logs in to a format suitable for + `flamegraph.pl`. + )"}; + + Setting useEvalCache{this, true, "eval-cache", + "Whether to use the flake evaluation cache."}; + + Setting ignoreExceptionsDuringTry{this, false, "ignore-try", + R"( + If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in + debug mode (using the --debugger flag). By default the debugger will pause on all exceptions. + )"}; + + Setting traceVerbose{this, false, "trace-verbose", + "Whether `builtins.traceVerbose` should trace its first argument when evaluated."}; +}; + +extern EvalSettings evalSettings; + +} diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index be1bdb806..e57de6c1d 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1,4 +1,5 @@ #include "eval.hh" +#include "eval-settings.hh" #include "hash.hh" #include "types.hh" #include "util.hh" @@ -420,44 +421,6 @@ void initGC() } -/* Very hacky way to parse $NIX_PATH, which is colon-separated, but - can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ -static Strings parseNixPath(const std::string & s) -{ - Strings res; - - auto p = s.begin(); - - while (p != s.end()) { - auto start = p; - auto start2 = p; - - while (p != s.end() && *p != ':') { - if (*p == '=') start2 = p + 1; - ++p; - } - - if (p == s.end()) { - if (p != start) res.push_back(std::string(start, p)); - break; - } - - if (*p == ':') { - auto prefix = std::string(start2, s.end()); - if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) { - ++p; - while (p != s.end() && *p != ':') ++p; - } - res.push_back(std::string(start, p)); - if (p == s.end()) break; - } - - ++p; - } - - return res; -} - ErrorBuilder & ErrorBuilder::atPos(PosIdx pos) { info.errPos = state.positions[pos]; @@ -2626,54 +2589,4 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { } -EvalSettings::EvalSettings() -{ - auto var = getEnv("NIX_PATH"); - if (var) nixPath = parseNixPath(*var); -} - -Strings EvalSettings::getDefaultNixPath() -{ - Strings res; - auto add = [&](const Path & p, const std::string & s = std::string()) { - if (pathAccessible(p)) { - if (s.empty()) { - res.push_back(p); - } else { - res.push_back(s + "=" + p); - } - } - }; - - if (!evalSettings.restrictEval && !evalSettings.pureEval) { - add(settings.useXDGBaseDirectories ? getStateDir() + "/nix/defexpr/channels" : getHome() + "/.nix-defexpr/channels"); - add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); - add(rootChannelsDir()); - } - - return res; -} - -bool EvalSettings::isPseudoUrl(std::string_view s) -{ - if (s.compare(0, 8, "channel:") == 0) return true; - size_t pos = s.find("://"); - if (pos == std::string::npos) return false; - std::string scheme(s, 0, pos); - return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; -} - -std::string EvalSettings::resolvePseudoUrl(std::string_view url) -{ - if (hasPrefix(url, "channel:")) - return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz"; - else - return std::string(url); -} - -EvalSettings evalSettings; - -static GlobalConfig::Register rEvalSettings(&evalSettings); - - } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 46fa96d05..887b9cb97 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -787,98 +787,6 @@ struct InvalidPathError : EvalError #endif }; -struct EvalSettings : Config -{ - EvalSettings(); - - static Strings getDefaultNixPath(); - - static bool isPseudoUrl(std::string_view s); - - static std::string resolvePseudoUrl(std::string_view url); - - Setting enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", - "Whether builtin functions that allow executing native code should be enabled."}; - - Setting nixPath{ - this, getDefaultNixPath(), "nix-path", - R"( - List of directories to be searched for `<...>` file references - - In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). - )"}; - - Setting restrictEval{ - this, false, "restrict-eval", - R"( - If set to `true`, the Nix evaluator will not allow access to any - files outside of the Nix search path (as set via the `NIX_PATH` - environment variable or the `-I` option), or to URIs outside of - [`allowed-uris`](../command-ref/conf-file.md#conf-allowed-uris). - The default is `false`. - )"}; - - Setting pureEval{this, false, "pure-eval", - R"( - Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state: - - - Restrict file system and network access to files specified by cryptographic hash - - Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) - )" - }; - - Setting enableImportFromDerivation{ - this, true, "allow-import-from-derivation", - R"( - By default, Nix allows you to `import` from a derivation, allowing - building at evaluation time. With this option set to false, Nix will - throw an error when evaluating an expression that uses this feature, - allowing users to ensure their evaluation will not require any - builds to take place. - )"}; - - Setting allowedUris{this, {}, "allowed-uris", - R"( - A list of URI prefixes to which access is allowed in restricted - evaluation mode. For example, when set to - `https://github.com/NixOS`, builtin functions such as `fetchGit` are - allowed to access `https://github.com/NixOS/patchelf.git`. - )"}; - - Setting traceFunctionCalls{this, false, "trace-function-calls", - R"( - If set to `true`, the Nix evaluator will trace every function call. - Nix will print a log message at the "vomit" level for every function - entrance and function exit. - - function-trace entered undefined position at 1565795816999559622 - function-trace exited undefined position at 1565795816999581277 - function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 - function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 - - The `undefined position` means the function call is a builtin. - - Use the `contrib/stack-collapse.py` script distributed with the Nix - source code to convert the trace logs in to a format suitable for - `flamegraph.pl`. - )"}; - - Setting useEvalCache{this, true, "eval-cache", - "Whether to use the flake evaluation cache."}; - - Setting ignoreExceptionsDuringTry{this, false, "ignore-try", - R"( - If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in - debug mode (using the --debugger flag). By default the debugger will pause on all exceptions. - )"}; - - Setting traceVerbose{this, false, "trace-verbose", - "Whether `builtins.traceVerbose` should trace its first argument when evaluated."}; -}; - -extern EvalSettings evalSettings; - static const std::string corepkgsPrefix{"/__corepkgs__/"}; template diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 9112becff..6a27ea2e8 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -1,5 +1,6 @@ #include "flake.hh" #include "eval.hh" +#include "eval-settings.hh" #include "lockfile.hh" #include "primops.hh" #include "eval-inline.hh" diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 217c17382..201370b90 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -22,6 +22,7 @@ #include "nixexpr.hh" #include "eval.hh" +#include "eval-settings.hh" #include "globals.hh" namespace nix { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7ff17b6ee..ddf529b9e 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3,6 +3,7 @@ #include "downstream-placeholder.hh" #include "eval-inline.hh" #include "eval.hh" +#include "eval-settings.hh" #include "globals.hh" #include "json-to-value.hh" #include "names.hh" diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 322692b52..b9ff01c16 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -1,5 +1,6 @@ #include "primops.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "store-api.hh" #include "fetchers.hh" #include "url.hh" diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 5e668c629..f040a3510 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -1,5 +1,6 @@ #include "primops.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "store-api.hh" #include "fetchers.hh" #include "filetransfer.hh" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index b5f5d0cac..3ce1de44a 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -4,6 +4,7 @@ #include "shared.hh" #include "eval.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "flake/flake.hh" #include "get-drvs.hh" #include "store-api.hh" diff --git a/src/nix/main.cc b/src/nix/main.cc index 650c79d14..df66beb8c 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -3,6 +3,7 @@ #include "command.hh" #include "common-args.hh" #include "eval.hh" +#include "eval-settings.hh" #include "globals.hh" #include "legacy.hh" #include "shared.hh" diff --git a/src/nix/repl.cc b/src/nix/repl.cc index bb14f3f99..9677c1b48 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -1,4 +1,5 @@ #include "eval.hh" +#include "eval-settings.hh" #include "globals.hh" #include "command.hh" #include "installable-value.hh" diff --git a/src/nix/search.cc b/src/nix/search.cc index c92ed1663..ef0139e09 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -2,6 +2,7 @@ #include "globals.hh" #include "eval.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "names.hh" #include "get-drvs.hh" #include "common-args.hh" diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index d05c23fb7..d238456db 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -3,6 +3,7 @@ #include "store-api.hh" #include "filetransfer.hh" #include "eval.hh" +#include "eval-settings.hh" #include "attr-path.hh" #include "names.hh" #include "progress-bar.hh" From c9a87ce7ca50b0a14c4c77fd2b6ca41a74ba6cb9 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Fri, 28 Jul 2023 10:11:45 +0100 Subject: [PATCH 153/909] Refactor verifyPath to take StorePath instead of Path. This way we avoid having to convert from Path to StorePath and vice versa in the body of verifyPath. --- src/libstore/local-store.cc | 21 ++++++++------------- src/libstore/local-store.hh | 4 ++-- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f8e0a0ca2..f78bd44ca 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1506,10 +1506,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printInfo("checking path existence..."); StorePathSet validPaths; - PathSet done; + StorePathSet done; for (auto & i : queryAllValidPaths()) - verifyPath(printStorePath(i), store, done, validPaths, repair, errors); + verifyPath(i, store, done, validPaths, repair, errors); /* Optionally, check the content hashes (slow). */ if (checkContents) { @@ -1595,19 +1595,12 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) } -void LocalStore::verifyPath(const Path & pathS, const StringSet & store, - PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) +void LocalStore::verifyPath(const StorePath & path, const StringSet & store, + StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) { checkInterrupt(); - if (!done.insert(pathS).second) return; - - if (!isStorePath(pathS)) { - printError("path '%s' is not in the Nix store", pathS); - return; - } - - auto path = parseStorePath(pathS); + if (!done.insert(path).second) return; if (!store.count(std::string(path.to_string()))) { /* Check any referrers first. If we can invalidate them @@ -1616,11 +1609,13 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, StorePathSet referrers; queryReferrers(path, referrers); for (auto & i : referrers) if (i != path) { - verifyPath(printStorePath(i), store, done, validPaths, repair, errors); + verifyPath(i, store, done, validPaths, repair, errors); if (validPaths.count(i)) canInvalidate = false; } + auto pathS = printStorePath(path); + if (canInvalidate) { printInfo("path '%s' disappeared, removing from database...", pathS); auto state(_state.lock()); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 4125cacf4..c9b570eaa 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -314,8 +314,8 @@ private: */ void invalidatePathChecked(const StorePath & path); - void verifyPath(const Path & path, const StringSet & store, - PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); + void verifyPath(const StorePath & path, const StringSet & store, + StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); std::shared_ptr queryPathInfoInternal(State & state, const StorePath & path); From 2a5f5fbb1776f5086646407e5296c372321863b9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 31 Jul 2023 11:13:38 -0400 Subject: [PATCH 154/909] `LocalStore::verifyPath`: Use `StorePathSet` for `store` local var We don't care about non-store-paths in there (things like `.links`, are, in fact, allowed). So let's just skip them up front and be more strongly typed. --- src/libstore/local-store.cc | 12 ++++++++---- src/libstore/local-store.hh | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f78bd44ca..9049f33aa 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1499,8 +1499,12 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) auto fdGCLock = openGCLock(); FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock..."); - StringSet store; - for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); + StorePathSet store; + for (auto & i : readDirectory(realStoreDir)) { + try { + store.insert({i.name}); + } catch (BadStorePath &) { } + } /* Check whether all valid paths actually exist. */ printInfo("checking path existence..."); @@ -1595,14 +1599,14 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) } -void LocalStore::verifyPath(const StorePath & path, const StringSet & store, +void LocalStore::verifyPath(const StorePath & path, const StorePathSet & store, StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) { checkInterrupt(); if (!done.insert(path).second) return; - if (!store.count(std::string(path.to_string()))) { + if (!store.count(path)) { /* Check any referrers first. If we can invalidate them first, then we can invalidate this path as well. */ bool canInvalidate = true; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c9b570eaa..e97195f5b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -314,7 +314,7 @@ private: */ void invalidatePathChecked(const StorePath & path); - void verifyPath(const StorePath & path, const StringSet & store, + void verifyPath(const StorePath & path, const StorePathSet & store, StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); std::shared_ptr queryPathInfoInternal(State & state, const StorePath & path); From 6525265f4640221efb0039ddbd6849a3b04babc9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 31 Jul 2023 12:22:06 -0400 Subject: [PATCH 155/909] `LocalStore::verifyPath`: Try to clarify data flow with more scopes It was initially unclear to me which of these are temporary state for the verify paths computation, and which of these are the results of that computation to be used in the rest of the function. Now, it is clear, and enforced. --- src/libstore/local-store.cc | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 9049f33aa..17e2ebc38 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1499,21 +1499,24 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) auto fdGCLock = openGCLock(); FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock..."); - StorePathSet store; - for (auto & i : readDirectory(realStoreDir)) { - try { - store.insert({i.name}); - } catch (BadStorePath &) { } - } - - /* Check whether all valid paths actually exist. */ - printInfo("checking path existence..."); - StorePathSet validPaths; - StorePathSet done; - for (auto & i : queryAllValidPaths()) - verifyPath(i, store, done, validPaths, repair, errors); + { + StorePathSet store; + for (auto & i : readDirectory(realStoreDir)) { + try { + store.insert({i.name}); + } catch (BadStorePath &) { } + } + + /* Check whether all valid paths actually exist. */ + printInfo("checking path existence..."); + + StorePathSet done; + + for (auto & i : queryAllValidPaths()) + verifyPath(i, store, done, validPaths, repair, errors); + } /* Optionally, check the content hashes (slow). */ if (checkContents) { From b9615419688353f4133520dc11456f7653e4cd08 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 31 Jul 2023 22:51:06 +0200 Subject: [PATCH 156/909] labeler: Stop removing labels > Whether or not to remove labels when matching files are reverted or no longer changed by the PR https://github.com/actions/labeler#inputs --- .github/workflows/labels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index 5f949ddc5..d83cb4f18 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -21,4 +21,4 @@ jobs: - uses: actions/labeler@v4 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - sync-labels: true + sync-labels: false From d9e7758f4756c10592407b3fe99dc253ef56eac9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 1 Aug 2023 16:07:20 +0200 Subject: [PATCH 157/909] Don't require .tar/.zip extension for tarball flakerefs Special-casing the file name is rather ugly, so we shouldn't do that. So now any {file,http,https} URL is handled by TarballInputScheme, except for non-flake inputs (i.e. inputs that have the attribute `flake = false`). --- src/libexpr/flake/flakeref.cc | 6 +++--- src/libfetchers/fetchers.cc | 8 ++++---- src/libfetchers/fetchers.hh | 6 +++--- src/libfetchers/git.cc | 2 +- src/libfetchers/github.cc | 2 +- src/libfetchers/indirect.cc | 2 +- src/libfetchers/mercurial.cc | 2 +- src/libfetchers/path.cc | 2 +- src/libfetchers/tarball.cc | 18 +++++++++--------- tests/fetchTree-file.sh | 14 +++++++++++++- 10 files changed, 37 insertions(+), 25 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 08adbe0c9..d3fa1d557 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -105,7 +105,7 @@ std::pair parseFlakeRefWithFragment( }; return std::make_pair( - FlakeRef(Input::fromURL(parsedURL), ""), + FlakeRef(Input::fromURL(parsedURL, isFlake), ""), percentDecode(match.str(6))); } @@ -176,7 +176,7 @@ std::pair parseFlakeRefWithFragment( parsedURL.query.insert_or_assign("shallow", "1"); return std::make_pair( - FlakeRef(Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")), + FlakeRef(Input::fromURL(parsedURL, isFlake), getOr(parsedURL.query, "dir", "")), fragment); } @@ -204,7 +204,7 @@ std::pair parseFlakeRefWithFragment( std::string fragment; std::swap(fragment, parsedURL.fragment); - auto input = Input::fromURL(parsedURL); + auto input = Input::fromURL(parsedURL, isFlake); input.parent = baseDir; return std::make_pair( diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index f86c0604e..e683b9f80 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -13,9 +13,9 @@ void registerInputScheme(std::shared_ptr && inputScheme) inputSchemes->push_back(std::move(inputScheme)); } -Input Input::fromURL(const std::string & url) +Input Input::fromURL(const std::string & url, bool requireTree) { - return fromURL(parseURL(url)); + return fromURL(parseURL(url), requireTree); } static void fixupInput(Input & input) @@ -31,10 +31,10 @@ static void fixupInput(Input & input) input.locked = true; } -Input Input::fromURL(const ParsedURL & url) +Input Input::fromURL(const ParsedURL & url, bool requireTree) { for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromURL(url); + auto res = inputScheme->inputFromURL(url, requireTree); if (res) { res->scheme = inputScheme; fixupInput(*res); diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index d0738f619..6e10e9513 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -44,9 +44,9 @@ struct Input std::optional parent; public: - static Input fromURL(const std::string & url); + static Input fromURL(const std::string & url, bool requireTree = true); - static Input fromURL(const ParsedURL & url); + static Input fromURL(const ParsedURL & url, bool requireTree = true); static Input fromAttrs(Attrs && attrs); @@ -129,7 +129,7 @@ struct InputScheme virtual ~InputScheme() { } - virtual std::optional inputFromURL(const ParsedURL & url) const = 0; + virtual std::optional inputFromURL(const ParsedURL & url, bool requireTree) const = 0; virtual std::optional inputFromAttrs(const Attrs & attrs) const = 0; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index be5842d53..f8d89ab2f 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -256,7 +256,7 @@ std::pair fetchFromWorkdir(ref store, Input & input, co struct GitInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != "git" && url.scheme != "git+http" && diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 80598e7f8..291f457f0 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -30,7 +30,7 @@ struct GitArchiveInputScheme : InputScheme virtual std::optional> accessHeaderFromToken(const std::string & token) const = 0; - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != type()) return {}; diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index b99504a16..4874a43ff 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -7,7 +7,7 @@ std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); struct IndirectInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != "flake") return {}; diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 86e8f81f4..51fd1ed42 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -43,7 +43,7 @@ static std::string runHg(const Strings & args, const std::optional struct MercurialInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != "hg+http" && url.scheme != "hg+https" && diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 61541e69d..01f1be978 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -6,7 +6,7 @@ namespace nix::fetchers { struct PathInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != "path") return {}; diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index a012234e0..107d38e92 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -194,11 +194,11 @@ struct CurlInputScheme : InputScheme || hasSuffix(path, ".tar.zst"); } - virtual bool isValidURL(const ParsedURL & url) const = 0; + virtual bool isValidURL(const ParsedURL & url, bool requireTree) const = 0; - std::optional inputFromURL(const ParsedURL & _url) const override + std::optional inputFromURL(const ParsedURL & _url, bool requireTree) const override { - if (!isValidURL(_url)) + if (!isValidURL(_url, requireTree)) return std::nullopt; Input input; @@ -265,13 +265,13 @@ struct FileInputScheme : CurlInputScheme { const std::string inputType() const override { return "file"; } - bool isValidURL(const ParsedURL & url) const override + bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application - ? parsedUrlScheme.application.value() == inputType() - : !hasTarballExtension(url.path)); + ? parsedUrlScheme.application.value() == inputType() + : (!requireTree && !hasTarballExtension(url.path))); } std::pair fetch(ref store, const Input & input) override @@ -285,14 +285,14 @@ struct TarballInputScheme : CurlInputScheme { const std::string inputType() const override { return "tarball"; } - bool isValidURL(const ParsedURL & url) const override + bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application - ? parsedUrlScheme.application.value() == inputType() - : hasTarballExtension(url.path)); + ? parsedUrlScheme.application.value() == inputType() + : (requireTree || hasTarballExtension(url.path))); } std::pair fetch(ref store, const Input & _input) override diff --git a/tests/fetchTree-file.sh b/tests/fetchTree-file.sh index fe569cfb8..6395c133d 100644 --- a/tests/fetchTree-file.sh +++ b/tests/fetchTree-file.sh @@ -27,6 +27,7 @@ test_file_flake_input () { mkdir inputs echo foo > inputs/test_input_file + echo '{ outputs = { self }: { }; }' > inputs/flake.nix tar cfa test_input.tar.gz inputs cp test_input.tar.gz test_input_no_ext input_tarball_hash="$(nix hash path test_input.tar.gz)" @@ -50,6 +51,9 @@ test_file_flake_input () { url = "file+file://$PWD/test_input.tar.gz"; flake = false; }; + inputs.flake_no_ext = { + url = "file://$PWD/test_input_no_ext"; + }; outputs = { ... }: {}; } EOF @@ -58,7 +62,7 @@ EOF nix eval --file - < Date: Wed, 2 Aug 2023 12:40:04 -0400 Subject: [PATCH 158/909] local-store verifying: Rename `store` to something more clear It is not a `Store` but a `StorePathSet`. --- src/libstore/local-store.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 17e2ebc38..982a9059c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1502,10 +1502,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) StorePathSet validPaths; { - StorePathSet store; + StorePathSet storePathsInStoreDir; for (auto & i : readDirectory(realStoreDir)) { try { - store.insert({i.name}); + storePathsInStoreDir.insert({i.name}); } catch (BadStorePath &) { } } @@ -1515,7 +1515,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) StorePathSet done; for (auto & i : queryAllValidPaths()) - verifyPath(i, store, done, validPaths, repair, errors); + verifyPath(i, storePathsInStoreDir, done, validPaths, repair, errors); } /* Optionally, check the content hashes (slow). */ @@ -1602,21 +1602,21 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) } -void LocalStore::verifyPath(const StorePath & path, const StorePathSet & store, +void LocalStore::verifyPath(const StorePath & path, const StorePathSet & storePathsInStoreDir, StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) { checkInterrupt(); if (!done.insert(path).second) return; - if (!store.count(path)) { + if (!storePathsInStoreDir.count(path)) { /* Check any referrers first. If we can invalidate them first, then we can invalidate this path as well. */ bool canInvalidate = true; StorePathSet referrers; queryReferrers(path, referrers); for (auto & i : referrers) if (i != path) { - verifyPath(i, store, done, validPaths, repair, errors); + verifyPath(i, storePathsInStoreDir, done, validPaths, repair, errors); if (validPaths.count(i)) canInvalidate = false; } From 66550878dffe2a4bf55ea7ab694738aa14e6a01e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 2 Aug 2023 12:45:55 -0400 Subject: [PATCH 159/909] Add comment explaining the use of `readDirectory(realStoreDir)` --- src/libstore/local-store.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 982a9059c..40a3bc194 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1503,6 +1503,15 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) { StorePathSet storePathsInStoreDir; + /* Why aren't we using `queryAllValidPaths`? Because that would + tell us about all the paths than the database knows about. Here we + want to know about all the store paths in the store directory, + regardless of what the database thinks. + + We will end up cross-referencing these two sources of truth (the + database and the filesystem) in the loop below, in order to catch + invalid states. + */ for (auto & i : readDirectory(realStoreDir)) { try { storePathsInStoreDir.insert({i.name}); From 9b908fa70a07219a110ddd63ec3593c2c2269918 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 24 Jul 2023 14:02:05 -0400 Subject: [PATCH 160/909] Factor out `nix-defexpr` path computation Avoid duplicated code, and also avoid "on the fly" path construction (which makes it harder to keep track of which paths we use). The factored out code doesn't create the Nix state dir anymore, but this is fine because other in nix-env and nix-channel does: - nix-channel: Line 158 in this commit - nix-env: Line 1407 in this commit --- src/libexpr/eval-settings.cc | 9 ++++++++- src/libexpr/eval-settings.hh | 5 +++++ src/nix-channel/nix-channel.cc | 3 ++- src/nix-env/nix-env.cc | 3 ++- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 422aaf8d5..93b4a5289 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -63,7 +63,7 @@ Strings EvalSettings::getDefaultNixPath() }; if (!evalSettings.restrictEval && !evalSettings.pureEval) { - add(settings.useXDGBaseDirectories ? getStateDir() + "/nix/defexpr/channels" : getHome() + "/.nix-defexpr/channels"); + add(getNixDefExpr() + "/channels"); add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); add(rootChannelsDir()); } @@ -92,4 +92,11 @@ EvalSettings evalSettings; static GlobalConfig::Register rEvalSettings(&evalSettings); +Path getNixDefExpr() +{ + return settings.useXDGBaseDirectories + ? getStateDir() + "/nix/defexpr" + : getHome() + "/.nix-defexpr"; +} + } diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 043af6cab..e6666061a 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -95,4 +95,9 @@ struct EvalSettings : Config extern EvalSettings evalSettings; +/** + * Conventionally part of the default nix path in impure mode. + */ +Path getNixDefExpr(); + } diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index c1c8edd1d..95f401441 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -5,6 +5,7 @@ #include "store-api.hh" #include "legacy.hh" #include "fetchers.hh" +#include "eval-settings.hh" // for defexpr #include "util.hh" #include @@ -165,7 +166,7 @@ static int main_nix_channel(int argc, char ** argv) // Figure out the name of the `.nix-channels' file to use auto home = getHome(); channelsList = settings.useXDGBaseDirectories ? createNixStateDir() + "/channels" : home + "/.nix-channels"; - nixDefExpr = settings.useXDGBaseDirectories ? createNixStateDir() + "/defexpr" : home + "/.nix-defexpr"; + nixDefExpr = getNixDefExpr(); // Figure out the name of the channels profile. profile = profilesDir() + "/channels"; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 91b073b49..3dc3db0fc 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -15,6 +15,7 @@ #include "value-to-json.hh" #include "xml-writer.hh" #include "legacy.hh" +#include "eval-settings.hh" // for defexpr #include #include @@ -1399,7 +1400,7 @@ static int main_nix_env(int argc, char * * argv) globals.instSource.type = srcUnknown; globals.instSource.systemFilter = "*"; - Path nixExprPath = settings.useXDGBaseDirectories ? createNixStateDir() + "/defexpr" : getHome() + "/.nix-defexpr"; + Path nixExprPath = getNixDefExpr(); if (!pathExists(nixExprPath)) { try { From 3b592c880ae76707cc832e5f227e261be29661bf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Apr 2023 11:58:47 -0400 Subject: [PATCH 161/909] Add infra for experimental store implemenations This is analogous to that for experimental settings and flags that we have also added as of late. --- doc/manual/generate-manpage.nix | 21 +++++++++++++++++++-- src/libstore/store-api.cc | 1 + src/libstore/store-api.hh | 15 +++++++++++++++ src/nix/main.cc | 6 ++++-- 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index fb34898f3..65eec42d0 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -137,12 +137,29 @@ let storeDocs = let - showStore = name: { settings, doc }: - '' + showStore = name: { settings, doc, experimentalFeature }: + let + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > This store is part of an + > [experimental feature](@docroot@/contributing/experimental-features.md). + + To use this store, you need to make sure the corresponding experimental feature, + [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), + is enabled. + For example, include the following in [`nix.conf`](#): + + ``` + extra-experimental-features = ${experimentalFeature} + ``` + ''; + in '' ## ${name} ${doc} + ${experimentalFeatureNote} + **Settings**: ${showSettings { useAnchors = false; } settings} diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 4ea16a1c0..28689e100 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1496,6 +1496,7 @@ ref openStore(const std::string & uri_, if (implem.uriSchemes.count(parsedUri.scheme)) { auto store = implem.create(parsedUri.scheme, baseURI, params); if (store) { + experimentalFeatureSettings.require(store->experimentalFeature()); store->init(); store->warnUnknownSettings(); return ref(store); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 3758c730f..f9029ade1 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -109,13 +109,28 @@ struct StoreConfig : public Config virtual ~StoreConfig() { } + /** + * The name of this type of store. + */ virtual const std::string name() = 0; + /** + * Documentation for this type of store. + */ virtual std::string doc() { return ""; } + /** + * An experimental feature this type store is gated, if it is to be + * experimental. + */ + virtual std::optional experimentalFeature() const + { + return std::nullopt; + } + const PathSetting storeDir_{this, settings.nixStore, "store", R"( diff --git a/src/nix/main.cc b/src/nix/main.cc index df66beb8c..c5a9c8b33 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -180,8 +180,10 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs for (auto & implem : *Implementations::registered) { auto storeConfig = implem.getConfig(); auto storeName = storeConfig->name(); - stores[storeName]["doc"] = storeConfig->doc(); - stores[storeName]["settings"] = storeConfig->toJSON(); + auto & j = stores[storeName]; + j["doc"] = storeConfig->doc(); + j["settings"] = storeConfig->toJSON(); + j["experimentalFeature"] = storeConfig->experimentalFeature(); } res["stores"] = std::move(stores); From 7c09104a943978a165885e10697b418f8bab2795 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 4 Aug 2023 23:11:08 +0200 Subject: [PATCH 162/909] nix/why-depends: fix output of `--precise` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I haven't checked when this was exactly introduced, but on Nix 2.16 I realized that the additional lines inserted when using `--precise` are completely separated from the tree: nix why-depends /nix/store/ccgr4faaxys39s091qridxg1947lggh4-evcxr-0.14.2 /nix/store/b7hvml0m3qmqraz1022fwvyyg6fc1vdy-gcc-12.2.0 --precise --extra-experimental-features nix-command /nix/store/ccgr4faaxys39s091qridxg1947lggh4-evcxr-0.14.2 → /nix/store/lcf37pgp3rgww67v9x2990hbfwx96c1w-gcc-wrapper-12.2.0 → /nix/store/b7hvml0m3qmqraz1022fwvyyg6fc1vdy-gcc-12.2.0 └───bin/evcxr: …':'}.PATH=${PATH/':''/nix/store/lcf37pgp3rgww67v9x2990hbfwx96c1w-gcc-wrapper-12.2.0/bin'':'/':'}… └───bin/cpp: …k disable=SC2193.[[ "/nix/store/b7hvml0m3qmqraz1022fwvyyg6fc1vdy-gcc-12.2.0/bin/cpp" = *++ ]] &&… This is apparently because `std::cout` is buffered and flushed in the end whereas the rest of the output isn't. The fix is rather simple, just use `logger->cout` as it's already the case for the rest of the code. This way we also don't need to insert additional newlines in the `hits` map since that's something the logger takes care of. Also added a small test to make sure that the layout of this is somehow tested to reduce the risk of further regressions here. --- src/nix/why-depends.cc | 10 +++++----- tests/why-depends.sh | 5 +++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index a3a9dc698..592de773c 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -239,7 +239,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions if (pos != std::string::npos) { size_t margin = 32; auto pos2 = pos >= margin ? pos - margin : 0; - hits[hash].emplace_back(fmt("%s: …%s…\n", + hits[hash].emplace_back(fmt("%s: …%s…", p2, hilite(filterPrintable( std::string(contents, pos2, pos - pos2 + hash.size() + margin)), @@ -255,7 +255,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions for (auto & hash : hashes) { auto pos = target.find(hash); if (pos != std::string::npos) - hits[hash].emplace_back(fmt("%s -> %s\n", p2, + hits[hash].emplace_back(fmt("%s -> %s", p2, hilite(target, pos, StorePath::HashLen, getColour(hash)))); } } @@ -272,9 +272,9 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions for (auto & hit : hits[hash]) { bool first = hit == *hits[hash].begin(); - std::cout << tailPad - << (first ? (last ? treeLast : treeConn) : (last ? treeNull : treeLine)) - << hit; + logger->cout("%s%s%s", tailPad, + (first ? (last ? treeLast : treeConn) : (last ? treeNull : treeLine)), + hit); if (!all) break; } diff --git a/tests/why-depends.sh b/tests/why-depends.sh index b35a0d1cf..9680bf80e 100644 --- a/tests/why-depends.sh +++ b/tests/why-depends.sh @@ -22,3 +22,8 @@ echo "$PRECISE_WHY_DEPENDS_OUTPUT" | grepQuiet input-2 # But only the “precise” one should refer to `reference-to-input-2` echo "$FAST_WHY_DEPENDS_OUTPUT" | grepQuietInverse reference-to-input-2 echo "$PRECISE_WHY_DEPENDS_OUTPUT" | grepQuiet reference-to-input-2 + +<<<"$PRECISE_WHY_DEPENDS_OUTPUT" sed -n '2p' | grepQuiet "└───reference-to-input-2 -> " +<<<"$PRECISE_WHY_DEPENDS_OUTPUT" sed -n '3p' | grep " →" | grepQuiet "dependencies-input-2" +<<<"$PRECISE_WHY_DEPENDS_OUTPUT" sed -n '4p' | grepQuiet " └───input0: …" # in input-2, file input0 +<<<"$PRECISE_WHY_DEPENDS_OUTPUT" sed -n '5p' | grep " →" | grepQuiet "dependencies-input-0" # is dependencies-input-0 referenced From 3fefc2b28494468c7a6078430eaaf8a6c8f17230 Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Fri, 28 Jul 2023 22:23:56 +0200 Subject: [PATCH 163/909] Fix derivation load assertion errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When loading a derivation from a JSON, malformed input would trigger cryptic "assertion failed" errors. Simply replacing calls to `operator []` with calls to `.at()` was not enough, as this would cause json.execptions to be printed verbatim. Display nice error messages instead and give some indication where the error happened. *Before:* ``` $ echo 4 | nix derivation add error: [json.exception.type_error.305] cannot use operator[] with a string argument with number $ nix derivation show nixpkgs#hello | nix derivation add Assertion failed: (it != m_value.object->end()), function operator[], file /nix/store/8h9pxgq1776ns6qi5arx08ifgnhmgl22-nlohmann_json-3.11.2/include/nlohmann/json.hpp, line 2135. $ nix derivation show nixpkgs#hello | jq '.[] | .name = 5' | nix derivation add error: [json.exception.type_error.302] type must be string, but is object $ nix derivation show nixpkgs#hello | jq '.[] | .outputs = { out: "/nix/store/8j3f8j-hello" }' | nix derivation add error: [json.exception.type_error.302] type must be object, but is string ``` *After:* ``` $ echo 4 | nix derivation add error: Expected JSON of derivation to be of type 'object', but it is of type 'number' $ nix derivation show nixpkgs#hello | nix derivation add error: Expected JSON object to contain key 'name' but it doesn't $ nix derivation show nixpkgs#hello | jq '.[] | .name = 5' | nix derivation add error: Expected JSON value to be of type 'string' but it is of type 'number' $ nix derivation show nixpkgs#hello | jq '.[] | .outputs = { out: "/nix/store/8j3f8j-hello" }' | nix derivation add error: … while reading key 'outputs' error: Expected JSON value to be of type 'object' but it is of type 'string' ``` --- doc/manual/src/release-notes/rl-next.md | 2 ++ src/libstore/derivations.cc | 40 +++++++++++++++++-------- src/libutil/json-utils.cc | 24 +++++++++++++++ src/libutil/json-utils.hh | 22 ++++++++++++++ 4 files changed, 76 insertions(+), 12 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index df30ce83d..526b64fde 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -8,3 +8,5 @@ These functions are useful for converting between flake references encoded as attribute sets and URLs. - [`builtins.toJSON`](@docroot@/language/builtins.md#builtins-parseFlakeRef) now prints [--show-trace](@docroot@/command-ref/conf-file.html#conf-show-trace) items for the path in which it finds an evaluation error. + +- Error messages regarding malformed input to [`derivation add`](@docroot@/command-ref/new-cli/nix3-derivation-add.md) are now clearer and more detailed. diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index b831b2fe5..8a84bb1c5 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -993,6 +993,7 @@ DerivationOutput DerivationOutput::fromJSON( const ExperimentalFeatureSettings & xpSettings) { std::set keys; + ensureType(_json, nlohmann::detail::value_t::object); auto json = (std::map) _json; for (const auto & [key, _] : json) @@ -1097,36 +1098,51 @@ Derivation Derivation::fromJSON( const Store & store, const nlohmann::json & json) { + using nlohmann::detail::value_t; + Derivation res; - res.name = json["name"]; + ensureType(json, value_t::object); - { - auto & outputsObj = json["outputs"]; + res.name = ensureType(valueAt(json, "name"), value_t::string); + + try { + auto & outputsObj = ensureType(valueAt(json, "outputs"), value_t::object); for (auto & [outputName, output] : outputsObj.items()) { res.outputs.insert_or_assign( outputName, DerivationOutput::fromJSON(store, res.name, outputName, output)); } + } catch (Error & e) { + e.addTrace({}, "while reading key 'outputs'"); + throw; } - { - auto & inputsList = json["inputSrcs"]; + try { + auto & inputsList = ensureType(valueAt(json, "inputSrcs"), value_t::array); for (auto & input : inputsList) res.inputSrcs.insert(store.parseStorePath(static_cast(input))); + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputSrcs'"); + throw; } - { - auto & inputDrvsObj = json["inputDrvs"]; - for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) + try { + auto & inputDrvsObj = ensureType(valueAt(json, "inputDrvs"), value_t::object); + for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) { + ensureType(inputOutputs, value_t::array); res.inputDrvs[store.parseStorePath(inputDrvPath)] = static_cast(inputOutputs); + } + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputDrvs'"); + throw; } - res.platform = json["system"]; - res.builder = json["builder"]; - res.args = json["args"]; - res.env = json["env"]; + res.platform = ensureType(valueAt(json, "system"), value_t::string); + res.builder = ensureType(valueAt(json, "builder"), value_t::string); + res.args = ensureType(valueAt(json, "args"), value_t::array); + res.env = ensureType(valueAt(json, "env"), value_t::object); return res; } diff --git a/src/libutil/json-utils.cc b/src/libutil/json-utils.cc index d7220e71d..61cef743d 100644 --- a/src/libutil/json-utils.cc +++ b/src/libutil/json-utils.cc @@ -1,4 +1,5 @@ #include "json-utils.hh" +#include "error.hh" namespace nix { @@ -16,4 +17,27 @@ nlohmann::json * get(nlohmann::json & map, const std::string & key) return &*i; } +const nlohmann::json & valueAt( + const nlohmann::json & map, + const std::string & key) +{ + if (!map.contains(key)) + throw Error("Expected JSON object to contain key '%s' but it doesn't", key); + + return map[key]; +} + +const nlohmann::json & ensureType( + const nlohmann::json & value, + nlohmann::json::value_type expectedType + ) +{ + if (value.type() != expectedType) + throw Error( + "Expected JSON value to be of type '%s' but it is of type '%s'", + nlohmann::json(expectedType).type_name(), + value.type_name()); + + return value; +} } diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh index 5e63c1af4..77c63595c 100644 --- a/src/libutil/json-utils.hh +++ b/src/libutil/json-utils.hh @@ -10,6 +10,28 @@ const nlohmann::json * get(const nlohmann::json & map, const std::string & key); nlohmann::json * get(nlohmann::json & map, const std::string & key); +/** + * Get the value of a json object at a key safely, failing + * with a Nix Error if the key does not exist. + * + * Use instead of nlohmann::json::at() to avoid ugly exceptions. + * + * _Does not check whether `map` is an object_, use `ensureType` for that. + */ +const nlohmann::json & valueAt( + const nlohmann::json & map, + const std::string & key); + +/** + * Ensure the type of a json object is what you expect, failing + * with a Nix Error if it isn't. + * + * Use before type conversions and element access to avoid ugly exceptions. + */ +const nlohmann::json & ensureType( + const nlohmann::json & value, + nlohmann::json::value_type expectedType); + /** * For `adl_serializer>` below, we need to track what * types are not already using `null`. Only for them can we use `null` From 31a6e10fe52fd4265501553ca479c51b510137e1 Mon Sep 17 00:00:00 2001 From: Simon Rainerson Date: Tue, 31 Jan 2023 11:36:13 +0100 Subject: [PATCH 164/909] Fix misread of source if path is already valid When receiving a stream of NARs through the ssh-ng protocol, an already existing path would cause the NAR archive to not be read in the stream, resulting in trying to parse the NAR as a ValidPathInfo. This results in the error message: error: not an absolute path: 'nix-archive-1' Fixes #6253 Usually this problem is avoided by running QueryValidPaths before AddMultipleToStore, but can arise when two parallel nix processes gets the same response from QueryValidPaths. This makes the problem more prominent when running builds in parallel. --- src/libstore/local-store.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 40a3bc194..17b4ecc73 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1196,6 +1196,15 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (checkSigs && pathInfoIsUntrusted(info)) throw Error("cannot add path '%s' because it lacks a signature by a trusted key", printStorePath(info.path)); + /* In case we are not interested in reading the NAR: discard it. */ + bool narRead = false; + Finally cleanup = [&]() { + if (!narRead) { + ParseSink sink; + parseDump(sink, source); + } + }; + addTempRoot(info.path); if (repair || !isValidPath(info.path)) { @@ -1220,6 +1229,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, TeeSource wrapperSource { source, hashSink }; + narRead = true; restorePath(realPath, wrapperSource); auto hashResult = hashSink.finish(); From ad410abbe0d66a72927bd715af355d88a4e939d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 12 May 2023 09:56:39 +0200 Subject: [PATCH 165/909] Stabilize `discard-references` It has been there for a few releases now (landed in 2.14.0), doesn't seem to cause any major issue and is wanted in a few places (https://github.com/NixOS/nix/pull/7087#issuecomment-1544471346). --- doc/manual/src/language/advanced-attributes.md | 10 ---------- doc/manual/src/release-notes/rl-next.md | 6 ++++++ src/libstore/build/local-derivation-goal.cc | 1 - src/libutil/experimental-features.cc | 11 +---------- src/libutil/experimental-features.hh | 1 - tests/check-refs.sh | 6 ++++-- 6 files changed, 11 insertions(+), 24 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 307971434..5e8aaeba0 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -320,16 +320,6 @@ Derivations can declare some infrequently used optional attributes. ``` - [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\ - > **Warning** - > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). - > - > To use this attribute, you must enable the - > [`discard-references`](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) experimental feature. - > For example, in [nix.conf](../command-ref/conf-file.md) you could add: - > - > ``` - > extra-experimental-features = discard-references - > ``` When using [structured attributes](#adv-attr-structuredAttrs), the attribute `unsafeDiscardReferences` is an attribute set with a boolean value for each output name. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 526b64fde..6516a2663 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -10,3 +10,9 @@ - [`builtins.toJSON`](@docroot@/language/builtins.md#builtins-parseFlakeRef) now prints [--show-trace](@docroot@/command-ref/conf-file.html#conf-show-trace) items for the path in which it finds an evaluation error. - Error messages regarding malformed input to [`derivation add`](@docroot@/command-ref/new-cli/nix3-derivation-add.md) are now clearer and more detailed. + +- The `discard-references` feature has been stabilized. + This means that the + [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) + attribute is no longer guarded by an experimental flag and can be used + freely. diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index b7a27490c..8b7fbdcda 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2307,7 +2307,6 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() bool discardReferences = false; if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) { - experimentalFeatureSettings.require(Xp::DiscardReferences); if (auto output = get(*udr, outputName)) { if (!output->is_boolean()) throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 7c4112d32..782331283 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -182,15 +182,6 @@ constexpr std::array xpFeatureDetails = {{ the [`use-cgroups`](#conf-use-cgroups) setting for details. )", }, - { - .tag = Xp::DiscardReferences, - .name = "discard-references", - .description = R"( - Allow the use of the [`unsafeDiscardReferences`](@docroot@/language/advanced-attributes.html#adv-attr-unsafeDiscardReferences) attribute in derivations - that use [structured attributes](@docroot@/language/advanced-attributes.html#adv-attr-structuredAttrs). This disables scanning of outputs for - runtime dependencies. - )", - }, { .tag = Xp::DaemonTrustOverride, .name = "daemon-trust-override", diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index faf2e9398..add592ae6 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -27,7 +27,6 @@ enum struct ExperimentalFeature ReplFlake, AutoAllocateUids, Cgroups, - DiscardReferences, DaemonTrustOverride, DynamicDerivations, ParseTomlTimestamps, diff --git a/tests/check-refs.sh b/tests/check-refs.sh index 2778e491d..3b587d1e5 100644 --- a/tests/check-refs.sh +++ b/tests/check-refs.sh @@ -42,8 +42,10 @@ nix-build -o $RESULT check-refs.nix -A test7 nix-build -o $RESULT check-refs.nix -A test10 if isDaemonNewer 2.12pre20230103; then - enableFeatures discard-references - restartDaemon + if ! isDaemonNewer 2.16.0; then + enableFeatures discard-references + restartDaemon + fi # test11 should succeed. test11=$(nix-build -o $RESULT check-refs.nix -A test11) From afac001c39aadedcbe52ce45fbde8220834cf13f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 7 Aug 2023 11:23:19 +0200 Subject: [PATCH 166/909] Test the parallel copy over ssh-ng Regression test for https://github.com/NixOS/nix/issues/6253 --- tests/local.mk | 1 + tests/nix-copy-ssh-ng.sh | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 tests/nix-copy-ssh-ng.sh diff --git a/tests/local.mk b/tests/local.mk index 2afe91220..4edf31303 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -90,6 +90,7 @@ nix_tests = \ zstd.sh \ compression-levels.sh \ nix-copy-ssh.sh \ + nix-copy-ssh-ng.sh \ post-hook.sh \ function-trace.sh \ flakes/config.sh \ diff --git a/tests/nix-copy-ssh-ng.sh b/tests/nix-copy-ssh-ng.sh new file mode 100644 index 000000000..45e53c9c0 --- /dev/null +++ b/tests/nix-copy-ssh-ng.sh @@ -0,0 +1,18 @@ +source common.sh + +clearStore +clearCache + +remoteRoot=$TEST_ROOT/store2 +chmod -R u+w "$remoteRoot" || true +rm -rf "$remoteRoot" + +outPath=$(nix-build --no-out-link dependencies.nix) + +nix store ping --store "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" + +# Regression test for https://github.com/NixOS/nix/issues/6253 +nix copy --to "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath --no-check-sigs & +nix copy --to "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath --no-check-sigs + +[ -f $remoteRoot$outPath/foobar ] From 4b1bd822ac7fd35b30f37c1b7705c523cc462761 Mon Sep 17 00:00:00 2001 From: Peter Waller Date: Fri, 30 Jun 2023 16:11:12 +0100 Subject: [PATCH 167/909] Try to realise CA derivations during queryMissing This enables nix to correctly report what will be fetched in the case that everything is a cache hit. Note however that if an intermediate build of something which is not cached could still cause products to end up being substituted if the intermediate build results in a CA path which is in the cache. Fixes #8615. Signed-off-by: Peter Waller --- src/libstore/misc.cc | 30 ++++++++++++++++++++ tests/ca/build-cache.sh | 51 ++++++++++++++++++++++++++++++++++ tests/ca/build.sh | 12 +------- tests/ca/content-addressed.nix | 18 ++++++++++++ tests/ca/local.mk | 1 + 5 files changed, 101 insertions(+), 11 deletions(-) create mode 100644 tests/ca/build-cache.sh diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 14160dc8b..2fdf0a68d 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -200,6 +200,36 @@ void Store::queryMissing(const std::vector & targets, auto drv = make_ref(derivationFromPath(bfd.drvPath)); ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv); + if (!knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { + experimentalFeatureSettings.require(Xp::CaDerivations); + + // If there are unknown output paths, attempt to find if the + // paths are known to substituters through a realisation. + auto outputHashes = staticOutputHashes(*this, *drv); + knownOutputPaths = true; + + for (auto [outputName, hash] : outputHashes) { + if (!bfd.outputs.contains(outputName)) + continue; + + bool found = false; + for (auto &sub : getDefaultSubstituters()) { + auto realisation = sub->queryRealisation({hash, outputName}); + if (!realisation) + continue; + found = true; + if (!isValidPath(realisation->outPath)) + invalid.insert(realisation->outPath); + break; + } + if (!found) { + // Some paths did not have a realisation, this must be built. + knownOutputPaths = false; + break; + } + } + } + if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref>(DrvState(invalid.size())); for (auto & output : invalid) diff --git a/tests/ca/build-cache.sh b/tests/ca/build-cache.sh new file mode 100644 index 000000000..6a4080fec --- /dev/null +++ b/tests/ca/build-cache.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +source common.sh + +# The substituters didn't work prior to this time. +requireDaemonNewerThan "2.18.0pre20230808" + +drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1)^out +nix derivation show "$drv" --arg seed 1 + +buildAttr () { + local derivationPath=$1 + local seedValue=$2 + shift; shift + local args=("./content-addressed.nix" "-A" "$derivationPath" --arg seed "$seedValue" "--no-out-link") + args+=("$@") + nix-build "${args[@]}" +} + +copyAttr () { + local derivationPath=$1 + local seedValue=$2 + shift; shift + local args=("-f" "./content-addressed.nix" "$derivationPath" --arg seed "$seedValue") + args+=("$@") + # Note: to copy CA derivations, we need to copy the realisations, which + # currently requires naming the installables, not just the derivation output + # path. + nix copy --to file://$cacheDir "${args[@]}" +} + +testRemoteCacheFor () { + local derivationPath=$1 + clearCache + copyAttr "$derivationPath" 1 + clearStore + # Check nothing gets built. + buildAttr "$derivationPath" 1 --option substituters file://$cacheDir --no-require-sigs |& grepQuietInverse " will be built:" +} + +testRemoteCache () { + testRemoteCacheFor rootCA + testRemoteCacheFor dependentCA + testRemoteCacheFor dependentNonCA + testRemoteCacheFor dependentFixedOutput + testRemoteCacheFor dependentForBuildCA + testRemoteCacheFor dependentForBuildNonCA +} + +clearStore +testRemoteCache \ No newline at end of file diff --git a/tests/ca/build.sh b/tests/ca/build.sh index 7754ad276..e1a8a7625 100644 --- a/tests/ca/build.sh +++ b/tests/ca/build.sh @@ -2,7 +2,7 @@ source common.sh -drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1) +drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1)^out nix derivation show "$drv" --arg seed 1 buildAttr () { @@ -14,14 +14,6 @@ buildAttr () { nix-build "${args[@]}" } -testRemoteCache () { - clearCache - local outPath=$(buildAttr dependentNonCA 1) - nix copy --to file://$cacheDir $outPath - clearStore - buildAttr dependentNonCA 1 --option substituters file://$cacheDir --no-require-sigs |& grepQuietInverse "building dependent-non-ca" -} - testDeterministicCA () { [[ $(buildAttr rootCA 1) = $(buildAttr rootCA 2) ]] } @@ -66,8 +58,6 @@ testNormalization () { test "$(stat -c %Y $outPath)" -eq 1 } -# Disabled until we have it properly working -# testRemoteCache clearStore testNormalization testDeterministicCA diff --git a/tests/ca/content-addressed.nix b/tests/ca/content-addressed.nix index 81bc4bf5c..2559c562f 100644 --- a/tests/ca/content-addressed.nix +++ b/tests/ca/content-addressed.nix @@ -61,6 +61,24 @@ rec { echo ${rootCA}/non-ca-hello > $out/dep ''; }; + dependentForBuildCA = mkCADerivation { + name = "dependent-for-build-ca"; + buildCommand = '' + echo "Depends on rootCA for building only" + mkdir -p $out + echo ${rootCA} + touch $out + ''; + }; + dependentForBuildNonCA = mkDerivation { + name = "dependent-for-build-non-ca"; + buildCommand = '' + echo "Depends on rootCA for building only" + mkdir -p $out + echo ${rootCA} + touch $out + ''; + }; dependentFixedOutput = mkDerivation { name = "dependent-fixed-output"; outputHashMode = "recursive"; diff --git a/tests/ca/local.mk b/tests/ca/local.mk index d15312708..0852e592e 100644 --- a/tests/ca/local.mk +++ b/tests/ca/local.mk @@ -1,6 +1,7 @@ ca-tests := \ $(d)/build-with-garbage-path.sh \ $(d)/build.sh \ + $(d)/build-cache.sh \ $(d)/concurrent-builds.sh \ $(d)/derivation-json.sh \ $(d)/duplicate-realisation-in-closure.sh \ From 60b7121d2c6d4322b7c2e8e7acfec7b701b2d3a1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 15 Jan 2023 17:39:04 -0500 Subject: [PATCH 168/909] Make the Derived Path family of types inductive for dynamic derivations We want to be able to write down `foo.drv^bar.drv^baz`: `foo.drv^bar.drv` is the dynamic derivation (since it is itself a derivation output, `bar.drv` from `foo.drv`). To that end, we create `Single{Derivation,BuiltPath}` types, that are very similar except instead of having multiple outputs (in a set or map), they have a single one. This is for everything to the left of the rightmost `^`. `NixStringContextElem` has an analogous change, and now can reuse `SingleDerivedPath` at the top level. In fact, if we ever get rid of `DrvDeep`, `NixStringContextElem` could be replaced with `SingleDerivedPath` entirely! Important note: some JSON formats have changed. We already can *produce* dynamic derivations, but we can't refer to them directly. Today, we can merely express building or example at the top imperatively over time by building `foo.drv^bar.drv`, and then with a second nix invocation doing `^baz`, but this is not declarative. The ethos of Nix of being able to write down the full plan everything you want to do, and then execute than plan with a single command, and for that we need the new inductive form of these types. Co-authored-by: Robert Hensing Co-authored-by: Valentin Gagarin --- doc/manual/src/release-notes/rl-next.md | 3 + src/build-remote/build-remote.cc | 7 +- src/libcmd/built-path.cc | 76 +++++- src/libcmd/built-path.hh | 50 +++- src/libcmd/installable-attr-path.cc | 2 +- src/libcmd/installable-derived-path.cc | 15 +- src/libcmd/installable-flake.cc | 2 +- src/libcmd/installable-value.cc | 3 +- src/libcmd/installables.cc | 36 ++- src/libcmd/repl.cc | 2 +- src/libexpr/eval-cache.cc | 2 +- src/libexpr/eval.cc | 59 +++-- src/libexpr/eval.hh | 10 +- src/libexpr/primops.cc | 15 +- src/libexpr/primops/context.cc | 7 +- src/libexpr/tests/derived-path.cc | 26 +-- src/libexpr/tests/value/context.cc | 70 ++++-- src/libexpr/value/context.cc | 97 +++++--- src/libexpr/value/context.hh | 21 +- src/libstore/build/derivation-goal.cc | 10 +- src/libstore/build/entry-points.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 15 +- src/libstore/build/worker.cc | 7 +- src/libstore/derived-path.cc | 247 ++++++++++++++++++-- src/libstore/derived-path.hh | 222 ++++++++++++++++-- src/libstore/downstream-placeholder.cc | 15 ++ src/libstore/downstream-placeholder.hh | 12 + src/libstore/legacy-ssh-store.cc | 3 + src/libstore/misc.cc | 85 ++++++- src/libstore/path-with-outputs.cc | 47 ++-- src/libstore/path-with-outputs.hh | 4 +- src/libstore/remote-store.cc | 21 +- src/libstore/store-api.hh | 1 + src/libstore/tests/derived-path.cc | 91 +++++++- src/libstore/tests/derived-path.hh | 14 +- src/libutil/comparator.hh | 16 ++ src/nix-build/nix-build.cc | 9 +- src/nix-env/nix-env.cc | 4 +- src/nix/app.cc | 7 +- src/nix/build.cc | 10 +- src/nix/bundle.cc | 2 +- src/nix/develop.cc | 2 +- src/nix/flake.cc | 6 +- src/nix/log.cc | 20 +- tests/build.sh | 2 +- tests/dyn-drv/build-built-drv.sh | 21 ++ tests/dyn-drv/local.mk | 3 +- tests/test-libstoreconsumer/main.cc | 2 +- 48 files changed, 1136 insertions(+), 267 deletions(-) create mode 100644 tests/dyn-drv/build-built-drv.sh diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 6516a2663..be922f95a 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -16,3 +16,6 @@ [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) attribute is no longer guarded by an experimental flag and can be used freely. + +- The JSON output for derived paths with are store paths is now a string, not an object with a single `path` field. + This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 2fb17d06f..c1f03a8ef 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -322,7 +322,12 @@ connected: throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); } else { copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute); - auto res = sshStore->buildPathsWithResults({ DerivedPath::Built { *drvPath, OutputsSpec::All {} } }); + auto res = sshStore->buildPathsWithResults({ + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(*drvPath), + .outputs = OutputsSpec::All {}, + } + }); // One path to build should produce exactly one build result assert(res.size() == 1); optResult = std::move(res[0]); diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index db9c440e3..c6cc7fa9c 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -8,13 +8,39 @@ namespace nix { -nlohmann::json BuiltPath::Built::toJSON(ref store) const { - nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); - for (const auto& [output, path] : outputs) { - res["outputs"][output] = store->printStorePath(path); +#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ + bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ + { \ + const MY_TYPE* me = this; \ + auto fields1 = std::make_tuple(*me->drvPath, me->FIELD); \ + me = &other; \ + auto fields2 = std::make_tuple(*me->drvPath, me->FIELD); \ + return fields1 COMPARATOR fields2; \ } - return res; +#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) + +#define FIELD_TYPE std::pair +CMP(SingleBuiltPath, SingleBuiltPathBuilt, output) +#undef FIELD_TYPE + +#define FIELD_TYPE std::map +CMP(SingleBuiltPath, BuiltPathBuilt, outputs) +#undef FIELD_TYPE + +#undef CMP +#undef CMP_ONE + +StorePath SingleBuiltPath::outPath() const +{ + return std::visit( + overloaded{ + [](const SingleBuiltPath::Opaque & p) { return p.path; }, + [](const SingleBuiltPath::Built & b) { return b.output.second; }, + }, raw() + ); } StorePathSet BuiltPath::outPaths() const @@ -32,6 +58,40 @@ StorePathSet BuiltPath::outPaths() const ); } +nlohmann::json BuiltPath::Built::toJSON(const Store & store) const +{ + nlohmann::json res; + res["drvPath"] = drvPath->toJSON(store); + for (const auto & [outputName, outputPath] : outputs) { + res["outputs"][outputName] = store.printStorePath(outputPath); + } + return res; +} + +nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const +{ + nlohmann::json res; + res["drvPath"] = drvPath->toJSON(store); + auto & [outputName, outputPath] = output; + res["output"] = outputName; + res["outputPath"] = store.printStorePath(outputPath); + return res; +} + +nlohmann::json SingleBuiltPath::toJSON(const Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + +nlohmann::json BuiltPath::toJSON(const Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const { RealisedPath::Set res; @@ -40,7 +100,7 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, [&](const BuiltPath::Built & p) { auto drvHashes = - staticOutputHashes(store, store.readDerivation(p.drvPath)); + staticOutputHashes(store, store.readDerivation(p.drvPath->outPath())); for (auto& [outputName, outputPath] : p.outputs) { if (experimentalFeatureSettings.isEnabled( Xp::CaDerivations)) { @@ -48,7 +108,7 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const if (!drvOutput) throw Error( "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", - store.printStorePath(p.drvPath), outputName); + store.printStorePath(p.drvPath->outPath()), outputName); auto thisRealisation = store.queryRealisation( DrvOutput{*drvOutput, outputName}); assert(thisRealisation); // We’ve built it, so we must diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index 744e8090b..747bcc440 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -3,19 +3,60 @@ namespace nix { +struct SingleBuiltPath; + +struct SingleBuiltPathBuilt { + ref drvPath; + std::pair output; + + std::string to_string(const Store & store) const; + static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); + nlohmann::json toJSON(const Store & store) const; + + DECLARE_CMP(SingleBuiltPathBuilt); +}; + +using _SingleBuiltPathRaw = std::variant< + DerivedPathOpaque, + SingleBuiltPathBuilt +>; + +struct SingleBuiltPath : _SingleBuiltPathRaw { + using Raw = _SingleBuiltPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = SingleBuiltPathBuilt; + + inline const Raw & raw() const { + return static_cast(*this); + } + + StorePath outPath() const; + + static SingleBuiltPath parse(const Store & store, std::string_view); + nlohmann::json toJSON(const Store & store) const; +}; + +static inline ref staticDrv(StorePath drvPath) +{ + return make_ref(SingleBuiltPath::Opaque { drvPath }); +} + /** * A built derived path with hints in the form of optional concrete output paths. * * See 'BuiltPath' for more an explanation. */ struct BuiltPathBuilt { - StorePath drvPath; + ref drvPath; std::map outputs; - nlohmann::json toJSON(ref store) const; - static BuiltPathBuilt parse(const Store & store, std::string_view); + std::string to_string(const Store & store) const; + static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); + nlohmann::json toJSON(const Store & store) const; - GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs); + DECLARE_CMP(BuiltPathBuilt); }; using _BuiltPathRaw = std::variant< @@ -41,6 +82,7 @@ struct BuiltPath : _BuiltPathRaw { StorePathSet outPaths() const; RealisedPath::Set toRealisedPaths(Store & store) const; + nlohmann::json toJSON(const Store & store) const; }; typedef std::vector BuiltPaths; diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index b35ca2910..2f89eee02 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -92,7 +92,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() for (auto & [drvPath, outputs] : byDrvPath) res.push_back({ .path = DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = outputs, }, .info = make_ref(ExtraPathInfoValue::Value { diff --git a/src/libcmd/installable-derived-path.cc b/src/libcmd/installable-derived-path.cc index 6ecf54b7c..b45641e8a 100644 --- a/src/libcmd/installable-derived-path.cc +++ b/src/libcmd/installable-derived-path.cc @@ -18,14 +18,7 @@ DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths() std::optional InstallableDerivedPath::getStorePath() { - return std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - return bfd.drvPath; - }, - [&](const DerivedPath::Opaque & bo) { - return bo.path; - }, - }, derivedPath.raw()); + return derivedPath.getBaseStorePath(); } InstallableDerivedPath InstallableDerivedPath::parse( @@ -42,7 +35,7 @@ InstallableDerivedPath InstallableDerivedPath::parse( // Remove this prior to stabilizing the new CLI. if (storePath.isDerivation()) { auto oldDerivedPath = DerivedPath::Built { - .drvPath = storePath, + .drvPath = makeConstantStorePathRef(storePath), .outputs = OutputsSpec::All { }, }; warn( @@ -55,8 +48,10 @@ InstallableDerivedPath InstallableDerivedPath::parse( }, // If the user did use ^, we just do exactly what is written. [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath { + auto drv = make_ref(SingleDerivedPath::parse(*store, prefix)); + drvRequireExperiment(*drv); return DerivedPath::Built { - .drvPath = store->parseStorePath(prefix), + .drvPath = std::move(drv), .outputs = outputSpec, }; }, diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 4da9b131b..1b169c3bd 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -118,7 +118,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() return {{ .path = DerivedPath::Built { - .drvPath = std::move(drvPath), + .drvPath = makeConstantStorePathRef(std::move(drvPath)), .outputs = std::visit(overloaded { [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { std::set outputsToInstall; diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 1eff293cc..08ad35105 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -55,7 +55,8 @@ std::optional InstallableValue::trySinglePathToDerivedPaths else if (v.type() == nString) { return {{ - .path = state->coerceToDerivedPath(pos, v, errorCtx), + .path = DerivedPath::fromSingle( + state->coerceToSingleDerivedPath(pos, v, errorCtx)), .info = make_ref(), }}; } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 9d593a01f..4c7d134ec 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -515,6 +515,30 @@ ref SourceExprCommand::parseInstallable( return installables.front(); } +static SingleBuiltPath getBuiltPath(ref evalStore, ref store, const SingleDerivedPath & b) +{ + return std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) -> SingleBuiltPath { + return SingleBuiltPath::Opaque { bo.path }; + }, + [&](const SingleDerivedPath::Built & bfd) -> SingleBuiltPath { + auto drvPath = getBuiltPath(evalStore, store, *bfd.drvPath); + // Resolving this instead of `bfd` will yield the same result, but avoid duplicative work. + SingleDerivedPath::Built truncatedBfd { + .drvPath = makeConstantStorePathRef(drvPath.outPath()), + .output = bfd.output, + }; + auto outputPath = resolveDerivedPath(*store, truncatedBfd, &*evalStore); + return SingleBuiltPath::Built { + .drvPath = make_ref(std::move(drvPath)), + .output = { bfd.output, outputPath }, + }; + }, + }, + b.raw()); +} + std::vector Installable::build( ref evalStore, ref store, @@ -568,7 +592,10 @@ std::vector, BuiltPathWithResult>> Installable::build [&](const DerivedPath::Built & bfd) { auto outputs = resolveDerivedPath(*store, bfd, &*evalStore); res.push_back({aux.installable, { - .path = BuiltPath::Built { bfd.drvPath, outputs }, + .path = BuiltPath::Built { + .drvPath = make_ref(getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }, .info = aux.info}}); }, [&](const DerivedPath::Opaque & bo) { @@ -597,7 +624,10 @@ std::vector, BuiltPathWithResult>> Installable::build for (auto & [outputName, realisation] : buildResult.builtOutputs) outputs.emplace(outputName, realisation.outPath); res.push_back({aux.installable, { - .path = BuiltPath::Built { bfd.drvPath, outputs }, + .path = BuiltPath::Built { + .drvPath = make_ref(getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }, .info = aux.info, .result = buildResult}}); }, @@ -691,7 +721,7 @@ StorePathSet Installable::toDerivations( : throw Error("argument '%s' did not evaluate to a derivation", i->what())); }, [&](const DerivedPath::Built & bfd) { - drvPaths.insert(bfd.drvPath); + drvPaths.insert(resolveDerivedPath(*store, *bfd.drvPath)); }, }, b.path.raw()); diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index d15162e76..ea7d0e2ec 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -648,7 +648,7 @@ bool NixRepl::processLine(std::string line) if (command == ":b" || command == ":bl") { state->store->buildPaths({ DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All { }, }, }); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 9e734e654..6a60ba87b 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -599,7 +599,7 @@ string_t AttrCursor::getStringWithContext() return d.drvPath; }, [&](const NixStringContextElem::Built & b) -> const StorePath & { - return b.drvPath; + return b.drvPath->getBaseStorePath(); }, [&](const NixStringContextElem::Opaque & o) -> const StorePath & { return o.path; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index a8e6baea6..3e521b732 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1042,7 +1042,7 @@ void EvalState::mkOutputString( : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName, xpSettings).render(), NixStringContext { NixStringContextElem::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .output = outputName, } }); @@ -2299,7 +2299,7 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon } -std::pair EvalState::coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx) +std::pair EvalState::coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx) { NixStringContext context; auto s = forceString(v, context, pos, errorCtx); @@ -2310,21 +2310,16 @@ std::pair EvalState::coerceToDerivedPathUnchecked s, csize) .withTrace(pos, errorCtx).debugThrow(); auto derivedPath = std::visit(overloaded { - [&](NixStringContextElem::Opaque && o) -> DerivedPath { - return DerivedPath::Opaque { - .path = std::move(o.path), - }; + [&](NixStringContextElem::Opaque && o) -> SingleDerivedPath { + return std::move(o); }, - [&](NixStringContextElem::DrvDeep &&) -> DerivedPath { + [&](NixStringContextElem::DrvDeep &&) -> SingleDerivedPath { error( "string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time", s).withTrace(pos, errorCtx).debugThrow(); }, - [&](NixStringContextElem::Built && b) -> DerivedPath { - return DerivedPath::Built { - .drvPath = std::move(b.drvPath), - .outputs = OutputsSpec::Names { std::move(b.output) }, - }; + [&](NixStringContextElem::Built && b) -> SingleDerivedPath { + return std::move(b); }, }, ((NixStringContextElem &&) *context.begin()).raw()); return { @@ -2334,12 +2329,12 @@ std::pair EvalState::coerceToDerivedPathUnchecked } -DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx) +SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx) { - auto [derivedPath, s_] = coerceToDerivedPathUnchecked(pos, v, errorCtx); + auto [derivedPath, s_] = coerceToSingleDerivedPathUnchecked(pos, v, errorCtx); auto s = s_; std::visit(overloaded { - [&](const DerivedPath::Opaque & o) { + [&](const SingleDerivedPath::Opaque & o) { auto sExpected = store->printStorePath(o.path); if (s != sExpected) error( @@ -2347,25 +2342,27 @@ DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::str s, sExpected) .withTrace(pos, errorCtx).debugThrow(); }, - [&](const DerivedPath::Built & b) { - // TODO need derived path with single output to make this - // total. Will add as part of RFC 92 work and then this is - // cleaned up. - auto output = *std::get(b.outputs).begin(); - - auto drv = store->readDerivation(b.drvPath); - auto i = drv.outputs.find(output); - if (i == drv.outputs.end()) - throw Error("derivation '%s' does not have output '%s'", store->printStorePath(b.drvPath), output); - auto optOutputPath = i->second.path(*store, drv.name, output); - // This is testing for the case of CA derivations - auto sExpected = optOutputPath - ? store->printStorePath(*optOutputPath) - : DownstreamPlaceholder::unknownCaOutput(b.drvPath, output).render(); + [&](const SingleDerivedPath::Built & b) { + auto sExpected = std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + auto drv = store->readDerivation(o.path); + auto i = drv.outputs.find(b.output); + if (i == drv.outputs.end()) + throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output); + auto optOutputPath = i->second.path(*store, drv.name, b.output); + // This is testing for the case of CA derivations + return optOutputPath + ? store->printStorePath(*optOutputPath) + : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render(); + }, + [&](const SingleDerivedPath::Built & o) { + return DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render(); + }, + }, b.drvPath->raw()); if (s != sExpected) error( "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'", - s, output, store->printStorePath(b.drvPath), sExpected) + s, b.output, b.drvPath->to_string(*store), sExpected) .withTrace(pos, errorCtx).debugThrow(); } }, derivedPath.raw()); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 29d0f05a1..0268a2a12 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -22,7 +22,7 @@ namespace nix { class Store; class EvalState; class StorePath; -struct DerivedPath; +struct SingleDerivedPath; enum RepairFlag : bool; @@ -532,12 +532,12 @@ public: StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); /** - * Part of `coerceToDerivedPath()` without any store IO which is exposed for unit testing only. + * Part of `coerceToSingleDerivedPath()` without any store IO which is exposed for unit testing only. */ - std::pair coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx); + std::pair coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx); /** - * Coerce to `DerivedPath`. + * Coerce to `SingleDerivedPath`. * * Must be a string which is either a literal store path or a * "placeholder (see `DownstreamPlaceholder`). @@ -551,7 +551,7 @@ public: * source of truth, and ultimately tells us what we want, and then * we ensure the string corresponds to it. */ - DerivedPath coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx); + SingleDerivedPath coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx); public: diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 430607214..54943b481 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -56,7 +56,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context) .drvPath = b.drvPath, .outputs = OutputsSpec::Names { b.output }, }); - ensureValid(b.drvPath); + ensureValid(b.drvPath->getBaseStorePath()); }, [&](const NixStringContextElem::Opaque & o) { auto ctxS = store->printStorePath(o.path); @@ -77,7 +77,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context) if (!evalSettings.enableImportFromDerivation) debugThrowLastTrace(Error( "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", - store->printStorePath(drvs.begin()->drvPath))); + drvs.begin()->to_string(*store))); /* Build/substitute the context. */ std::vector buildReqs; @@ -95,7 +95,11 @@ StringMap EvalState::realiseContext(const NixStringContext & context) /* Get all the output paths corresponding to the placeholders we had */ if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { res.insert_or_assign( - DownstreamPlaceholder::unknownCaOutput(drv.drvPath, outputName).render(), + DownstreamPlaceholder::fromSingleDerivedPathBuilt( + SingleDerivedPath::Built { + .drvPath = drv.drvPath, + .output = outputName, + }).render(), store->printStorePath(outputPath) ); } @@ -1251,7 +1255,10 @@ drvName, Bindings * attrs, Value & v) } }, [&](const NixStringContextElem::Built & b) { - drv.inputDrvs[b.drvPath].insert(b.output); + if (auto * p = std::get_if(&*b.drvPath)) + drv.inputDrvs[p->path].insert(b.output); + else + throw UnimplementedError("Dependencies on the outputs of dynamic derivations are not yet supported"); }, [&](const NixStringContextElem::Opaque & o) { drv.inputSrcs.insert(o.path); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 8b3468009..bfc731744 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -106,7 +106,10 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, contextInfos[std::move(d.drvPath)].allOutputs = true; }, [&](NixStringContextElem::Built && b) { - contextInfos[std::move(b.drvPath)].outputs.emplace_back(std::move(b.output)); + // FIXME should eventually show string context as is, no + // resolving here. + auto drvPath = resolveDerivedPath(*state.store, *b.drvPath); + contextInfos[std::move(drvPath)].outputs.emplace_back(std::move(b.output)); }, [&](NixStringContextElem::Opaque && o) { contextInfos[std::move(o.path)].path = true; @@ -222,7 +225,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar for (auto elem : iter->value->listItems()) { auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context"); context.emplace(NixStringContextElem::Built { - .drvPath = namePath, + .drvPath = makeConstantStorePathRef(namePath), .output = std::string { outputName }, }); } diff --git a/src/libexpr/tests/derived-path.cc b/src/libexpr/tests/derived-path.cc index c713fe28a..2a5ca64f6 100644 --- a/src/libexpr/tests/derived-path.cc +++ b/src/libexpr/tests/derived-path.cc @@ -21,12 +21,12 @@ TEST_F(DerivedPathExpressionTest, force_init) RC_GTEST_FIXTURE_PROP( DerivedPathExpressionTest, prop_opaque_path_round_trip, - (const DerivedPath::Opaque & o)) + (const SingleDerivedPath::Opaque & o)) { auto * v = state.allocValue(); state.mkStorePathString(o.path, *v); - auto d = state.coerceToDerivedPath(noPos, *v, ""); - RC_ASSERT(DerivedPath { o } == d); + auto d = state.coerceToSingleDerivedPath(noPos, *v, ""); + RC_ASSERT(SingleDerivedPath { o } == d); } // TODO use DerivedPath::Built for parameter once it supports a single output @@ -46,12 +46,12 @@ RC_GTEST_FIXTURE_PROP( auto * v = state.allocValue(); state.mkOutputString(*v, drvPath, outputName.name, std::nullopt, mockXpSettings); - auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, ""); - DerivedPath::Built b { - .drvPath = drvPath, - .outputs = OutputsSpec::Names { outputName.name }, + auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, ""); + SingleDerivedPath::Built b { + .drvPath = makeConstantStorePathRef(drvPath), + .output = outputName.name, }; - RC_ASSERT(DerivedPath { b } == d); + RC_ASSERT(SingleDerivedPath { b } == d); } RC_GTEST_FIXTURE_PROP( @@ -61,12 +61,12 @@ RC_GTEST_FIXTURE_PROP( { auto * v = state.allocValue(); state.mkOutputString(*v, drvPath, outputName.name, outPath); - auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, ""); - DerivedPath::Built b { - .drvPath = drvPath, - .outputs = OutputsSpec::Names { outputName.name }, + auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, ""); + SingleDerivedPath::Built b { + .drvPath = makeConstantStorePathRef(drvPath), + .output = outputName.name, }; - RC_ASSERT(DerivedPath { b } == d); + RC_ASSERT(SingleDerivedPath { b } == d); } } /* namespace nix */ diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index 0d9381577..c56b50b59 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -8,6 +8,8 @@ namespace nix { +// Test a few cases of invalid string context elements. + TEST(NixStringContextElemTest, empty_invalid) { EXPECT_THROW( NixStringContextElem::parse(""), @@ -38,6 +40,10 @@ TEST(NixStringContextElemTest, slash_invalid) { BadStorePath); } +/** + * Round trip (string <-> data structure) test for + * `NixStringContextElem::Opaque`. + */ TEST(NixStringContextElemTest, opaque) { std::string_view opaque = "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; auto elem = NixStringContextElem::parse(opaque); @@ -47,6 +53,10 @@ TEST(NixStringContextElemTest, opaque) { ASSERT_EQ(elem.to_string(), opaque); } +/** + * Round trip (string <-> data structure) test for + * `NixStringContextElem::DrvDeep`. + */ TEST(NixStringContextElemTest, drvDeep) { std::string_view drvDeep = "=g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; auto elem = NixStringContextElem::parse(drvDeep); @@ -56,28 +66,62 @@ TEST(NixStringContextElemTest, drvDeep) { ASSERT_EQ(elem.to_string(), drvDeep); } -TEST(NixStringContextElemTest, built) { +/** + * Round trip (string <-> data structure) test for a simpler + * `NixStringContextElem::Built`. + */ +TEST(NixStringContextElemTest, built_opaque) { std::string_view built = "!foo!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; auto elem = NixStringContextElem::parse(built); auto * p = std::get_if(&elem); ASSERT_TRUE(p); ASSERT_EQ(p->output, "foo"); - ASSERT_EQ(p->drvPath, StorePath { built.substr(5) }); + ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = StorePath { built.substr(5) }, + })); ASSERT_EQ(elem.to_string(), built); } +/** + * Round trip (string <-> data structure) test for a more complex, + * inductive `NixStringContextElem::Built`. + */ +TEST(NixStringContextElemTest, built_built) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + + std::string_view built = "!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(built, mockXpSettings); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->output, "foo"); + auto * drvPath = std::get_if(&*p->drvPath); + ASSERT_TRUE(drvPath); + ASSERT_EQ(drvPath->output, "bar"); + ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = StorePath { built.substr(9) }, + })); + ASSERT_EQ(elem.to_string(), built); +} + +/** + * Without the right experimental features enabled, we cannot parse a + * complex inductive string context element. + */ +TEST(NixStringContextElemTest, built_built_xp) { + ASSERT_THROW( + NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"), MissingExperimentalFeature); +} + } namespace rc { using namespace nix; -Gen Arbitrary::arbitrary() -{ - return gen::just(NixStringContextElem::Opaque { - .path = *gen::arbitrary(), - }); -} - Gen Arbitrary::arbitrary() { return gen::just(NixStringContextElem::DrvDeep { @@ -85,14 +129,6 @@ Gen Arbitrary::arb }); } -Gen Arbitrary::arbitrary() -{ - return gen::just(NixStringContextElem::Built { - .drvPath = *gen::arbitrary(), - .output = (*gen::arbitrary()).name, - }); -} - Gen Arbitrary::arbitrary() { switch (*gen::inRange(0, std::variant_size_v)) { diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index f76fc76e4..d8116011e 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -4,29 +4,52 @@ namespace nix { -NixStringContextElem NixStringContextElem::parse(std::string_view s0) +NixStringContextElem NixStringContextElem::parse( + std::string_view s0, + const ExperimentalFeatureSettings & xpSettings) { std::string_view s = s0; + std::function parseRest; + parseRest = [&]() -> SingleDerivedPath { + // Case on whether there is a '!' + size_t index = s.find("!"); + if (index == std::string_view::npos) { + return SingleDerivedPath::Opaque { + .path = StorePath { s }, + }; + } else { + std::string output { s.substr(0, index) }; + // Advance string to parse after the '!' + s = s.substr(index + 1); + auto drv = make_ref(parseRest()); + drvRequireExperiment(*drv, xpSettings); + return SingleDerivedPath::Built { + .drvPath = std::move(drv), + .output = std::move(output), + }; + } + }; + if (s.size() == 0) { throw BadNixStringContextElem(s0, "String context element should never be an empty string"); } + switch (s.at(0)) { case '!': { - s = s.substr(1); // advance string to parse after first ! - size_t index = s.find("!"); - // This makes index + 1 safe. Index can be the length (one after index - // of last character), so given any valid character index --- a - // successful find --- we can add one. - if (index == std::string_view::npos) { + // Advance string to parse after the '!' + s = s.substr(1); + + // Find *second* '!' + if (s.find("!") == std::string_view::npos) { throw BadNixStringContextElem(s0, "String content element beginning with '!' should have a second '!'"); } - return NixStringContextElem::Built { - .drvPath = StorePath { s.substr(index + 1) }, - .output = std::string(s.substr(0, index)), - }; + + return std::visit( + [&](auto x) -> NixStringContextElem { return std::move(x); }, + parseRest()); } case '=': { return NixStringContextElem::DrvDeep { @@ -34,33 +57,51 @@ NixStringContextElem NixStringContextElem::parse(std::string_view s0) }; } default: { - return NixStringContextElem::Opaque { - .path = StorePath { s }, - }; + // Ensure no '!' + if (s.find("!") != std::string_view::npos) { + throw BadNixStringContextElem(s0, + "String content element not beginning with '!' should not have a second '!'"); + } + return std::visit( + [&](auto x) -> NixStringContextElem { return std::move(x); }, + parseRest()); } } } -std::string NixStringContextElem::to_string() const { - return std::visit(overloaded { +std::string NixStringContextElem::to_string() const +{ + std::string res; + + std::function toStringRest; + toStringRest = [&](auto & p) { + std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + res += o.path.to_string(); + }, + [&](const SingleDerivedPath::Built & o) { + res += o.output; + res += '!'; + toStringRest(*o.drvPath); + }, + }, p.raw()); + }; + + std::visit(overloaded { [&](const NixStringContextElem::Built & b) { - std::string res; res += '!'; - res += b.output; - res += '!'; - res += b.drvPath.to_string(); - return res; - }, - [&](const NixStringContextElem::DrvDeep & d) { - std::string res; - res += '='; - res += d.drvPath.to_string(); - return res; + toStringRest(b); }, [&](const NixStringContextElem::Opaque & o) { - return std::string { o.path.to_string() }; + toStringRest(o); + }, + [&](const NixStringContextElem::DrvDeep & d) { + res += '='; + res += d.drvPath.to_string(); }, }, raw()); + + return res; } } diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 287ae08a9..a1b71695b 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -3,7 +3,7 @@ #include "util.hh" #include "comparator.hh" -#include "path.hh" +#include "derived-path.hh" #include @@ -31,11 +31,7 @@ public: * * Encoded as just the path: ‘’. */ -struct NixStringContextElem_Opaque { - StorePath path; - - GENERATE_CMP(NixStringContextElem_Opaque, me->path); -}; +typedef SingleDerivedPath::Opaque NixStringContextElem_Opaque; /** * Path to a derivation and its entire build closure. @@ -57,12 +53,7 @@ struct NixStringContextElem_DrvDeep { * * Encoded in the form ‘!!’. */ -struct NixStringContextElem_Built { - StorePath drvPath; - std::string output; - - GENERATE_CMP(NixStringContextElem_Built, me->drvPath, me->output); -}; +typedef SingleDerivedPath::Built NixStringContextElem_Built; using _NixStringContextElem_Raw = std::variant< NixStringContextElem_Opaque, @@ -93,8 +84,12 @@ struct NixStringContextElem : _NixStringContextElem_Raw { * - ‘’ * - ‘=’ * - ‘!!’ + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static NixStringContextElem parse(std::string_view s); + static NixStringContextElem parse( + std::string_view s, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); std::string to_string() const; }; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5e37f7ecb..8bdf2f367 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -65,7 +65,7 @@ namespace nix { DerivationGoal::DerivationGoal(const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) + : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs }) , useDerivation(true) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -74,7 +74,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", - DerivedPath::Built { drvPath, wantedOutputs }.to_string(worker.store)); + DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -84,7 +84,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) + : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs }) , useDerivation(false) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -95,7 +95,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", - DerivedPath::Built { drvPath, drv.outputNames() }.to_string(worker.store)); + DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -1490,7 +1490,7 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) for (auto & outputName : outputs->second) { auto buildResult = dg->getBuildResult(DerivedPath::Built { - .drvPath = dg->drvPath, + .drvPath = makeConstantStorePathRef(dg->drvPath), .outputs = OutputsSpec::Names { outputName }, }); if (buildResult.success()) { diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 4aa4d6dca..e941b4e65 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -77,7 +77,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat try { worker.run(Goals{goal}); return goal->getBuildResult(DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All {}, }); } catch (Error & e) { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 8b7fbdcda..920097680 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1172,6 +1172,19 @@ void LocalDerivationGoal::writeStructuredAttrs() } +static StorePath pathPartOfReq(const SingleDerivedPath & req) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + return bo.path; + }, + [&](const SingleDerivedPath::Built & bfd) { + return pathPartOfReq(*bfd.drvPath); + }, + }, req.raw()); +} + + static StorePath pathPartOfReq(const DerivedPath & req) { return std::visit(overloaded { @@ -1179,7 +1192,7 @@ static StorePath pathPartOfReq(const DerivedPath & req) return bo.path; }, [&](const DerivedPath::Built & bfd) { - return bfd.drvPath; + return pathPartOfReq(*bfd.drvPath); }, }, req.raw()); } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index a9ca9cbbc..6779dbcf3 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -111,7 +111,10 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { [&](const DerivedPath::Built & bfd) -> GoalPtr { - return makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode); + if (auto bop = std::get_if(&*bfd.drvPath)) + return makeDerivationGoal(bop->path, bfd.outputs, buildMode); + else + throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); }, [&](const DerivedPath::Opaque & bo) -> GoalPtr { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); @@ -265,7 +268,7 @@ void Worker::run(const Goals & _topGoals) for (auto & i : _topGoals) { topGoals.insert(i); if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(DerivedPath::Built{goal->drvPath, goal->wantedOutputs}); + topPaths.push_back(DerivedPath::Built{makeConstantStorePathRef(goal->drvPath), goal->wantedOutputs}); } else if (auto goal = dynamic_cast(i.get())) { topPaths.push_back(DerivedPath::Opaque{goal->storePath}); } diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 52d073f81..3594b7570 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -7,52 +7,133 @@ namespace nix { -nlohmann::json DerivedPath::Opaque::toJSON(ref store) const { +#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ + bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ + { \ + const MY_TYPE* me = this; \ + auto fields1 = std::make_tuple(*me->drvPath, me->FIELD); \ + me = &other; \ + auto fields2 = std::make_tuple(*me->drvPath, me->FIELD); \ + return fields1 COMPARATOR fields2; \ + } +#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) + +#define FIELD_TYPE std::string +CMP(SingleDerivedPath, SingleDerivedPathBuilt, output) +#undef FIELD_TYPE + +#define FIELD_TYPE OutputsSpec +CMP(SingleDerivedPath, DerivedPathBuilt, outputs) +#undef FIELD_TYPE + +#undef CMP +#undef CMP_ONE + +nlohmann::json DerivedPath::Opaque::toJSON(const Store & store) const +{ + return store.printStorePath(path); +} + +nlohmann::json SingleDerivedPath::Built::toJSON(Store & store) const { nlohmann::json res; - res["path"] = store->printStorePath(path); + res["drvPath"] = drvPath->toJSON(store); + // Fallback for the input-addressed derivation case: We expect to always be + // able to print the output paths, so let’s do it + // FIXME try-resolve on drvPath + const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath)); + res["output"] = output; + auto outputPathIter = outputMap.find(output); + if (outputPathIter == outputMap.end()) + res["outputPath"] = nullptr; + else if (std::optional p = outputPathIter->second) + res["outputPath"] = store.printStorePath(*p); + else + res["outputPath"] = nullptr; return res; } -nlohmann::json DerivedPath::Built::toJSON(ref store) const { +nlohmann::json DerivedPath::Built::toJSON(Store & store) const { nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); + res["drvPath"] = drvPath->toJSON(store); // Fallback for the input-addressed derivation case: We expect to always be // able to print the output paths, so let’s do it - const auto outputMap = store->queryPartialDerivationOutputMap(drvPath); + // FIXME try-resolve on drvPath + const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath)); for (const auto & [output, outputPathOpt] : outputMap) { if (!outputs.contains(output)) continue; if (outputPathOpt) - res["outputs"][output] = store->printStorePath(*outputPathOpt); + res["outputs"][output] = store.printStorePath(*outputPathOpt); else res["outputs"][output] = nullptr; } return res; } +nlohmann::json SingleDerivedPath::toJSON(Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + +nlohmann::json DerivedPath::toJSON(Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + std::string DerivedPath::Opaque::to_string(const Store & store) const { return store.printStorePath(path); } +std::string SingleDerivedPath::Built::to_string(const Store & store) const +{ + return drvPath->to_string(store) + "^" + output; +} + +std::string SingleDerivedPath::Built::to_string_legacy(const Store & store) const +{ + return drvPath->to_string(store) + "!" + output; +} + std::string DerivedPath::Built::to_string(const Store & store) const { - return store.printStorePath(drvPath) + return drvPath->to_string(store) + '^' + outputs.to_string(); } std::string DerivedPath::Built::to_string_legacy(const Store & store) const { - return store.printStorePath(drvPath) - + '!' + return drvPath->to_string_legacy(store) + + "!" + outputs.to_string(); } +std::string SingleDerivedPath::to_string(const Store & store) const +{ + return std::visit( + [&](const auto & req) { return req.to_string(store); }, + raw()); +} + std::string DerivedPath::to_string(const Store & store) const +{ + return std::visit( + [&](const auto & req) { return req.to_string(store); }, + raw()); +} + +std::string SingleDerivedPath::to_string_legacy(const Store & store) const { return std::visit(overloaded { - [&](const DerivedPath::Built & req) { return req.to_string(store); }, - [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, + [&](const SingleDerivedPath::Built & req) { return req.to_string_legacy(store); }, + [&](const SingleDerivedPath::Opaque & req) { return req.to_string(store); }, }, this->raw()); } @@ -70,30 +151,156 @@ DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_ return {store.parseStorePath(s)}; } -DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view drvS, std::string_view outputsS) +void drvRequireExperiment( + const SingleDerivedPath & drv, + const ExperimentalFeatureSettings & xpSettings) { + std::visit(overloaded { + [&](const SingleDerivedPath::Opaque &) { + // plain drv path; no experimental features required. + }, + [&](const SingleDerivedPath::Built &) { + xpSettings.require(Xp::DynamicDerivations); + }, + }, drv.raw()); +} + +SingleDerivedPath::Built SingleDerivedPath::Built::parse( + const Store & store, ref drv, + std::string_view output, + const ExperimentalFeatureSettings & xpSettings) +{ + drvRequireExperiment(*drv, xpSettings); return { - .drvPath = store.parseStorePath(drvS), + .drvPath = drv, + .output = std::string { output }, + }; +} + +DerivedPath::Built DerivedPath::Built::parse( + const Store & store, ref drv, + std::string_view outputsS, + const ExperimentalFeatureSettings & xpSettings) +{ + drvRequireExperiment(*drv, xpSettings); + return { + .drvPath = drv, .outputs = OutputsSpec::parse(outputsS), }; } -static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator) +static SingleDerivedPath parseWithSingle( + const Store & store, std::string_view s, std::string_view separator, + const ExperimentalFeatureSettings & xpSettings) { - size_t n = s.find(separator); + size_t n = s.rfind(separator); + return n == s.npos + ? (SingleDerivedPath) SingleDerivedPath::Opaque::parse(store, s) + : (SingleDerivedPath) SingleDerivedPath::Built::parse(store, + make_ref(parseWithSingle( + store, + s.substr(0, n), + separator, + xpSettings)), + s.substr(n + 1), + xpSettings); +} + +SingleDerivedPath SingleDerivedPath::parse( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWithSingle(store, s, "^", xpSettings); +} + +SingleDerivedPath SingleDerivedPath::parseLegacy( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWithSingle(store, s, "!", xpSettings); +} + +static DerivedPath parseWith( + const Store & store, std::string_view s, std::string_view separator, + const ExperimentalFeatureSettings & xpSettings) +{ + size_t n = s.rfind(separator); return n == s.npos ? (DerivedPath) DerivedPath::Opaque::parse(store, s) - : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1)); + : (DerivedPath) DerivedPath::Built::parse(store, + make_ref(parseWithSingle( + store, + s.substr(0, n), + separator, + xpSettings)), + s.substr(n + 1), + xpSettings); } -DerivedPath DerivedPath::parse(const Store & store, std::string_view s) +DerivedPath DerivedPath::parse( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) { - return parseWith(store, s, "^"); + return parseWith(store, s, "^", xpSettings); } -DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s) +DerivedPath DerivedPath::parseLegacy( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) { - return parseWith(store, s, "!"); + return parseWith(store, s, "!", xpSettings); +} + +DerivedPath DerivedPath::fromSingle(const SingleDerivedPath & req) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) -> DerivedPath { + return o; + }, + [&](const SingleDerivedPath::Built & b) -> DerivedPath { + return DerivedPath::Built { + .drvPath = b.drvPath, + .outputs = OutputsSpec::Names { b.output }, + }; + }, + }, req.raw()); +} + +const StorePath & SingleDerivedPath::Built::getBaseStorePath() const +{ + return drvPath->getBaseStorePath(); +} + +const StorePath & DerivedPath::Built::getBaseStorePath() const +{ + return drvPath->getBaseStorePath(); +} + +template +static inline const StorePath & getBaseStorePath_(const DP & derivedPath) +{ + return std::visit(overloaded { + [&](const typename DP::Built & bfd) -> auto & { + return bfd.drvPath->getBaseStorePath(); + }, + [&](const typename DP::Opaque & bo) -> auto & { + return bo.path; + }, + }, derivedPath.raw()); +} + +const StorePath & SingleDerivedPath::getBaseStorePath() const +{ + return getBaseStorePath_(*this); +} + +const StorePath & DerivedPath::getBaseStorePath() const +{ + return getBaseStorePath_(*this); } } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 7a4261ce0..ec30dac61 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -24,28 +24,37 @@ class Store; struct DerivedPathOpaque { StorePath path; - nlohmann::json toJSON(ref store) const; std::string to_string(const Store & store) const; static DerivedPathOpaque parse(const Store & store, std::string_view); + nlohmann::json toJSON(const Store & store) const; GENERATE_CMP(DerivedPathOpaque, me->path); }; +struct SingleDerivedPath; + /** - * A derived path that is built from a derivation + * A single derived path that is built from a derivation * - * Built derived paths are pair of a derivation and some output names. - * They are evaluated by building the derivation, and then replacing the - * output names with the resulting outputs. - * - * Note that does mean a derived store paths evaluates to multiple - * opaque paths, which is sort of icky as expressions are supposed to - * evaluate to single values. Perhaps this should have just a single - * output name. + * Built derived paths are pair of a derivation and an output name. They are + * evaluated by building the derivation, and then taking the resulting output + * path of the given output name. */ -struct DerivedPathBuilt { - StorePath drvPath; - OutputsSpec outputs; +struct SingleDerivedPathBuilt { + ref drvPath; + std::string output; + + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; /** * Uses `^` as the separator @@ -57,11 +66,139 @@ struct DerivedPathBuilt { std::string to_string_legacy(const Store & store) const; /** * The caller splits on the separator, so it works for both variants. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPathBuilt parse(const Store & store, std::string_view drvPath, std::string_view outputs); - nlohmann::json toJSON(ref store) const; + static SingleDerivedPathBuilt parse( + const Store & store, ref drvPath, + std::string_view outputs, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; - GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs); + DECLARE_CMP(SingleDerivedPathBuilt); +}; + +using _SingleDerivedPathRaw = std::variant< + DerivedPathOpaque, + SingleDerivedPathBuilt +>; + +/** + * A "derived path" is a very simple sort of expression (not a Nix + * language expression! But an expression in a the general sense) that + * evaluates to (concrete) store path. It is either: + * + * - opaque, in which case it is just a concrete store path with + * possibly no known derivation + * + * - built, in which case it is a pair of a derivation path and an + * output name. + */ +struct SingleDerivedPath : _SingleDerivedPathRaw { + using Raw = _SingleDerivedPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = SingleDerivedPathBuilt; + + inline const Raw & raw() const { + return static_cast(*this); + } + + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + + /** + * Uses `^` as the separator + */ + std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * Uses `^` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static SingleDerivedPath parse( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + /** + * Uses `!` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static SingleDerivedPath parseLegacy( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; +}; + +static inline ref makeConstantStorePathRef(StorePath drvPath) +{ + return make_ref(SingleDerivedPath::Opaque { drvPath }); +} + +/** + * A set of derived paths that are built from a derivation + * + * Built derived paths are pair of a derivation and some output names. + * They are evaluated by building the derivation, and then replacing the + * output names with the resulting outputs. + * + * Note that does mean a derived store paths evaluates to multiple + * opaque paths, which is sort of icky as expressions are supposed to + * evaluate to single values. Perhaps this should have just a single + * output name. + */ +struct DerivedPathBuilt { + ref drvPath; + OutputsSpec outputs; + + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + + /** + * Uses `^` as the separator + */ + std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * The caller splits on the separator, so it works for both variants. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static DerivedPathBuilt parse( + const Store & store, ref, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; + + DECLARE_CMP(DerivedPathBuilt); }; using _DerivedPathRaw = std::variant< @@ -71,13 +208,13 @@ using _DerivedPathRaw = std::variant< /** * A "derived path" is a very simple sort of expression that evaluates - * to (concrete) store path. It is either: + * to one or more (concrete) store paths. It is either: * - * - opaque, in which case it is just a concrete store path with + * - opaque, in which case it is just a single concrete store path with * possibly no known derivation * - * - built, in which case it is a pair of a derivation path and an - * output name. + * - built, in which case it is a pair of a derivation path and some + * output names. */ struct DerivedPath : _DerivedPathRaw { using Raw = _DerivedPathRaw; @@ -90,6 +227,18 @@ struct DerivedPath : _DerivedPathRaw { return static_cast(*this); } + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + /** * Uses `^` as the separator */ @@ -100,14 +249,43 @@ struct DerivedPath : _DerivedPathRaw { std::string to_string_legacy(const Store & store) const; /** * Uses `^` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPath parse(const Store & store, std::string_view); + static DerivedPath parse( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** * Uses `!` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPath parseLegacy(const Store & store, std::string_view); + static DerivedPath parseLegacy( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + /** + * Convert a `SingleDerivedPath` to a `DerivedPath`. + */ + static DerivedPath fromSingle(const SingleDerivedPath &); + + nlohmann::json toJSON(Store & store) const; }; typedef std::vector DerivedPaths; +/** + * Used by various parser functions to require experimental features as + * needed. + * + * Somewhat unfortunate this cannot just be an implementation detail for + * this module. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ +void drvRequireExperiment( + const SingleDerivedPath & drv, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); } diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index d623c05e2..885b3604d 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -38,4 +38,19 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( }; } +DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt( + const SingleDerivedPath::Built & b) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + return DownstreamPlaceholder::unknownCaOutput(o.path, b.output); + }, + [&](const SingleDerivedPath::Built & b2) { + return DownstreamPlaceholder::unknownDerivation( + DownstreamPlaceholder::fromSingleDerivedPathBuilt(b2), + b.output); + }, + }, b.drvPath->raw()); +} + } diff --git a/src/libstore/downstream-placeholder.hh b/src/libstore/downstream-placeholder.hh index 97f77e6b8..9372dcd58 100644 --- a/src/libstore/downstream-placeholder.hh +++ b/src/libstore/downstream-placeholder.hh @@ -3,6 +3,7 @@ #include "hash.hh" #include "path.hh" +#include "derived-path.hh" namespace nix { @@ -73,6 +74,17 @@ public: const DownstreamPlaceholder & drvPlaceholder, std::string_view outputName, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + /** + * Convenience constructor that handles both cases (unknown + * content-addressed output and unknown derivation), delegating as + * needed to `unknownCaOutput` and `unknownDerivation`. + * + * Recursively builds up a placeholder from a + * `SingleDerivedPath::Built.drvPath` chain. + */ + static DownstreamPlaceholder fromSingleDerivedPathBuilt( + const SingleDerivedPath::Built & built); }; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index fa17d606d..78b05031a 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -358,6 +358,9 @@ public: [&](const StorePath & drvPath) { throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath)); }, + [&](std::monostate) { + throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://"); + }, }, sOrDrvPath); } conn->to << ss; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 14160dc8b..1ece7a4c0 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -132,7 +132,7 @@ void Store::queryMissing(const std::vector & targets, } for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, DerivedPath::Built { i.first, i.second })); + pool.enqueue(std::bind(doPath, DerivedPath::Built { makeConstantStorePathRef(i.first), i.second })); }; auto checkOutput = [&]( @@ -176,10 +176,18 @@ void Store::queryMissing(const std::vector & targets, std::visit(overloaded { [&](const DerivedPath::Built & bfd) { - if (!isValidPath(bfd.drvPath)) { + auto drvPathP = std::get_if(&*bfd.drvPath); + if (!drvPathP) { + // TODO make work in this case. + warn("Ignoring dynamic derivation %s while querying missing paths; not yet implemented", bfd.drvPath->to_string(*this)); + return; + } + auto & drvPath = drvPathP->path; + + if (!isValidPath(drvPath)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); - state->unknown.insert(bfd.drvPath); + state->unknown.insert(drvPath); return; } @@ -187,7 +195,7 @@ void Store::queryMissing(const std::vector & targets, /* true for regular derivations, and CA derivations for which we have a trust mapping for all wanted outputs. */ auto knownOutputPaths = true; - for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(bfd.drvPath)) { + for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(drvPath)) { if (!pathOpt) { knownOutputPaths = false; break; @@ -197,15 +205,15 @@ void Store::queryMissing(const std::vector & targets, } if (knownOutputPaths && invalid.empty()) return; - auto drv = make_ref(derivationFromPath(bfd.drvPath)); - ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv); + auto drv = make_ref(derivationFromPath(drvPath)); + ParsedDerivation parsedDrv(StorePath(drvPath), *drv); if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref>(DrvState(invalid.size())); for (auto & output : invalid) - pool.enqueue(std::bind(checkOutput, bfd.drvPath, drv, output, drvState)); + pool.enqueue(std::bind(checkOutput, drvPath, drv, output, drvState)); } else - mustBuildDrv(bfd.drvPath, *drv); + mustBuildDrv(drvPath, *drv); }, [&](const DerivedPath::Opaque & bo) { @@ -310,7 +318,9 @@ std::map drvOutputReferences( OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) { - auto outputsOpt_ = store.queryPartialDerivationOutputMap(bfd.drvPath, evalStore_); + auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); + + auto outputsOpt_ = store.queryPartialDerivationOutputMap(drvPath, evalStore_); auto outputsOpt = std::visit(overloaded { [&](const OutputsSpec::All &) { @@ -325,7 +335,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, if (!pOutputPathOpt) throw Error( "the derivation '%s' doesn't have an output named '%s'", - store.printStorePath(bfd.drvPath), output); + bfd.drvPath->to_string(store), output); outputsOpt.insert_or_assign(output, std::move(*pOutputPathOpt)); } return outputsOpt; @@ -335,11 +345,64 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, OutputPathMap outputs; for (auto & [outputName, outputPathOpt] : outputsOpt) { if (!outputPathOpt) - throw MissingRealisation(store.printStorePath(bfd.drvPath), outputName); + throw MissingRealisation(bfd.drvPath->to_string(store), outputName); auto & outputPath = *outputPathOpt; outputs.insert_or_assign(outputName, outputPath); } return outputs; } + +StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store * evalStore_) +{ + auto & evalStore = evalStore_ ? *evalStore_ : store; + + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + return bo.path; + }, + [&](const SingleDerivedPath::Built & bfd) { + auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); + auto outputPaths = evalStore.queryPartialDerivationOutputMap(drvPath, evalStore_); + if (outputPaths.count(bfd.output) == 0) + throw Error("derivation '%s' does not have an output named '%s'", + store.printStorePath(drvPath), bfd.output); + auto & optPath = outputPaths.at(bfd.output); + if (!optPath) + throw Error("'%s' does not yet map to a known concrete store path", + bfd.to_string(store)); + return *optPath; + }, + }, req.raw()); +} + + +OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd) +{ + auto drvPath = resolveDerivedPath(store, *bfd.drvPath); + auto outputMap = store.queryDerivationOutputMap(drvPath); + auto outputsLeft = std::visit(overloaded { + [&](const OutputsSpec::All &) { + return StringSet {}; + }, + [&](const OutputsSpec::Names & names) { + return static_cast(names); + }, + }, bfd.outputs.raw()); + for (auto iter = outputMap.begin(); iter != outputMap.end();) { + auto & outputName = iter->first; + if (bfd.outputs.contains(outputName)) { + outputsLeft.erase(outputName); + ++iter; + } else { + iter = outputMap.erase(iter); + } + } + if (!outputsLeft.empty()) + throw Error("derivation '%s' does not have an outputs %s", + store.printStorePath(drvPath), + concatStringsSep(", ", quoteStrings(std::get(bfd.outputs)))); + return outputMap; +} + } diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 869b490ad..81f360f3a 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -16,10 +16,16 @@ std::string StorePathWithOutputs::to_string(const Store & store) const DerivedPath StorePathWithOutputs::toDerivedPath() const { if (!outputs.empty()) { - return DerivedPath::Built { path, OutputsSpec::Names { outputs } }; + return DerivedPath::Built { + .drvPath = makeConstantStorePathRef(path), + .outputs = OutputsSpec::Names { outputs }, + }; } else if (path.isDerivation()) { assert(outputs.empty()); - return DerivedPath::Built { path, OutputsSpec::All { } }; + return DerivedPath::Built { + .drvPath = makeConstantStorePathRef(path), + .outputs = OutputsSpec::All { }, + }; } else { return DerivedPath::Opaque { path }; } @@ -34,29 +40,36 @@ std::vector toDerivedPaths(const std::vector } -std::variant StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) +StorePathWithOutputs::ParseResult StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) { return std::visit(overloaded { - [&](const DerivedPath::Opaque & bo) -> std::variant { + [&](const DerivedPath::Opaque & bo) -> StorePathWithOutputs::ParseResult { if (bo.path.isDerivation()) { // drv path gets interpreted as "build", not "get drv file itself" return bo.path; } return StorePathWithOutputs { bo.path }; }, - [&](const DerivedPath::Built & bfd) -> std::variant { - return StorePathWithOutputs { - .path = bfd.drvPath, - // Use legacy encoding of wildcard as empty set - .outputs = std::visit(overloaded { - [&](const OutputsSpec::All &) -> StringSet { - return {}; - }, - [&](const OutputsSpec::Names & outputs) { - return static_cast(outputs); - }, - }, bfd.outputs.raw()), - }; + [&](const DerivedPath::Built & bfd) -> StorePathWithOutputs::ParseResult { + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) -> StorePathWithOutputs::ParseResult { + return StorePathWithOutputs { + .path = bo.path, + // Use legacy encoding of wildcard as empty set + .outputs = std::visit(overloaded { + [&](const OutputsSpec::All &) -> StringSet { + return {}; + }, + [&](const OutputsSpec::Names & outputs) { + return static_cast(outputs); + }, + }, bfd.outputs.raw()), + }; + }, + [&](const SingleDerivedPath::Built &) -> StorePathWithOutputs::ParseResult { + return std::monostate {}; + }, + }, bfd.drvPath->raw()); }, }, p.raw()); } diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index d75850868..57e03252d 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -23,7 +23,9 @@ struct StorePathWithOutputs DerivedPath toDerivedPath() const; - static std::variant tryFromDerivedPath(const DerivedPath &); + typedef std::variant ParseResult; + + static StorePathWithOutputs::ParseResult tryFromDerivedPath(const DerivedPath &); }; std::vector toDerivedPaths(const std::vector); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 21258daec..58f72beb9 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -656,6 +656,9 @@ static void writeDerivedPaths(RemoteStore & store, RemoteStore::Connection & con GET_PROTOCOL_MAJOR(conn.daemonVersion), GET_PROTOCOL_MINOR(conn.daemonVersion)); }, + [&](std::monostate) { + throw Error("wanted to build a derivation that is itself a build product, but the legacy 'ssh://' protocol doesn't support that. Try using 'ssh-ng://'"); + }, }, sOrDrvPath); } conn.to << ss; @@ -670,9 +673,16 @@ void RemoteStore::copyDrvsFromEvalStore( /* The remote doesn't have a way to access evalStore, so copy the .drvs. */ RealisedPath::Set drvPaths2; - for (auto & i : paths) - if (auto p = std::get_if(&i)) - drvPaths2.insert(p->drvPath); + for (const auto & i : paths) { + std::visit(overloaded { + [&](const DerivedPath::Opaque & bp) { + // Do nothing, path is hopefully there already + }, + [&](const DerivedPath::Built & bp) { + drvPaths2.insert(bp.drvPath->getBaseStorePath()); + }, + }, i.raw()); + } copyClosure(*evalStore, *this, drvPaths2); } } @@ -742,7 +752,8 @@ std::vector RemoteStore::buildPathsWithResults( }; OutputPathMap outputs; - auto drv = evalStore->readDerivation(bfd.drvPath); + auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath); + auto drv = evalStore->readDerivation(drvPath); const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive auto built = resolveDerivedPath(*this, bfd, &*evalStore); for (auto & [output, outputPath] : built) { @@ -750,7 +761,7 @@ std::vector RemoteStore::buildPathsWithResults( if (!outputHash) throw Error( "the derivation '%s' doesn't have an output named '%s'", - printStorePath(bfd.drvPath), output); + printStorePath(drvPath), output); auto outputId = DrvOutput{ *outputHash, output }; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto realisation = diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f9029ade1..da1a3eefb 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -945,6 +945,7 @@ void removeTempRoots(); * Resolve the derived path completely, failing if any derivation output * is unknown. */ +StorePath resolveDerivedPath(Store &, const SingleDerivedPath &, Store * evalStore = nullptr); OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr); diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc index 160443ec1..d6549f66f 100644 --- a/src/libstore/tests/derived-path.cc +++ b/src/libstore/tests/derived-path.cc @@ -17,14 +17,34 @@ Gen Arbitrary::arbitrary() }); } +Gen Arbitrary::arbitrary() +{ + return gen::just(SingleDerivedPath::Built { + .drvPath = make_ref(*gen::arbitrary()), + .output = (*gen::arbitrary()).name, + }); +} + Gen Arbitrary::arbitrary() { return gen::just(DerivedPath::Built { - .drvPath = *gen::arbitrary(), + .drvPath = make_ref(*gen::arbitrary()), .outputs = *gen::arbitrary(), }); } +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + Gen Arbitrary::arbitrary() { switch (*gen::inRange(0, std::variant_size_v)) { @@ -45,12 +65,69 @@ class DerivedPathTest : public LibStoreTest { }; -// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is -// no a real fixture. -// -// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args -TEST_F(DerivedPathTest, force_init) -{ +/** + * Round trip (string <-> data structure) test for + * `DerivedPath::Opaque`. + */ +TEST_F(DerivedPathTest, opaque) { + std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; + auto elem = DerivedPath::parse(*store, opaque); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->path, store->parseStorePath(opaque)); + ASSERT_EQ(elem.to_string(*store), opaque); +} + +/** + * Round trip (string <-> data structure) test for a simpler + * `DerivedPath::Built`. + */ +TEST_F(DerivedPathTest, built_opaque) { + std::string_view built = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^bar,foo"; + auto elem = DerivedPath::parse(*store, built); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->outputs, ((OutputsSpec) OutputsSpec::Names { "foo", "bar" })); + ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = store->parseStorePath(built.substr(0, 49)), + })); + ASSERT_EQ(elem.to_string(*store), built); +} + +/** + * Round trip (string <-> data structure) test for a more complex, + * inductive `DerivedPath::Built`. + */ +TEST_F(DerivedPathTest, built_built) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + + std::string_view built = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^foo^bar,baz"; + auto elem = DerivedPath::parse(*store, built, mockXpSettings); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->outputs, ((OutputsSpec) OutputsSpec::Names { "bar", "baz" })); + auto * drvPath = std::get_if(&*p->drvPath); + ASSERT_TRUE(drvPath); + ASSERT_EQ(drvPath->output, "foo"); + ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = store->parseStorePath(built.substr(0, 49)), + })); + ASSERT_EQ(elem.to_string(*store), built); +} + +/** + * Without the right experimental features enabled, we cannot parse a + * complex inductive derived path. + */ +TEST_F(DerivedPathTest, built_built_xp) { + ASSERT_THROW( + DerivedPath::parse(*store, "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^foo^bar,baz"), + MissingExperimentalFeature); } RC_GTEST_FIXTURE_PROP( diff --git a/src/libstore/tests/derived-path.hh b/src/libstore/tests/derived-path.hh index 506f3ccb1..98d61f228 100644 --- a/src/libstore/tests/derived-path.hh +++ b/src/libstore/tests/derived-path.hh @@ -12,8 +12,18 @@ namespace rc { using namespace nix; template<> -struct Arbitrary { - static Gen arbitrary(); +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); }; template<> diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index 9f661c5c3..7982fdc5e 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -1,6 +1,22 @@ #pragma once ///@file +/** + * Declare comparison methods without defining them. + */ +#define DECLARE_ONE_CMP(COMPARATOR, MY_TYPE) \ + bool operator COMPARATOR(const MY_TYPE & other) const; +#define DECLARE_EQUAL(my_type) \ + DECLARE_ONE_CMP(==, my_type) +#define DECLARE_LEQ(my_type) \ + DECLARE_ONE_CMP(<, my_type) +#define DECLARE_NEQ(my_type) \ + DECLARE_ONE_CMP(!=, my_type) +#define DECLARE_CMP(my_type) \ + DECLARE_EQUAL(my_type) \ + DECLARE_LEQ(my_type) \ + DECLARE_NEQ(my_type) + /** * Awful hacky generation of the comparison operators by doing a lexicographic * comparison between the choosen fields. diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 6510df8f0..66f319c3e 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -393,7 +393,7 @@ static void main_nix_build(int argc, char * * argv) auto bashDrv = drv->requireDrvPath(); pathsToBuild.push_back(DerivedPath::Built { - .drvPath = bashDrv, + .drvPath = makeConstantStorePathRef(bashDrv), .outputs = OutputsSpec::Names {"out"}, }); pathsToCopy.insert(bashDrv); @@ -417,7 +417,7 @@ static void main_nix_build(int argc, char * * argv) })) { pathsToBuild.push_back(DerivedPath::Built { - .drvPath = inputDrv, + .drvPath = makeConstantStorePathRef(inputDrv), .outputs = OutputsSpec::Names { inputOutputs }, }); pathsToCopy.insert(inputDrv); @@ -590,7 +590,10 @@ static void main_nix_build(int argc, char * * argv) if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath)); - pathsToBuild.push_back(DerivedPath::Built{drvPath, OutputsSpec::Names{outputName}}); + pathsToBuild.push_back(DerivedPath::Built{ + .drvPath = makeConstantStorePathRef(drvPath), + .outputs = OutputsSpec::Names{outputName}, + }); pathsToBuildOrdered.push_back({drvPath, {outputName}}); drvsToCopy.insert(drvPath); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 91b073b49..8dd071aa6 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -481,7 +481,7 @@ static void printMissing(EvalState & state, DrvInfos & elems) for (auto & i : elems) if (auto drvPath = i.queryDrvPath()) targets.push_back(DerivedPath::Built{ - .drvPath = *drvPath, + .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, }); else @@ -759,7 +759,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) std::vector paths { drvPath ? (DerivedPath) (DerivedPath::Built { - .drvPath = *drvPath, + .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, }) : (DerivedPath) (DerivedPath::Opaque { diff --git a/src/nix/app.cc b/src/nix/app.cc index e0f68b4fc..16a921194 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -25,7 +25,8 @@ StringPairs resolveRewrites( if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) for (auto & [ outputName, outputPath ] : drvDep->outputs) res.emplace( - DownstreamPlaceholder::unknownCaOutput(drvDep->drvPath, outputName).render(), + DownstreamPlaceholder::unknownCaOutput( + drvDep->drvPath->outPath(), outputName).render(), store.printStorePath(outputPath) ); return res; @@ -65,7 +66,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) [&](const NixStringContextElem::DrvDeep & d) -> DerivedPath { /* We want all outputs of the drv */ return DerivedPath::Built { - .drvPath = d.drvPath, + .drvPath = makeConstantStorePathRef(d.drvPath), .outputs = OutputsSpec::All {}, }; }, @@ -106,7 +107,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) auto program = outPath + "/bin/" + mainProgram; return UnresolvedApp { App { .context = { DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::Names { outputName }, } }, .program = program, diff --git a/src/nix/build.cc b/src/nix/build.cc index ad1842a4e..479100186 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -9,18 +9,18 @@ using namespace nix; -nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref store) +static nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, Store & store) { auto res = nlohmann::json::array(); for (auto & t : paths) { - std::visit([&res, store](const auto & t) { + std::visit([&](const auto & t) { res.push_back(t.toJSON(store)); }, t.raw()); } return res; } -nlohmann::json builtPathsWithResultToJSON(const std::vector & buildables, ref store) +static nlohmann::json builtPathsWithResultToJSON(const std::vector & buildables, const Store & store) { auto res = nlohmann::json::array(); for (auto & b : buildables) { @@ -125,7 +125,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile printMissing(store, pathsToBuild, lvlError); if (json) - logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump()); + logger->cout("%s", derivedPathsToJSON(pathsToBuild, *store).dump()); return; } @@ -136,7 +136,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile installables, repair ? bmRepair : buildMode); - if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, store).dump()); + if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, *store).dump()); if (outLink != "") if (auto store2 = store.dynamic_pointer_cast()) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index bcc00d490..5a80f0308 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -109,7 +109,7 @@ struct CmdBundle : InstallableValueCommand store->buildPaths({ DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All { }, }, }); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 195eeaa21..c033804e4 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -235,7 +235,7 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore /* Build the derivation. */ store->buildPaths( { DerivedPath::Built { - .drvPath = shellDrvPath, + .drvPath = makeConstantStorePathRef(shellDrvPath), .outputs = OutputsSpec::All { }, }}, bmNormal, evalStore); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3ce1de44a..83b74c8ca 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -544,9 +544,9 @@ struct CmdFlakeCheck : FlakeCommand *attr2.value, attr2.pos); if (drvPath && attr_name == settings.thisSystem.get()) { drvPaths.push_back(DerivedPath::Built { - .drvPath = *drvPath, - .outputs = OutputsSpec::All { }, - }); + .drvPath = makeConstantStorePathRef(*drvPath), + .outputs = OutputsSpec::All { }, + }); } } } diff --git a/src/nix/log.cc b/src/nix/log.cc index aaf829764..9a9bd30f9 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -33,6 +33,17 @@ struct CmdLog : InstallableCommand auto b = installable->toDerivedPath(); + // For compat with CLI today, TODO revisit + auto oneUp = std::visit(overloaded { + [&](const DerivedPath::Opaque & bo) { + return make_ref(bo); + }, + [&](const DerivedPath::Built & bfd) { + return bfd.drvPath; + }, + }, b.path.raw()); + auto path = resolveDerivedPath(*store, *oneUp); + RunPager pager; for (auto & sub : subs) { auto * logSubP = dynamic_cast(&*sub); @@ -42,14 +53,7 @@ struct CmdLog : InstallableCommand } auto & logSub = *logSubP; - auto log = std::visit(overloaded { - [&](const DerivedPath::Opaque & bo) { - return logSub.getBuildLog(bo.path); - }, - [&](const DerivedPath::Built & bfd) { - return logSub.getBuildLog(bfd.drvPath); - }, - }, b.path.raw()); + auto log = logSub.getBuildLog(path); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri()); diff --git a/tests/build.sh b/tests/build.sh index 8ae20f0df..7fbdb0f07 100644 --- a/tests/build.sh +++ b/tests/build.sh @@ -78,7 +78,7 @@ expectStderr 1 nix build --impure --expr 'with (import ./multiple-outputs.nix).e | grepQuiet "has 2 entries in its context. It should only have exactly one entry" nix build --impure --json --expr 'builtins.unsafeDiscardOutputDependency (import ./multiple-outputs.nix).e.a_a.drvPath' --no-link | jq --exit-status ' - (.[0] | .path | match(".*multiple-outputs-e.drv")) + (.[0] | match(".*multiple-outputs-e.drv")) ' # Test building from raw store path to drv not expression. diff --git a/tests/dyn-drv/build-built-drv.sh b/tests/dyn-drv/build-built-drv.sh new file mode 100644 index 000000000..647be9457 --- /dev/null +++ b/tests/dyn-drv/build-built-drv.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +source common.sh + +# In the corresponding nix file, we have two derivations: the first, named `hello`, +# is a normal recursive derivation, while the second, named dependent, has the +# new outputHashMode "text". Note that in "dependent", we don't refer to the +# build output of `hello`, but only to the path of the drv file. For this reason, +# we only need to: +# +# - instantiate `hello` +# - build `producingDrv` +# - check that the path of the output coincides with that of the original derivation + +out1=$(nix build -f ./text-hashed-output.nix hello --no-link) + +clearStore + +drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv) + +expectStderr 1 nix build "${drvDep}^out^out" --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" diff --git a/tests/dyn-drv/local.mk b/tests/dyn-drv/local.mk index f065a5627..b087ecd1c 100644 --- a/tests/dyn-drv/local.mk +++ b/tests/dyn-drv/local.mk @@ -1,6 +1,7 @@ dyn-drv-tests := \ $(d)/text-hashed-output.sh \ - $(d)/recursive-mod-json.sh + $(d)/recursive-mod-json.sh \ + $(d)/build-built-drv.sh install-tests-groups += dyn-drv diff --git a/tests/test-libstoreconsumer/main.cc b/tests/test-libstoreconsumer/main.cc index 31b6d8ef1..c61489af6 100644 --- a/tests/test-libstoreconsumer/main.cc +++ b/tests/test-libstoreconsumer/main.cc @@ -23,7 +23,7 @@ int main (int argc, char **argv) std::vector paths { DerivedPath::Built { - .drvPath = store->parseStorePath(drvPath), + .drvPath = makeConstantStorePathRef(store->parseStorePath(drvPath)), .outputs = OutputsSpec::Names{"out"} } }; From b9b51f9579c15c9789814cd4b6969b0f85e13980 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Fri, 11 Aug 2023 11:58:33 +0200 Subject: [PATCH 169/909] Prevent overriding virtual methods that are called in a destructor Virtual methods are no longer valid once the derived destructor has run. This means the compiler is free to optimize them to be non-virtual. Found using clang-tidy --- src/libmain/progress-bar.cc | 3 ++- src/libstore/build/local-derivation-goal.hh | 4 +++- src/libstore/build/substitution-goal.hh | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 6600ec177..45b1fdfd1 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -108,7 +108,8 @@ public: stop(); } - void stop() override + /* Called by destructor, can't be overridden */ + void stop() override final { { auto state(state_.lock()); diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 9acd7593d..8827bfca3 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -272,8 +272,10 @@ struct LocalDerivationGoal : public DerivationGoal /** * Forcibly kill the child process, if any. + * + * Called by destructor, can't be overridden */ - void killChild() override; + void killChild() override final; /** * Kill any processes running under the build user UID or in the diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 9fc041920..1b693baa1 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -114,7 +114,8 @@ public: void handleChildOutput(int fd, std::string_view data) override; void handleEOF(int fd) override; - void cleanup() override; + /* Called by destructor, can't be overridden */ + void cleanup() override final; JobCategory jobCategory() override { return JobCategory::Substitution; }; }; From 1ffb26311b5d74e9f607ec31bee6c17bbae6fdae Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Fri, 11 Aug 2023 12:00:11 +0200 Subject: [PATCH 170/909] MultiCommand::toJSON: Fix use-after-move --- src/libutil/args.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 3cf3ed9ca..00281ce6f 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -410,8 +410,8 @@ nlohmann::json MultiCommand::toJSON() auto cat = nlohmann::json::object(); cat["id"] = command->category(); cat["description"] = trim(categories[command->category()]); - j["category"] = std::move(cat); cat["experimental-feature"] = command->experimentalFeature(); + j["category"] = std::move(cat); cmds[name] = std::move(j); } From 2e5096e4f0959cb2974a17fe50ad5ad891b2384f Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Fri, 11 Aug 2023 12:00:31 +0200 Subject: [PATCH 171/909] FileTransfer::download: fix use-after-move std::move(state->data) and data.empty() were called in a loop, and could run with no other threads intervening. Accessing moved objects is undefined behavior, and could cause a crash. --- src/libstore/filetransfer.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 38b691279..a283af5a2 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -863,6 +863,8 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) } chunk = std::move(state->data); + /* Reset state->data after the move, since we check data.empty() */ + state->data = ""; state->request.notify_one(); } From e78e9a6bd1c5f26684a9da0d94ede7d960788e0e Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Fri, 11 Aug 2023 12:05:45 +0200 Subject: [PATCH 172/909] SimpleLogger::log: fix unintended fallthrough --- src/libutil/logging.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 5a2dd99af..54702e4ea 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -67,7 +67,7 @@ public: case lvlWarn: c = '4'; break; case lvlNotice: case lvlInfo: c = '5'; break; case lvlTalkative: case lvlChatty: c = '6'; break; - case lvlDebug: case lvlVomit: c = '7'; + case lvlDebug: case lvlVomit: c = '7'; break; default: c = '7'; break; // should not happen, and missing enum case is reported by -Werror=switch-enum } prefix = std::string("<") + c + ">"; From c4dbb55ba93c9c81d798faa6a19285ebe2717a25 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 11 Aug 2023 17:22:55 +0200 Subject: [PATCH 173/909] initLibUtil: Add exception handling self-check --- src/libutil/error.cc | 5 +++++ src/libutil/error.hh | 4 ++++ src/libutil/util.cc | 17 +++++++++++++++++ 3 files changed, 26 insertions(+) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index c9d61942a..4a1b346ef 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -14,6 +14,11 @@ void BaseError::addTrace(std::shared_ptr && e, hintformat hint, boo err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame }); } +void throwExceptionSelfCheck(){ + // This is meant to be caught in initLibUtil() + throw SysError("C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded."); +} + // c++ std::exception descendants must have a 'const char* what()' function. // This stringifies the error and caches it for use by what(), or similarly by msg(). const std::string & BaseError::calcWhat() const diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 6a0923081..be5c5e252 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -214,4 +214,8 @@ public: } }; +/** Throw an exception for the purpose of checking that exception handling works; see 'initLibUtil()'. + */ +void throwExceptionSelfCheck(); + } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 26f9dc8a8..f24a6e165 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -48,6 +48,23 @@ extern char * * environ __attribute__((weak)); namespace nix { void initLibUtil() { + // Check that exception handling works. Exception handling has been observed + // not to work on darwin when the linker flags aren't quite right. + // In this case we don't want to expose the user to some unrelated uncaught + // exception, but rather tell them exactly that exception handling is + // broken. + // When exception handling fails, the message tends to be printed by the + // C++ runtime, followed by an abort. + // For example on macOS we might see an error such as + // libc++abi: terminating with uncaught exception of type nix::SysError: error: C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded. + bool caught = false; + try { + throwExceptionSelfCheck(); + } catch (nix::Error _e) { + caught = true; + } + // This is not actually the main point of this check, but let's make sure anyway: + assert(caught); } std::optional getEnv(const std::string & key) From a04720e68ce45943ce0eae3d477f9d554b0d63a4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 14 Aug 2023 08:10:39 -0400 Subject: [PATCH 174/909] Rename `optOutputPath` to `optStaticOutputPath` This choice of variable name makes it more clear what is going on. Co-authored-by: Robert Hensing --- src/libexpr/eval.cc | 12 ++++++------ src/libexpr/eval.hh | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 3e521b732..063e34c56 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1031,12 +1031,12 @@ void EvalState::mkOutputString( Value & value, const StorePath & drvPath, const std::string outputName, - std::optional optOutputPath, + std::optional optStaticOutputPath, const ExperimentalFeatureSettings & xpSettings) { value.mkString( - optOutputPath - ? store->printStorePath(*std::move(optOutputPath)) + optStaticOutputPath + ? store->printStorePath(*std::move(optStaticOutputPath)) /* Downstream we would substitute this for an actual path once we build the floating CA derivation */ : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName, xpSettings).render(), @@ -2349,10 +2349,10 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & auto i = drv.outputs.find(b.output); if (i == drv.outputs.end()) throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output); - auto optOutputPath = i->second.path(*store, drv.name, b.output); + auto optStaticOutputPath = i->second.path(*store, drv.name, b.output); // This is testing for the case of CA derivations - return optOutputPath - ? store->printStorePath(*optOutputPath) + return optStaticOutputPath + ? store->printStorePath(*optStaticOutputPath) : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render(); }, [&](const SingleDerivedPath::Built & o) { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 0268a2a12..662c123b9 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -685,7 +685,7 @@ public: * * @param outputName Name of the output * - * @param optOutputPath Optional output path for that string. Must + * @param optStaticOutputPath Optional output path for that string. Must * be passed if and only if output store object is input-addressed. * Will be printed to form string if passed, otherwise a placeholder * will be used (see `DownstreamPlaceholder`). @@ -696,7 +696,7 @@ public: Value & value, const StorePath & drvPath, const std::string outputName, - std::optional optOutputPath, + std::optional optStaticOutputPath, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); From e7c39ff00b81dfdc48ebe7f41847db84ba779676 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 10 Mar 2021 04:22:56 +0000 Subject: [PATCH 175/909] Rework evaluator `SingleDerivedPath` infra `EvalState::mkSingleDerivedPathString` previously contained its own inverse (printing, rather than parsing) in order to validate what was parsed. Now that is pulled out into its own separate function: `EvalState::coerceToSingleDerivedPath`. In additional that pulled out logic is deduplicated with `EvalState::mkOutputString` via `EvalState::mkOutputStringRaw`, which is itself deduplicated (and generalized) with `DownstreamPlaceholder::mkOutputStringRaw`. All these changes make the unit tests simpler. (We would ideally write more unit tests for `mkSingleDerivedPathString` `coerceToSingleDerivedPath` directly, but we cannot yet do that because the IO in reading the store path won't work when the dummy store cannot hold anything. Someday we'll have a proper in-memory store which will work for this.) Co-authored-by: Robert Hensing --- src/libexpr/eval.cc | 101 ++++++++++++++++--------- src/libexpr/eval.hh | 53 +++++++++---- src/libexpr/primops.cc | 6 +- src/libexpr/tests/derived-path.cc | 20 ++--- src/libstore/downstream-placeholder.cc | 10 ++- src/libstore/downstream-placeholder.hh | 3 +- 6 files changed, 122 insertions(+), 71 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 063e34c56..79e2c4727 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1027,24 +1027,67 @@ void EvalState::mkStorePathString(const StorePath & p, Value & v) } +std::string EvalState::mkOutputStringRaw( + const SingleDerivedPath::Built & b, + std::optional optStaticOutputPath, + const ExperimentalFeatureSettings & xpSettings) +{ + /* In practice, this is testing for the case of CA derivations, or + dynamic derivations. */ + return optStaticOutputPath + ? store->printStorePath(*std::move(optStaticOutputPath)) + /* Downstream we would substitute this for an actual path once + we build the floating CA derivation */ + : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b, xpSettings).render(); +} + + void EvalState::mkOutputString( Value & value, - const StorePath & drvPath, - const std::string outputName, + const SingleDerivedPath::Built & b, std::optional optStaticOutputPath, const ExperimentalFeatureSettings & xpSettings) { value.mkString( - optStaticOutputPath - ? store->printStorePath(*std::move(optStaticOutputPath)) - /* Downstream we would substitute this for an actual path once - we build the floating CA derivation */ - : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName, xpSettings).render(), + mkOutputStringRaw(b, optStaticOutputPath, xpSettings), + NixStringContext { b }); +} + + +std::string EvalState::mkSingleDerivedPathStringRaw( + const SingleDerivedPath & p) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + return store->printStorePath(o.path); + }, + [&](const SingleDerivedPath::Built & b) { + auto optStaticOutputPath = std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + auto drv = store->readDerivation(o.path); + auto i = drv.outputs.find(b.output); + if (i == drv.outputs.end()) + throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output); + return i->second.path(*store, drv.name, b.output); + }, + [&](const SingleDerivedPath::Built & o) -> std::optional { + return std::nullopt; + }, + }, b.drvPath->raw()); + return mkOutputStringRaw(b, optStaticOutputPath); + } + }, p.raw()); +} + + +void EvalState::mkSingleDerivedPathString( + const SingleDerivedPath & p, + Value & v) +{ + v.mkString( + mkSingleDerivedPathStringRaw(p), NixStringContext { - NixStringContextElem::Built { - .drvPath = makeConstantStorePathRef(drvPath), - .output = outputName, - } + std::visit([](auto && v) -> NixStringContextElem { return v; }, p), }); } @@ -2333,39 +2376,25 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & { auto [derivedPath, s_] = coerceToSingleDerivedPathUnchecked(pos, v, errorCtx); auto s = s_; - std::visit(overloaded { - [&](const SingleDerivedPath::Opaque & o) { - auto sExpected = store->printStorePath(o.path); - if (s != sExpected) + auto sExpected = mkSingleDerivedPathStringRaw(derivedPath); + if (s != sExpected) { + /* `std::visit` is used here just to provide a more precise + error message. */ + std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { error( "path string '%s' has context with the different path '%s'", s, sExpected) .withTrace(pos, errorCtx).debugThrow(); - }, - [&](const SingleDerivedPath::Built & b) { - auto sExpected = std::visit(overloaded { - [&](const SingleDerivedPath::Opaque & o) { - auto drv = store->readDerivation(o.path); - auto i = drv.outputs.find(b.output); - if (i == drv.outputs.end()) - throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output); - auto optStaticOutputPath = i->second.path(*store, drv.name, b.output); - // This is testing for the case of CA derivations - return optStaticOutputPath - ? store->printStorePath(*optStaticOutputPath) - : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render(); - }, - [&](const SingleDerivedPath::Built & o) { - return DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render(); - }, - }, b.drvPath->raw()); - if (s != sExpected) + }, + [&](const SingleDerivedPath::Built & b) { error( "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'", s, b.output, b.drvPath->to_string(*store), sExpected) .withTrace(pos, errorCtx).debugThrow(); - } - }, derivedPath.raw()); + } + }, derivedPath.raw()); + } return derivedPath; } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 662c123b9..0c3eb6505 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -668,37 +668,46 @@ public: /** * Create a string representing a store path. * - * The string is the printed store path with a context containing a single - * `NixStringContextElem::Opaque` element of that store path. + * The string is the printed store path with a context containing a + * single `NixStringContextElem::Opaque` element of that store path. */ void mkStorePathString(const StorePath & storePath, Value & v); /** - * Create a string representing a `DerivedPath::Built`. + * Create a string representing a `SingleDerivedPath::Built`. * - * The string is the printed store path with a context containing a single - * `NixStringContextElem::Built` element of the drv path and output name. + * The string is the printed store path with a context containing a + * single `NixStringContextElem::Built` element of the drv path and + * output name. * * @param value Value we are settings * - * @param drvPath Path the drv whose output we are making a string for + * @param b the drv whose output we are making a string for, and the + * output * - * @param outputName Name of the output - * - * @param optStaticOutputPath Optional output path for that string. Must - * be passed if and only if output store object is input-addressed. - * Will be printed to form string if passed, otherwise a placeholder - * will be used (see `DownstreamPlaceholder`). + * @param optStaticOutputPath Optional output path for that string. + * Must be passed if and only if output store object is + * input-addressed or fixed output. Will be printed to form string + * if passed, otherwise a placeholder will be used (see + * `DownstreamPlaceholder`). * * @param xpSettings Stop-gap to avoid globals during unit tests. */ void mkOutputString( Value & value, - const StorePath & drvPath, - const std::string outputName, + const SingleDerivedPath::Built & b, std::optional optStaticOutputPath, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + /** + * Create a string representing a `SingleDerivedPath`. + * + * A combination of `mkStorePathString` and `mkOutputString`. + */ + void mkSingleDerivedPathString( + const SingleDerivedPath & p, + Value & v); + void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); /** @@ -714,6 +723,22 @@ public: private: + /** + * Like `mkOutputString` but just creates a raw string, not an + * string Value, which would also have a string context. + */ + std::string mkOutputStringRaw( + const SingleDerivedPath::Built & b, + std::optional optStaticOutputPath, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + /** + * Like `mkSingleDerivedPathStringRaw` but just creates a raw string + * Value, which would also have a string context. + */ + std::string mkSingleDerivedPathStringRaw( + const SingleDerivedPath & p); + unsigned long nrEnvs = 0; unsigned long nrValuesInEnvs = 0; unsigned long nrValues = 0; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 54943b481..be78bca02 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -156,8 +156,10 @@ static void mkOutputString( { state.mkOutputString( attrs.alloc(o.first), - drvPath, - o.first, + SingleDerivedPath::Built { + .drvPath = makeConstantStorePathRef(drvPath), + .output = o.first, + }, o.second.path(*state.store, Derivation::nameFromPath(drvPath), o.first)); } diff --git a/src/libexpr/tests/derived-path.cc b/src/libexpr/tests/derived-path.cc index 2a5ca64f6..d01735db8 100644 --- a/src/libexpr/tests/derived-path.cc +++ b/src/libexpr/tests/derived-path.cc @@ -34,8 +34,8 @@ RC_GTEST_FIXTURE_PROP( RC_GTEST_FIXTURE_PROP( DerivedPathExpressionTest, - prop_built_path_placeholder_round_trip, - (const StorePath & drvPath, const StorePathName & outputName)) + prop_derived_path_built_placeholder_round_trip, + (const SingleDerivedPath::Built & b)) { /** * We set these in tests rather than the regular globals so we don't have @@ -45,27 +45,19 @@ RC_GTEST_FIXTURE_PROP( mockXpSettings.set("experimental-features", "ca-derivations"); auto * v = state.allocValue(); - state.mkOutputString(*v, drvPath, outputName.name, std::nullopt, mockXpSettings); + state.mkOutputString(*v, b, std::nullopt, mockXpSettings); auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, ""); - SingleDerivedPath::Built b { - .drvPath = makeConstantStorePathRef(drvPath), - .output = outputName.name, - }; RC_ASSERT(SingleDerivedPath { b } == d); } RC_GTEST_FIXTURE_PROP( DerivedPathExpressionTest, - prop_built_path_out_path_round_trip, - (const StorePath & drvPath, const StorePathName & outputName, const StorePath & outPath)) + prop_derived_path_built_out_path_round_trip, + (const SingleDerivedPath::Built & b, const StorePath & outPath)) { auto * v = state.allocValue(); - state.mkOutputString(*v, drvPath, outputName.name, outPath); + state.mkOutputString(*v, b, outPath); auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, ""); - SingleDerivedPath::Built b { - .drvPath = makeConstantStorePathRef(drvPath), - .output = outputName.name, - }; RC_ASSERT(SingleDerivedPath { b } == d); } diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index 885b3604d..d951b7b7d 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -39,16 +39,18 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( } DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt( - const SingleDerivedPath::Built & b) + const SingleDerivedPath::Built & b, + const ExperimentalFeatureSettings & xpSettings) { return std::visit(overloaded { [&](const SingleDerivedPath::Opaque & o) { - return DownstreamPlaceholder::unknownCaOutput(o.path, b.output); + return DownstreamPlaceholder::unknownCaOutput(o.path, b.output, xpSettings); }, [&](const SingleDerivedPath::Built & b2) { return DownstreamPlaceholder::unknownDerivation( - DownstreamPlaceholder::fromSingleDerivedPathBuilt(b2), - b.output); + DownstreamPlaceholder::fromSingleDerivedPathBuilt(b2, xpSettings), + b.output, + xpSettings); }, }, b.drvPath->raw()); } diff --git a/src/libstore/downstream-placeholder.hh b/src/libstore/downstream-placeholder.hh index 9372dcd58..d58a2ac14 100644 --- a/src/libstore/downstream-placeholder.hh +++ b/src/libstore/downstream-placeholder.hh @@ -84,7 +84,8 @@ public: * `SingleDerivedPath::Built.drvPath` chain. */ static DownstreamPlaceholder fromSingleDerivedPathBuilt( - const SingleDerivedPath::Built & built); + const SingleDerivedPath::Built & built, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); }; } From 44c8d83831e310d445493d7071161c622bc302aa Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 10 Mar 2021 04:22:56 +0000 Subject: [PATCH 176/909] Create `outputOf` primop. In the Nix language, given a drv path, we should be able to construct another string referencing to one of its output. We can do this today with `(import drvPath).output`, but this only works for derivations we already have. With dynamic derivations, however, that doesn't work well because the `drvPath` isn't yet built: importing it like would need to trigger IFD, when the whole point of this feature is to do "dynamic build graph" without IFD! Instead, what we want to do is create a placeholder value with the right string context to refer to the output of the as-yet unbuilt derivation. A new primop in the language, analogous to `builtins.placeholder` can be used to create one. This will achieve all the right properties. The placeholder machinery also will match out the `outPath` attribute for CA derivations works. In 60b7121d2c6d4322b7c2e8e7acfec7b701b2d3a1 we added that type of placeholder, and the derived path and string holder changes necessary to support it. Then in the previous commit we cleaned up the code (inspiration finally hit me!) to deduplicate the code and expose exactly what we need. Now, we can wire up the primop trivally! Part of RFC 92: dynamic derivations (tracking issue #6316) Co-authored-by: Robert Hensing --- doc/manual/src/release-notes/rl-next.md | 3 + src/libexpr/primops.cc | 39 ++++++++++++ src/nix/main.cc | 1 + tests/dyn-drv/dep-built-drv.sh | 9 +++ tests/dyn-drv/eval-outputOf.sh | 80 +++++++++++++++++++++++++ tests/dyn-drv/local.mk | 4 +- tests/dyn-drv/recursive-mod-json.sh | 2 + tests/dyn-drv/text-hashed-output.nix | 10 +++- 8 files changed, 144 insertions(+), 4 deletions(-) create mode 100644 tests/dyn-drv/dep-built-drv.sh create mode 100644 tests/dyn-drv/eval-outputOf.sh diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index be922f95a..7ddb5ca00 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -19,3 +19,6 @@ - The JSON output for derived paths with are store paths is now a string, not an object with a single `path` field. This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. + +- Introduce a new [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) builtin. + It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index be78bca02..283f99a48 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1838,6 +1838,45 @@ static RegisterPrimOp primop_readDir({ .fun = prim_readDir, }); +/* Extend single element string context with another output. */ +static void prim_outputOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + SingleDerivedPath drvPath = state.coerceToSingleDerivedPath(pos, *args[0], "while evaluating the first argument to builtins.outputOf"); + + std::string_view outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf"); + + state.mkSingleDerivedPathString( + SingleDerivedPath::Built { + .drvPath = make_ref(drvPath), + .output = std::string { outputName }, + }, + v); +} + +static RegisterPrimOp primop_outputOf({ + .name = "__outputOf", + .args = {"derivation-reference", "output-name"}, + .doc = R"( + Return the output path of a derivation, literally or using a placeholder if needed. + + If the derivation has a statically-known output path (i.e. the derivation output is input-addressed, or fixed content-addresed), the output path will just be returned. + But if the derivation is content-addressed or if the derivation is itself not-statically produced (i.e. is the output of another derivation), a placeholder will be returned instead. + + *`derivation reference`* must be a string that may contain a regular store path to a derivation, or may be a placeholder reference. If the derivation is produced by a derivation, you must explicitly select `drv.outPath`. + This primop can be chained arbitrarily deeply. + For instance, + ```nix + builtins.outputOf + (builtins.outputOf myDrv "out) + "out" + ``` + will return a placeholder for the output of the output of `myDrv`. + + This primop corresponds to the `^` sigil for derivable paths, e.g. as part of installable syntax on the command line. + )", + .fun = prim_outputOf, + .experimentalFeature = Xp::DynamicDerivations, +}); /************************************************************* * Creating files diff --git a/src/nix/main.cc b/src/nix/main.cc index c5a9c8b33..d05bac68e 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -359,6 +359,7 @@ void mainWrapped(int argc, char * * argv) experimentalFeatureSettings.experimentalFeatures = { Xp::Flakes, Xp::FetchClosure, + Xp::DynamicDerivations, }; evalSettings.pureEval = false; EvalState state({}, openStore("dummy://")); diff --git a/tests/dyn-drv/dep-built-drv.sh b/tests/dyn-drv/dep-built-drv.sh new file mode 100644 index 000000000..8c4a45e3b --- /dev/null +++ b/tests/dyn-drv/dep-built-drv.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +source common.sh + +out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link) + +clearStore + +expectStderr 1 nix-build ./text-hashed-output.nix -A wrapper --no-out-link | grepQuiet "Dependencies on the outputs of dynamic derivations are not yet supported" diff --git a/tests/dyn-drv/eval-outputOf.sh b/tests/dyn-drv/eval-outputOf.sh new file mode 100644 index 000000000..99d917c06 --- /dev/null +++ b/tests/dyn-drv/eval-outputOf.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +source ./common.sh + +# Without the dynamic-derivations XP feature, we don't have the builtin. +nix --experimental-features 'nix-command' eval --impure --expr \ + 'assert ! (builtins ? outputOf); ""' + +# Test that a string is required. +# +# We currently require a string to be passed, rather than a derivation +# object that could be coerced to a string. We might liberalise this in +# the future so it does work, but there are some design questions to +# resolve first. Adding a test so we don't liberalise it by accident. +expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \ + 'builtins.outputOf (import ../dependencies.nix) "out"' \ + | grepQuiet "value is a set while a string was expected" + +# Test that "DrvDeep" string contexts are not supported at this time +# +# Like the above, this is a restriction we could relax later. +expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \ + 'builtins.outputOf (import ../dependencies.nix).drvPath "out"' \ + | grepQuiet "has a context which refers to a complete source and binary closure. This is not supported at this time" + +# Test using `builtins.outputOf` with static derivations +testStaticHello () { + nix eval --impure --expr \ + 'with (import ./text-hashed-output.nix); let + a = hello.outPath; + b = builtins.outputOf (builtins.unsafeDiscardOutputDependency hello.drvPath) "out"; + in builtins.trace a + (builtins.trace b + (assert a == b; null))' +} + +# Test with a regular old input-addresed derivation +# +# `builtins.outputOf` works without ca-derivations and doesn't create a +# placeholder but just returns the output path. +testStaticHello + +# Test with content addressed derivation. +NIX_TESTS_CA_BY_DEFAULT=1 testStaticHello + +# Test with derivation-producing derivation +# +# This is hardly different from the preceding cases, except that we're +# only taking 1 outputOf out of 2 possible outputOfs. Note that +# `.outPath` could be defined as `outputOf drvPath`, which is what we're +# testing here. The other `outputOf` that we're not testing here is the +# use of _dynamic_ derivations. +nix eval --impure --expr \ + 'with (import ./text-hashed-output.nix); let + a = producingDrv.outPath; + b = builtins.outputOf (builtins.builtins.unsafeDiscardOutputDependency producingDrv.drvPath) "out"; + in builtins.trace a + (builtins.trace b + (assert a == b; null))' + +# Test with unbuilt output of derivation-producing derivation. +# +# This function similar to `testStaticHello` used above, but instead of +# checking the property on a constant derivation, we check it on a +# derivation that's from another derivation's output (outPath). +testDynamicHello () { + nix eval --impure --expr \ + 'with (import ./text-hashed-output.nix); let + a = builtins.outputOf producingDrv.outPath "out"; + b = builtins.outputOf (builtins.outputOf (builtins.unsafeDiscardOutputDependency producingDrv.drvPath) "out") "out"; + in builtins.trace a + (builtins.trace b + (assert a == b; null))' +} + +# inner dynamic derivation is input-addressed +testDynamicHello + +# inner dynamic derivation is content-addressed +NIX_TESTS_CA_BY_DEFAULT=1 testDynamicHello diff --git a/tests/dyn-drv/local.mk b/tests/dyn-drv/local.mk index b087ecd1c..0ce7cd37d 100644 --- a/tests/dyn-drv/local.mk +++ b/tests/dyn-drv/local.mk @@ -1,7 +1,9 @@ dyn-drv-tests := \ $(d)/text-hashed-output.sh \ $(d)/recursive-mod-json.sh \ - $(d)/build-built-drv.sh + $(d)/build-built-drv.sh \ + $(d)/eval-outputOf.sh \ + $(d)/dep-built-drv.sh install-tests-groups += dyn-drv diff --git a/tests/dyn-drv/recursive-mod-json.sh b/tests/dyn-drv/recursive-mod-json.sh index 070c5c2cb..0698b81bd 100644 --- a/tests/dyn-drv/recursive-mod-json.sh +++ b/tests/dyn-drv/recursive-mod-json.sh @@ -3,6 +3,8 @@ source common.sh # FIXME if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi +export NIX_TESTS_CA_BY_DEFAULT=1 + enableFeatures 'recursive-nix' restartDaemon diff --git a/tests/dyn-drv/text-hashed-output.nix b/tests/dyn-drv/text-hashed-output.nix index a700fd102..99203b518 100644 --- a/tests/dyn-drv/text-hashed-output.nix +++ b/tests/dyn-drv/text-hashed-output.nix @@ -12,9 +12,6 @@ rec { mkdir -p $out echo "Hello World" > $out/hello ''; - __contentAddressed = true; - outputHashMode = "recursive"; - outputHashAlgo = "sha256"; }; producingDrv = mkDerivation { name = "hello.drv"; @@ -26,4 +23,11 @@ rec { outputHashMode = "text"; outputHashAlgo = "sha256"; }; + wrapper = mkDerivation { + name = "use-dynamic-drv-in-non-dynamic-drv"; + buildCommand = '' + echo "Copying the output of the dynamic derivation" + cp -r ${builtins.outputOf producingDrv.outPath "out"} $out + ''; + }; } From 1ef8008ca7a7b38570f06abdbb7960e93ac044ef Mon Sep 17 00:00:00 2001 From: Alex Zero Date: Sun, 13 Aug 2023 01:31:32 +0100 Subject: [PATCH 177/909] Fix follow path checking at depths greater than 2 We need to recurse into the input tree to handle follows paths that trarverse multiple inputs that may or may not be follow paths themselves. --- src/libexpr/flake/lockfile.cc | 2 +- tests/flakes/follow-paths.sh | 63 +++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index ba2fd46f0..3c202967a 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -345,7 +345,7 @@ void LockFile::check() for (auto & [inputPath, input] : inputs) { if (auto follows = std::get_if<1>(&input)) { - if (!follows->empty() && !get(inputs, *follows)) + if (!follows->empty() && !findInput(*follows)) throw Error("input '%s' follows a non-existent input '%s'", printInputPath(inputPath), printInputPath(*follows)); diff --git a/tests/flakes/follow-paths.sh b/tests/flakes/follow-paths.sh index fe9b51c65..a70d9acb2 100644 --- a/tests/flakes/follow-paths.sh +++ b/tests/flakes/follow-paths.sh @@ -148,3 +148,66 @@ git -C $flakeFollowsA add flake.nix nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid'" nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid2'" + +# Now test follow path overloading +flakeFollowsOverloadA=$TEST_ROOT/follows/overload/flakeA +flakeFollowsOverloadB=$TEST_ROOT/follows/overload/flakeA/flakeB +flakeFollowsOverloadC=$TEST_ROOT/follows/overload/flakeA/flakeB/flakeC +flakeFollowsOverloadD=$TEST_ROOT/follows/overload/flakeA/flakeB/flakeC/flakeD + +# Test following path flakerefs. +createGitRepo $flakeFollowsOverloadA +mkdir -p $flakeFollowsOverloadB +mkdir -p $flakeFollowsOverloadC +mkdir -p $flakeFollowsOverloadD + +cat > $flakeFollowsOverloadD/flake.nix < $flakeFollowsOverloadC/flake.nix < $flakeFollowsOverloadB/flake.nix < $flakeFollowsOverloadA/flake.nix < Date: Mon, 14 Aug 2023 18:52:21 +0100 Subject: [PATCH 178/909] Add release notes for the previous commit --- doc/manual/src/release-notes/rl-next.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 7ddb5ca00..5a870f1ab 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -22,3 +22,5 @@ - Introduce a new [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) builtin. It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + +- Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. From b74962c92b2d8d9b957934e0aefcf4983169ae1e Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Mon, 14 Aug 2023 22:07:37 +0100 Subject: [PATCH 179/909] src/libexpr/search-path.cc: avoid out-of-bounds read on string_view Without the change build with `-D_GLIBCXX_ASSERTIONS` exposes testsuite assertion: $ gdb src/libexpr/tests/libnixexpr-tests Reading symbols from src/libexpr/tests/libnixexpr-tests... (gdb) break __glibcxx_assert_fail (gdb) run (gdb) bt in std::__glibcxx_assert_fail(char const*, int, char const*, char const*)@plt () from /mnt/archive/big/git/nix/src/libexpr/libnixexpr.so in std::basic_string_view >::operator[] (this=0x7fffffff56c0, __pos=4) at /nix/store/r74fw2j8rx5idb0w8s1s6ynwwgs0qmh9-gcc-14.0.0/include/c++/14.0.0/string_view:258 in nix::SearchPath::Prefix::suffixIfPotentialMatch (this=0x7fffffff5780, path=...) at src/libexpr/search-path.cc:15 in nix::SearchPathElem_suffixIfPotentialMatch_partialPrefix_Test::TestBody (this=0x555555a17540) at src/libexpr/tests/search-path.cc:62 As string sizes are usigned types `(a - b) > 0` effectively means `a != b`. While the intention should be `a > b`. The change fixes test suite pass. --- src/libexpr/search-path.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/search-path.cc b/src/libexpr/search-path.cc index 36bb4c3a5..180d5f8b1 100644 --- a/src/libexpr/search-path.cc +++ b/src/libexpr/search-path.cc @@ -10,7 +10,7 @@ std::optional SearchPath::Prefix::suffixIfPotentialMatch( /* Non-empty prefix and suffix must be separated by a /, or the prefix is not a valid path prefix. */ - bool needSeparator = n > 0 && (path.size() - n) > 0; + bool needSeparator = n > 0 && n < path.size(); if (needSeparator && path[n] != '/') { return std::nullopt; From 20d9c672d14de3f05a791251a67618d679cf4173 Mon Sep 17 00:00:00 2001 From: Vertex <5567402+VertexA115@users.noreply.github.com> Date: Tue, 15 Aug 2023 10:10:27 +0100 Subject: [PATCH 180/909] Update tests/flakes/follow-paths.sh Co-authored-by: Robert Hensing --- tests/flakes/follow-paths.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/flakes/follow-paths.sh b/tests/flakes/follow-paths.sh index a70d9acb2..f9d5c8c80 100644 --- a/tests/flakes/follow-paths.sh +++ b/tests/flakes/follow-paths.sh @@ -150,6 +150,25 @@ nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override fo nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid2'" # Now test follow path overloading +# This tests a lockfile checking regression https://github.com/NixOS/nix/pull/8819 +# +# We construct the following graph, where p->q means p has input q. +# A double edge means that the edge gets overridden using `follows`. +# +# A +# / \ +# / \ +# v v +# B ==> C --- follows declared in A +# \\ / +# \\/ --- follows declared in B +# v +# D +# +# The message was +# error: input 'B/D' follows a non-existent input 'B/C/D' +# +# Note that for `B` to resolve its follow for `D`, it needs `C/D`, for which it needs to resolve the follow on `C` first. flakeFollowsOverloadA=$TEST_ROOT/follows/overload/flakeA flakeFollowsOverloadB=$TEST_ROOT/follows/overload/flakeA/flakeB flakeFollowsOverloadC=$TEST_ROOT/follows/overload/flakeA/flakeB/flakeC From b13fc7101fb06a65935d145081319145f7fef6f9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 14 Aug 2023 12:24:26 +0200 Subject: [PATCH 181/909] Add positive source filter Source filtering is a really cool Nix feature that lets us avoid a lot of rebuilds, which speeds up the iteration cycle a lot in cases where the relevant source files aren't actually modified. We used to have a source filter that marked a few files as irrelevant, but this is the wrong approach, as we have many more files that are irrelevant. We may call this negative filtering. This commit switches the source filtering to positive filtering, which is a lot more robust. Instead of marking which files we don't need we marked the files that we do need. It's a superior approach because it is fail safe. Instead of allowing build performance problems to creep in over time, we require that all source inputs are declared. I shouldn't have to explain that declaring inputs is a good practice, so I'll stop over-explaining here. I do have to acknowledge that this will cause a build failure when the filter is incomplete. This is *good*, because it's the only realistic way we could be reminded of these problems. These events will be infrequent, so the small cost of extending the filter is worth it, compared to the hidden cost of longer dev cycles for things like tests, docker image, etc, etc. (Also rebuilding Nix for stupid unnecessary reasons makes my blood boil) --- flake.nix | 61 +++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/flake.nix b/flake.nix index bdbf54169..28e581309 100644 --- a/flake.nix +++ b/flake.nix @@ -40,6 +40,52 @@ }) stdenvs); + # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981 + # Not an "idiomatic" flake input because: + # - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730 + # - Subflake would download redundant and huge parent flake + # - No git tree hash support: https://github.com/NixOS/nix/issues/6044 + inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nixpkgs/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; })) + fileset; + + baseFiles = + # .gitignore has already been processed, so any changes in it are irrelevant + # at this point. It is not represented verbatim for test purposes because + # that would interfere with repo semantics. + fileset.fileFilter (f: f.name != ".gitignore") ./.; + + nixSrc = fileset.toSource { + root = ./.; + fileset = fileset.intersect baseFiles ( + fileset.difference + (fileset.unions [ + ./.version + ./boehmgc-coroutine-sp-fallback.diff + ./bootstrap.sh + ./configure.ac + ./doc + ./local.mk + ./m4 + ./Makefile + ./Makefile.config.in + ./misc + ./mk + ./precompiled-headers.h + ./src + ./tests + ./COPYING + ./scripts/local.mk + (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) + # TODO: do we really need README.md? It doesn't seem used in the build. + ./README.md + ]) + (fileset.unions [ + # Removed file sets + ./tests/nixos + ./tests/installer + ]) + ); + }; # Memoize nixpkgs for different platforms for efficiency. nixpkgsFor = forAllSystems @@ -209,7 +255,7 @@ "-${client.version}-against-${daemon.version}"; inherit version; - src = self; + src = nixSrc; VERSION_SUFFIX = versionSuffix; @@ -320,18 +366,11 @@ }; let canRunInstalled = currentStdenv.buildPlatform.canExecute currentStdenv.hostPlatform; - - sourceByRegexInverted = rxs: origSrc: final.lib.cleanSourceWith { - filter = (path: type: - let relPath = final.lib.removePrefix (toString origSrc + "/") (toString path); - in ! lib.any (re: builtins.match re relPath != null) rxs); - src = origSrc; - }; in currentStdenv.mkDerivation (finalAttrs: { name = "nix-${version}"; inherit version; - src = sourceByRegexInverted [ "tests/nixos/.*" "tests/installer/.*" ] self; + src = nixSrc; VERSION_SUFFIX = versionSuffix; outputs = [ "out" "dev" "doc" ]; @@ -529,7 +568,7 @@ releaseTools.coverageAnalysis { name = "nix-coverage-${version}"; - src = self; + src = nixSrc; configureFlags = testConfigureFlags; @@ -557,7 +596,7 @@ pname = "nix-internal-api-docs"; inherit version; - src = self; + src = nixSrc; configureFlags = testConfigureFlags ++ internalApiDocsConfigureFlags; From 63e0b5d081fed582ac6a0a66f402dc525953524b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 16 Aug 2023 15:46:37 +0200 Subject: [PATCH 182/909] GC root for fetched nixpkgs/lib content --- ascii-table.nix | 99 + asserts.nix | 53 + attrsets.nix | 1018 ++++++++++ cli.nix | 83 + customisation.nix | 315 ++++ debug.nix | 253 +++ default.nix | 167 ++ deprecated.nix | 307 +++ derivations.nix | 101 + fetchers.nix | 13 + fileset.nix | 993 ++++++++++ filesystem.nix | 194 ++ fixed-points.nix | 113 ++ flake.nix | 5 + generators.nix | 526 ++++++ kernel.nix | 27 + licenses.nix | 1104 +++++++++++ lists.nix | 724 +++++++ meta.nix | 148 ++ minver.nix | 2 + modules.nix | 1287 +++++++++++++ options.nix | 432 +++++ path/README.md | 196 ++ path/default.nix | 433 +++++ path/tests/default.nix | 34 + path/tests/generate.awk | 64 + path/tests/prop.nix | 60 + path/tests/prop.sh | 179 ++ path/tests/unit.nix | 193 ++ source-types.nix | 19 + sources.nix | 292 +++ strings-with-deps.nix | 84 + strings.nix | 1243 +++++++++++++ systems/architectures.nix | 114 ++ systems/default.nix | 228 +++ systems/doubles.nix | 119 ++ systems/examples.nix | 335 ++++ systems/flake-systems.nix | 29 + systems/inspect.nix | 117 ++ systems/parse.nix | 503 +++++ systems/platforms.nix | 565 ++++++ tests/check-eval.nix | 7 + tests/filesystem.sh | 84 + tests/maintainer-module.nix | 32 + tests/maintainers.nix | 53 + tests/misc.nix | 1657 +++++++++++++++++ tests/modules.sh | 408 ++++ ...adhoc-freeformType-survives-type-merge.nix | 14 + .../alias-with-priority-can-override.nix | 55 + tests/modules/alias-with-priority.nix | 55 + tests/modules/attrsOf-conditional-check.nix | 7 + tests/modules/attrsOf-lazy-check.nix | 7 + tests/modules/class-check.nix | 76 + tests/modules/declare-attrsOf.nix | 13 + .../modules/declare-attrsOfSub-any-enable.nix | 29 + ...e-bare-submodule-deep-option-duplicate.nix | 10 + .../declare-bare-submodule-deep-option.nix | 10 + .../declare-bare-submodule-nested-option.nix | 19 + tests/modules/declare-bare-submodule.nix | 18 + .../modules/declare-coerced-value-unsound.nix | 10 + tests/modules/declare-coerced-value.nix | 10 + tests/modules/declare-either.nix | 5 + tests/modules/declare-enable-nested.nix | 14 + tests/modules/declare-enable.nix | 14 + tests/modules/declare-int-between-value.nix | 9 + .../declare-int-positive-value-nested.nix | 9 + tests/modules/declare-int-positive-value.nix | 9 + tests/modules/declare-int-unsigned-value.nix | 9 + tests/modules/declare-lazyAttrsOf.nix | 6 + tests/modules/declare-mkPackageOption.nix | 19 + tests/modules/declare-oneOf.nix | 9 + tests/modules/declare-set.nix | 12 + .../declare-submodule-via-evalModules.nix | 28 + .../modules/declare-submoduleWith-modules.nix | 28 + .../declare-submoduleWith-noshorthand.nix | 13 + tests/modules/declare-submoduleWith-path.nix | 12 + .../declare-submoduleWith-shorthand.nix | 14 + .../modules/declare-submoduleWith-special.nix | 17 + tests/modules/declare-variants.nix | 9 + tests/modules/default.nix | 8 + tests/modules/deferred-module-error.nix | 20 + tests/modules/deferred-module.nix | 58 + tests/modules/define-_module-args-custom.nix | 7 + .../modules/define-attrsOfSub-bar-enable.nix | 3 + tests/modules/define-attrsOfSub-bar.nix | 3 + .../define-attrsOfSub-foo-enable-force.nix | 5 + .../define-attrsOfSub-foo-enable-if.nix | 5 + .../modules/define-attrsOfSub-foo-enable.nix | 3 + .../define-attrsOfSub-foo-force-enable.nix | 7 + .../define-attrsOfSub-foo-if-enable.nix | 7 + tests/modules/define-attrsOfSub-foo.nix | 3 + .../define-attrsOfSub-force-foo-enable.nix | 7 + .../define-attrsOfSub-if-foo-enable.nix | 7 + .../modules/define-bare-submodule-values.nix | 4 + tests/modules/define-enable-abort.nix | 3 + tests/modules/define-enable-force.nix | 5 + tests/modules/define-enable-throw.nix | 3 + .../modules/define-enable-with-custom-arg.nix | 7 + .../define-enable-with-top-level-mkIf.nix | 5 + tests/modules/define-enable.nix | 3 + .../define-force-attrsOfSub-foo-enable.nix | 5 + tests/modules/define-force-enable.nix | 5 + .../define-freeform-keywords-shorthand.nix | 15 + .../define-if-attrsOfSub-foo-enable.nix | 5 + tests/modules/define-module-check.nix | 3 + .../define-option-dependently-nested.nix | 16 + tests/modules/define-option-dependently.nix | 16 + tests/modules/define-settingsDict-a-is-b.nix | 3 + ...define-shorthandOnlyDefinesConfig-true.nix | 1 + .../define-submoduleWith-noshorthand.nix | 3 + .../define-submoduleWith-shorthand.nix | 3 + tests/modules/define-value-int-negative.nix | 3 + tests/modules/define-value-int-positive.nix | 3 + tests/modules/define-value-int-zero.nix | 3 + tests/modules/define-value-list.nix | 3 + .../modules/define-value-string-arbitrary.nix | 3 + tests/modules/define-value-string-bigint.nix | 3 + .../define-value-string-properties.nix | 12 + tests/modules/define-value-string.nix | 3 + tests/modules/define-variant.nix | 22 + tests/modules/disable-declare-enable.nix | 5 + .../disable-define-enable-string-path.nix | 5 + tests/modules/disable-define-enable.nix | 5 + tests/modules/disable-enable-modules.nix | 5 + tests/modules/disable-module-bad-key.nix | 16 + tests/modules/disable-module-with-key.nix | 34 + .../disable-module-with-toString-key.nix | 34 + tests/modules/disable-recursive/bar.nix | 5 + .../modules/disable-recursive/disable-bar.nix | 7 + .../modules/disable-recursive/disable-foo.nix | 7 + tests/modules/disable-recursive/foo.nix | 5 + tests/modules/disable-recursive/main.nix | 8 + tests/modules/doRename-basic.nix | 11 + tests/modules/doRename-warnings.nix | 14 + tests/modules/emptyValues.nix | 36 + .../modules/extendModules-168767-imports.nix | 41 + tests/modules/freeform-attrsOf.nix | 3 + tests/modules/freeform-lazyAttrsOf.nix | 3 + tests/modules/freeform-nested.nix | 14 + tests/modules/freeform-str-dep-unstr.nix | 8 + tests/modules/freeform-submodules.nix | 22 + tests/modules/freeform-unstr-dep-str.nix | 8 + tests/modules/functionTo/list-order.nix | 25 + tests/modules/functionTo/merging-attrs.nix | 27 + tests/modules/functionTo/merging-list.nix | 24 + .../modules/functionTo/submodule-options.nix | 61 + tests/modules/functionTo/trivial.nix | 17 + tests/modules/functionTo/wrong-type.nix | 18 + tests/modules/import-configuration.nix | 12 + tests/modules/import-custom-arg.nix | 6 + tests/modules/import-from-store.nix | 11 + tests/modules/merge-module-with-key.nix | 49 + tests/modules/module-class-is-darwin.nix | 4 + tests/modules/module-class-is-nixos.nix | 4 + tests/modules/module-imports-_type-check.nix | 3 + tests/modules/optionTypeFile.nix | 28 + tests/modules/optionTypeMerging.nix | 27 + tests/modules/raw.nix | 30 + tests/modules/shorthand-meta.nix | 19 + tests/modules/submoduleFiles.nix | 21 + .../types-anything/attrs-coercible.nix | 12 + tests/modules/types-anything/equal-atoms.nix | 26 + tests/modules/types-anything/functions.nix | 23 + tests/modules/types-anything/lists.nix | 16 + tests/modules/types-anything/mk-mods.nix | 44 + tests/modules/types-anything/nested-attrs.nix | 22 + tests/release.nix | 64 + tests/sources.sh | 66 + tests/systems.nix | 42 + tests/teams.nix | 50 + tests/test-to-plist-expected.plist | 46 + trivial.nix | 522 ++++++ types.nix | 908 +++++++++ versions.nix | 64 + zip-int-bits.nix | 39 + 175 files changed, 18510 insertions(+) create mode 100644 ascii-table.nix create mode 100644 asserts.nix create mode 100644 attrsets.nix create mode 100644 cli.nix create mode 100644 customisation.nix create mode 100644 debug.nix create mode 100644 default.nix create mode 100644 deprecated.nix create mode 100644 derivations.nix create mode 100644 fetchers.nix create mode 100644 fileset.nix create mode 100644 filesystem.nix create mode 100644 fixed-points.nix create mode 100644 flake.nix create mode 100644 generators.nix create mode 100644 kernel.nix create mode 100644 licenses.nix create mode 100644 lists.nix create mode 100644 meta.nix create mode 100644 minver.nix create mode 100644 modules.nix create mode 100644 options.nix create mode 100644 path/README.md create mode 100644 path/default.nix create mode 100644 path/tests/default.nix create mode 100644 path/tests/generate.awk create mode 100644 path/tests/prop.nix create mode 100755 path/tests/prop.sh create mode 100644 path/tests/unit.nix create mode 100644 source-types.nix create mode 100644 sources.nix create mode 100644 strings-with-deps.nix create mode 100644 strings.nix create mode 100644 systems/architectures.nix create mode 100644 systems/default.nix create mode 100644 systems/doubles.nix create mode 100644 systems/examples.nix create mode 100644 systems/flake-systems.nix create mode 100644 systems/inspect.nix create mode 100644 systems/parse.nix create mode 100644 systems/platforms.nix create mode 100644 tests/check-eval.nix create mode 100755 tests/filesystem.sh create mode 100644 tests/maintainer-module.nix create mode 100644 tests/maintainers.nix create mode 100644 tests/misc.nix create mode 100755 tests/modules.sh create mode 100644 tests/modules/adhoc-freeformType-survives-type-merge.nix create mode 100644 tests/modules/alias-with-priority-can-override.nix create mode 100644 tests/modules/alias-with-priority.nix create mode 100644 tests/modules/attrsOf-conditional-check.nix create mode 100644 tests/modules/attrsOf-lazy-check.nix create mode 100644 tests/modules/class-check.nix create mode 100644 tests/modules/declare-attrsOf.nix create mode 100644 tests/modules/declare-attrsOfSub-any-enable.nix create mode 100644 tests/modules/declare-bare-submodule-deep-option-duplicate.nix create mode 100644 tests/modules/declare-bare-submodule-deep-option.nix create mode 100644 tests/modules/declare-bare-submodule-nested-option.nix create mode 100644 tests/modules/declare-bare-submodule.nix create mode 100644 tests/modules/declare-coerced-value-unsound.nix create mode 100644 tests/modules/declare-coerced-value.nix create mode 100644 tests/modules/declare-either.nix create mode 100644 tests/modules/declare-enable-nested.nix create mode 100644 tests/modules/declare-enable.nix create mode 100644 tests/modules/declare-int-between-value.nix create mode 100644 tests/modules/declare-int-positive-value-nested.nix create mode 100644 tests/modules/declare-int-positive-value.nix create mode 100644 tests/modules/declare-int-unsigned-value.nix create mode 100644 tests/modules/declare-lazyAttrsOf.nix create mode 100644 tests/modules/declare-mkPackageOption.nix create mode 100644 tests/modules/declare-oneOf.nix create mode 100644 tests/modules/declare-set.nix create mode 100644 tests/modules/declare-submodule-via-evalModules.nix create mode 100644 tests/modules/declare-submoduleWith-modules.nix create mode 100644 tests/modules/declare-submoduleWith-noshorthand.nix create mode 100644 tests/modules/declare-submoduleWith-path.nix create mode 100644 tests/modules/declare-submoduleWith-shorthand.nix create mode 100644 tests/modules/declare-submoduleWith-special.nix create mode 100644 tests/modules/declare-variants.nix create mode 100644 tests/modules/default.nix create mode 100644 tests/modules/deferred-module-error.nix create mode 100644 tests/modules/deferred-module.nix create mode 100644 tests/modules/define-_module-args-custom.nix create mode 100644 tests/modules/define-attrsOfSub-bar-enable.nix create mode 100644 tests/modules/define-attrsOfSub-bar.nix create mode 100644 tests/modules/define-attrsOfSub-foo-enable-force.nix create mode 100644 tests/modules/define-attrsOfSub-foo-enable-if.nix create mode 100644 tests/modules/define-attrsOfSub-foo-enable.nix create mode 100644 tests/modules/define-attrsOfSub-foo-force-enable.nix create mode 100644 tests/modules/define-attrsOfSub-foo-if-enable.nix create mode 100644 tests/modules/define-attrsOfSub-foo.nix create mode 100644 tests/modules/define-attrsOfSub-force-foo-enable.nix create mode 100644 tests/modules/define-attrsOfSub-if-foo-enable.nix create mode 100644 tests/modules/define-bare-submodule-values.nix create mode 100644 tests/modules/define-enable-abort.nix create mode 100644 tests/modules/define-enable-force.nix create mode 100644 tests/modules/define-enable-throw.nix create mode 100644 tests/modules/define-enable-with-custom-arg.nix create mode 100644 tests/modules/define-enable-with-top-level-mkIf.nix create mode 100644 tests/modules/define-enable.nix create mode 100644 tests/modules/define-force-attrsOfSub-foo-enable.nix create mode 100644 tests/modules/define-force-enable.nix create mode 100644 tests/modules/define-freeform-keywords-shorthand.nix create mode 100644 tests/modules/define-if-attrsOfSub-foo-enable.nix create mode 100644 tests/modules/define-module-check.nix create mode 100644 tests/modules/define-option-dependently-nested.nix create mode 100644 tests/modules/define-option-dependently.nix create mode 100644 tests/modules/define-settingsDict-a-is-b.nix create mode 100644 tests/modules/define-shorthandOnlyDefinesConfig-true.nix create mode 100644 tests/modules/define-submoduleWith-noshorthand.nix create mode 100644 tests/modules/define-submoduleWith-shorthand.nix create mode 100644 tests/modules/define-value-int-negative.nix create mode 100644 tests/modules/define-value-int-positive.nix create mode 100644 tests/modules/define-value-int-zero.nix create mode 100644 tests/modules/define-value-list.nix create mode 100644 tests/modules/define-value-string-arbitrary.nix create mode 100644 tests/modules/define-value-string-bigint.nix create mode 100644 tests/modules/define-value-string-properties.nix create mode 100644 tests/modules/define-value-string.nix create mode 100644 tests/modules/define-variant.nix create mode 100644 tests/modules/disable-declare-enable.nix create mode 100644 tests/modules/disable-define-enable-string-path.nix create mode 100644 tests/modules/disable-define-enable.nix create mode 100644 tests/modules/disable-enable-modules.nix create mode 100644 tests/modules/disable-module-bad-key.nix create mode 100644 tests/modules/disable-module-with-key.nix create mode 100644 tests/modules/disable-module-with-toString-key.nix create mode 100644 tests/modules/disable-recursive/bar.nix create mode 100644 tests/modules/disable-recursive/disable-bar.nix create mode 100644 tests/modules/disable-recursive/disable-foo.nix create mode 100644 tests/modules/disable-recursive/foo.nix create mode 100644 tests/modules/disable-recursive/main.nix create mode 100644 tests/modules/doRename-basic.nix create mode 100644 tests/modules/doRename-warnings.nix create mode 100644 tests/modules/emptyValues.nix create mode 100644 tests/modules/extendModules-168767-imports.nix create mode 100644 tests/modules/freeform-attrsOf.nix create mode 100644 tests/modules/freeform-lazyAttrsOf.nix create mode 100644 tests/modules/freeform-nested.nix create mode 100644 tests/modules/freeform-str-dep-unstr.nix create mode 100644 tests/modules/freeform-submodules.nix create mode 100644 tests/modules/freeform-unstr-dep-str.nix create mode 100644 tests/modules/functionTo/list-order.nix create mode 100644 tests/modules/functionTo/merging-attrs.nix create mode 100644 tests/modules/functionTo/merging-list.nix create mode 100644 tests/modules/functionTo/submodule-options.nix create mode 100644 tests/modules/functionTo/trivial.nix create mode 100644 tests/modules/functionTo/wrong-type.nix create mode 100644 tests/modules/import-configuration.nix create mode 100644 tests/modules/import-custom-arg.nix create mode 100644 tests/modules/import-from-store.nix create mode 100644 tests/modules/merge-module-with-key.nix create mode 100644 tests/modules/module-class-is-darwin.nix create mode 100644 tests/modules/module-class-is-nixos.nix create mode 100644 tests/modules/module-imports-_type-check.nix create mode 100644 tests/modules/optionTypeFile.nix create mode 100644 tests/modules/optionTypeMerging.nix create mode 100644 tests/modules/raw.nix create mode 100644 tests/modules/shorthand-meta.nix create mode 100644 tests/modules/submoduleFiles.nix create mode 100644 tests/modules/types-anything/attrs-coercible.nix create mode 100644 tests/modules/types-anything/equal-atoms.nix create mode 100644 tests/modules/types-anything/functions.nix create mode 100644 tests/modules/types-anything/lists.nix create mode 100644 tests/modules/types-anything/mk-mods.nix create mode 100644 tests/modules/types-anything/nested-attrs.nix create mode 100644 tests/release.nix create mode 100755 tests/sources.sh create mode 100644 tests/systems.nix create mode 100644 tests/teams.nix create mode 100644 tests/test-to-plist-expected.plist create mode 100644 trivial.nix create mode 100644 types.nix create mode 100644 versions.nix create mode 100644 zip-int-bits.nix diff --git a/ascii-table.nix b/ascii-table.nix new file mode 100644 index 000000000..74989936e --- /dev/null +++ b/ascii-table.nix @@ -0,0 +1,99 @@ +{ "\t" = 9; + "\n" = 10; + "\r" = 13; + " " = 32; + "!" = 33; + "\"" = 34; + "#" = 35; + "$" = 36; + "%" = 37; + "&" = 38; + "'" = 39; + "(" = 40; + ")" = 41; + "*" = 42; + "+" = 43; + "," = 44; + "-" = 45; + "." = 46; + "/" = 47; + "0" = 48; + "1" = 49; + "2" = 50; + "3" = 51; + "4" = 52; + "5" = 53; + "6" = 54; + "7" = 55; + "8" = 56; + "9" = 57; + ":" = 58; + ";" = 59; + "<" = 60; + "=" = 61; + ">" = 62; + "?" = 63; + "@" = 64; + "A" = 65; + "B" = 66; + "C" = 67; + "D" = 68; + "E" = 69; + "F" = 70; + "G" = 71; + "H" = 72; + "I" = 73; + "J" = 74; + "K" = 75; + "L" = 76; + "M" = 77; + "N" = 78; + "O" = 79; + "P" = 80; + "Q" = 81; + "R" = 82; + "S" = 83; + "T" = 84; + "U" = 85; + "V" = 86; + "W" = 87; + "X" = 88; + "Y" = 89; + "Z" = 90; + "[" = 91; + "\\" = 92; + "]" = 93; + "^" = 94; + "_" = 95; + "`" = 96; + "a" = 97; + "b" = 98; + "c" = 99; + "d" = 100; + "e" = 101; + "f" = 102; + "g" = 103; + "h" = 104; + "i" = 105; + "j" = 106; + "k" = 107; + "l" = 108; + "m" = 109; + "n" = 110; + "o" = 111; + "p" = 112; + "q" = 113; + "r" = 114; + "s" = 115; + "t" = 116; + "u" = 117; + "v" = 118; + "w" = 119; + "x" = 120; + "y" = 121; + "z" = 122; + "{" = 123; + "|" = 124; + "}" = 125; + "~" = 126; +} diff --git a/asserts.nix b/asserts.nix new file mode 100644 index 000000000..98e0b490a --- /dev/null +++ b/asserts.nix @@ -0,0 +1,53 @@ +{ lib }: + +rec { + + /* Throw if pred is false, else return pred. + Intended to be used to augment asserts with helpful error messages. + + Example: + assertMsg false "nope" + stderr> error: nope + + assert assertMsg ("foo" == "bar") "foo is not bar, silly"; "" + stderr> error: foo is not bar, silly + + Type: + assertMsg :: Bool -> String -> Bool + */ + # TODO(Profpatsch): add tests that check stderr + assertMsg = + # Predicate that needs to succeed, otherwise `msg` is thrown + pred: + # Message to throw in case `pred` fails + msg: + pred || builtins.throw msg; + + /* Specialized `assertMsg` for checking if `val` is one of the elements + of the list `xs`. Useful for checking enums. + + Example: + let sslLibrary = "libressl"; + in assertOneOf "sslLibrary" sslLibrary [ "openssl" "bearssl" ] + stderr> error: sslLibrary must be one of [ + stderr> "openssl" + stderr> "bearssl" + stderr> ], but is: "libressl" + + Type: + assertOneOf :: String -> ComparableVal -> List ComparableVal -> Bool + */ + assertOneOf = + # The name of the variable the user entered `val` into, for inclusion in the error message + name: + # The value of what the user provided, to be compared against the values in `xs` + val: + # The list of valid values + xs: + assertMsg + (lib.elem val xs) + "${name} must be one of ${ + lib.generators.toPretty {} xs}, but is: ${ + lib.generators.toPretty {} val}"; + +} diff --git a/attrsets.nix b/attrsets.nix new file mode 100644 index 000000000..73b71f68d --- /dev/null +++ b/attrsets.nix @@ -0,0 +1,1018 @@ +{ lib }: +# Operations on attribute sets. + +let + inherit (builtins) head tail length; + inherit (lib.trivial) flip id mergeAttrs pipe; + inherit (lib.strings) concatStringsSep concatMapStringsSep escapeNixIdentifier sanitizeDerivationName; + inherit (lib.lists) foldr foldl' concatMap concatLists elemAt all partition groupBy take foldl; +in + +rec { + inherit (builtins) attrNames listToAttrs hasAttr isAttrs getAttr removeAttrs; + + + /* Return an attribute from nested attribute sets. + + Example: + x = { a = { b = 3; }; } + # ["a" "b"] is equivalent to x.a.b + # 6 is a default value to return if the path does not exist in attrset + attrByPath ["a" "b"] 6 x + => 3 + attrByPath ["z" "z"] 6 x + => 6 + + Type: + attrByPath :: [String] -> Any -> AttrSet -> Any + + */ + attrByPath = + # A list of strings representing the attribute path to return from `set` + attrPath: + # Default value if `attrPath` does not resolve to an existing value + default: + # The nested attribute set to select values from + set: + let attr = head attrPath; + in + if attrPath == [] then set + else if set ? ${attr} + then attrByPath (tail attrPath) default set.${attr} + else default; + + /* Return if an attribute from nested attribute set exists. + + Example: + x = { a = { b = 3; }; } + hasAttrByPath ["a" "b"] x + => true + hasAttrByPath ["z" "z"] x + => false + + Type: + hasAttrByPath :: [String] -> AttrSet -> Bool + */ + hasAttrByPath = + # A list of strings representing the attribute path to check from `set` + attrPath: + # The nested attribute set to check + e: + let attr = head attrPath; + in + if attrPath == [] then true + else if e ? ${attr} + then hasAttrByPath (tail attrPath) e.${attr} + else false; + + + /* Create a new attribute set with `value` set at the nested attribute location specified in `attrPath`. + + Example: + setAttrByPath ["a" "b"] 3 + => { a = { b = 3; }; } + + Type: + setAttrByPath :: [String] -> Any -> AttrSet + */ + setAttrByPath = + # A list of strings representing the attribute path to set + attrPath: + # The value to set at the location described by `attrPath` + value: + let + len = length attrPath; + atDepth = n: + if n == len + then value + else { ${elemAt attrPath n} = atDepth (n + 1); }; + in atDepth 0; + + /* Like `attrByPath`, but without a default value. If it doesn't find the + path it will throw an error. + + Example: + x = { a = { b = 3; }; } + getAttrFromPath ["a" "b"] x + => 3 + getAttrFromPath ["z" "z"] x + => error: cannot find attribute `z.z' + + Type: + getAttrFromPath :: [String] -> AttrSet -> Any + */ + getAttrFromPath = + # A list of strings representing the attribute path to get from `set` + attrPath: + # The nested attribute set to find the value in. + set: + let errorMsg = "cannot find attribute `" + concatStringsSep "." attrPath + "'"; + in attrByPath attrPath (abort errorMsg) set; + + /* Map each attribute in the given set and merge them into a new attribute set. + + Type: + concatMapAttrs :: (String -> a -> AttrSet) -> AttrSet -> AttrSet + + Example: + concatMapAttrs + (name: value: { + ${name} = value; + ${name + value} = value; + }) + { x = "a"; y = "b"; } + => { x = "a"; xa = "a"; y = "b"; yb = "b"; } + */ + concatMapAttrs = f: v: + foldl' mergeAttrs { } + (attrValues + (mapAttrs f v) + ); + + + /* Update or set specific paths of an attribute set. + + Takes a list of updates to apply and an attribute set to apply them to, + and returns the attribute set with the updates applied. Updates are + represented as `{ path = ...; update = ...; }` values, where `path` is a + list of strings representing the attribute path that should be updated, + and `update` is a function that takes the old value at that attribute path + as an argument and returns the new + value it should be. + + Properties: + + - Updates to deeper attribute paths are applied before updates to more + shallow attribute paths + + - Multiple updates to the same attribute path are applied in the order + they appear in the update list + + - If any but the last `path` element leads into a value that is not an + attribute set, an error is thrown + + - If there is an update for an attribute path that doesn't exist, + accessing the argument in the update function causes an error, but + intermediate attribute sets are implicitly created as needed + + Example: + updateManyAttrsByPath [ + { + path = [ "a" "b" ]; + update = old: { d = old.c; }; + } + { + path = [ "a" "b" "c" ]; + update = old: old + 1; + } + { + path = [ "x" "y" ]; + update = old: "xy"; + } + ] { a.b.c = 0; } + => { a = { b = { d = 1; }; }; x = { y = "xy"; }; } + + Type: updateManyAttrsByPath :: [{ path :: [String]; update :: (Any -> Any); }] -> AttrSet -> AttrSet + */ + updateManyAttrsByPath = let + # When recursing into attributes, instead of updating the `path` of each + # update using `tail`, which needs to allocate an entirely new list, + # we just pass a prefix length to use and make sure to only look at the + # path without the prefix length, so that we can reuse the original list + # entries. + go = prefixLength: hasValue: value: updates: + let + # Splits updates into ones on this level (split.right) + # And ones on levels further down (split.wrong) + split = partition (el: length el.path == prefixLength) updates; + + # Groups updates on further down levels into the attributes they modify + nested = groupBy (el: elemAt el.path prefixLength) split.wrong; + + # Applies only nested modification to the input value + withNestedMods = + # Return the value directly if we don't have any nested modifications + if split.wrong == [] then + if hasValue then value + else + # Throw an error if there is no value. This `head` call here is + # safe, but only in this branch since `go` could only be called + # with `hasValue == false` for nested updates, in which case + # it's also always called with at least one update + let updatePath = (head split.right).path; in + throw + ( "updateManyAttrsByPath: Path '${showAttrPath updatePath}' does " + + "not exist in the given value, but the first update to this " + + "path tries to access the existing value.") + else + # If there are nested modifications, try to apply them to the value + if ! hasValue then + # But if we don't have a value, just use an empty attribute set + # as the value, but simplify the code a bit + mapAttrs (name: go (prefixLength + 1) false null) nested + else if isAttrs value then + # If we do have a value and it's an attribute set, override it + # with the nested modifications + value // + mapAttrs (name: go (prefixLength + 1) (value ? ${name}) value.${name}) nested + else + # However if it's not an attribute set, we can't apply the nested + # modifications, throw an error + let updatePath = (head split.wrong).path; in + throw + ( "updateManyAttrsByPath: Path '${showAttrPath updatePath}' needs to " + + "be updated, but path '${showAttrPath (take prefixLength updatePath)}' " + + "of the given value is not an attribute set, so we can't " + + "update an attribute inside of it."); + + # We get the final result by applying all the updates on this level + # after having applied all the nested updates + # We use foldl instead of foldl' so that in case of multiple updates, + # intermediate values aren't evaluated if not needed + in foldl (acc: el: el.update acc) withNestedMods split.right; + + in updates: value: go 0 true value updates; + + /* Return the specified attributes from a set. + + Example: + attrVals ["a" "b" "c"] as + => [as.a as.b as.c] + + Type: + attrVals :: [String] -> AttrSet -> [Any] + */ + attrVals = + # The list of attributes to fetch from `set`. Each attribute name must exist on the attrbitue set + nameList: + # The set to get attribute values from + set: map (x: set.${x}) nameList; + + + /* Return the values of all attributes in the given set, sorted by + attribute name. + + Example: + attrValues {c = 3; a = 1; b = 2;} + => [1 2 3] + + Type: + attrValues :: AttrSet -> [Any] + */ + attrValues = builtins.attrValues or (attrs: attrVals (attrNames attrs) attrs); + + + /* Given a set of attribute names, return the set of the corresponding + attributes from the given set. + + Example: + getAttrs [ "a" "b" ] { a = 1; b = 2; c = 3; } + => { a = 1; b = 2; } + + Type: + getAttrs :: [String] -> AttrSet -> AttrSet + */ + getAttrs = + # A list of attribute names to get out of `set` + names: + # The set to get the named attributes from + attrs: genAttrs names (name: attrs.${name}); + + /* Collect each attribute named `attr` from a list of attribute + sets. Sets that don't contain the named attribute are ignored. + + Example: + catAttrs "a" [{a = 1;} {b = 0;} {a = 2;}] + => [1 2] + + Type: + catAttrs :: String -> [AttrSet] -> [Any] + */ + catAttrs = builtins.catAttrs or + (attr: l: concatLists (map (s: if s ? ${attr} then [s.${attr}] else []) l)); + + + /* Filter an attribute set by removing all attributes for which the + given predicate return false. + + Example: + filterAttrs (n: v: n == "foo") { foo = 1; bar = 2; } + => { foo = 1; } + + Type: + filterAttrs :: (String -> Any -> Bool) -> AttrSet -> AttrSet + */ + filterAttrs = + # Predicate taking an attribute name and an attribute value, which returns `true` to include the attribute, or `false` to exclude the attribute. + pred: + # The attribute set to filter + set: + listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set)); + + + /* Filter an attribute set recursively by removing all attributes for + which the given predicate return false. + + Example: + filterAttrsRecursive (n: v: v != null) { foo = { bar = null; }; } + => { foo = {}; } + + Type: + filterAttrsRecursive :: (String -> Any -> Bool) -> AttrSet -> AttrSet + */ + filterAttrsRecursive = + # Predicate taking an attribute name and an attribute value, which returns `true` to include the attribute, or `false` to exclude the attribute. + pred: + # The attribute set to filter + set: + listToAttrs ( + concatMap (name: + let v = set.${name}; in + if pred name v then [ + (nameValuePair name ( + if isAttrs v then filterAttrsRecursive pred v + else v + )) + ] else [] + ) (attrNames set) + ); + + /* + Like builtins.foldl' but for attribute sets. + Iterates over every name-value pair in the given attribute set. + The result of the callback function is often called `acc` for accumulator. It is passed between callbacks from left to right and the final `acc` is the return value of `foldlAttrs`. + + Attention: + There is a completely different function + `lib.foldAttrs` + which has nothing to do with this function, despite the similar name. + + Example: + foldlAttrs + (acc: name: value: { + sum = acc.sum + value; + names = acc.names ++ [name]; + }) + { sum = 0; names = []; } + { + foo = 1; + bar = 10; + } + -> + { + sum = 11; + names = ["bar" "foo"]; + } + + foldlAttrs + (throw "function not needed") + 123 + {}; + -> + 123 + + foldlAttrs + (_: _: v: v) + (throw "initial accumulator not needed") + { z = 3; a = 2; }; + -> + 3 + + The accumulator doesn't have to be an attrset. + It can be as simple as a number or string. + + foldlAttrs + (acc: _: v: acc * 10 + v) + 1 + { z = 1; a = 2; }; + -> + 121 + + Type: + foldlAttrs :: ( a -> String -> b -> a ) -> a -> { ... :: b } -> a + */ + foldlAttrs = f: init: set: + foldl' + (acc: name: f acc name set.${name}) + init + (attrNames set); + + /* Apply fold functions to values grouped by key. + + Example: + foldAttrs (item: acc: [item] ++ acc) [] [{ a = 2; } { a = 3; }] + => { a = [ 2 3 ]; } + + Type: + foldAttrs :: (Any -> Any -> Any) -> Any -> [AttrSets] -> Any + + */ + foldAttrs = + # A function, given a value and a collector combines the two. + op: + # The starting value. + nul: + # A list of attribute sets to fold together by key. + list_of_attrs: + foldr (n: a: + foldr (name: o: + o // { ${name} = op n.${name} (a.${name} or nul); } + ) a (attrNames n) + ) {} list_of_attrs; + + + /* Recursively collect sets that verify a given predicate named `pred` + from the set `attrs`. The recursion is stopped when the predicate is + verified. + + Example: + collect isList { a = { b = ["b"]; }; c = [1]; } + => [["b"] [1]] + + collect (x: x ? outPath) + { a = { outPath = "a/"; }; b = { outPath = "b/"; }; } + => [{ outPath = "a/"; } { outPath = "b/"; }] + + Type: + collect :: (AttrSet -> Bool) -> AttrSet -> [x] + */ + collect = + # Given an attribute's value, determine if recursion should stop. + pred: + # The attribute set to recursively collect. + attrs: + if pred attrs then + [ attrs ] + else if isAttrs attrs then + concatMap (collect pred) (attrValues attrs) + else + []; + + /* Return the cartesian product of attribute set value combinations. + + Example: + cartesianProductOfSets { a = [ 1 2 ]; b = [ 10 20 ]; } + => [ + { a = 1; b = 10; } + { a = 1; b = 20; } + { a = 2; b = 10; } + { a = 2; b = 20; } + ] + Type: + cartesianProductOfSets :: AttrSet -> [AttrSet] + */ + cartesianProductOfSets = + # Attribute set with attributes that are lists of values + attrsOfLists: + foldl' (listOfAttrs: attrName: + concatMap (attrs: + map (listValue: attrs // { ${attrName} = listValue; }) attrsOfLists.${attrName} + ) listOfAttrs + ) [{}] (attrNames attrsOfLists); + + + /* Utility function that creates a `{name, value}` pair as expected by `builtins.listToAttrs`. + + Example: + nameValuePair "some" 6 + => { name = "some"; value = 6; } + + Type: + nameValuePair :: String -> Any -> { name :: String; value :: Any; } + */ + nameValuePair = + # Attribute name + name: + # Attribute value + value: + { inherit name value; }; + + + /* Apply a function to each element in an attribute set, creating a new attribute set. + + Example: + mapAttrs (name: value: name + "-" + value) + { x = "foo"; y = "bar"; } + => { x = "x-foo"; y = "y-bar"; } + + Type: + mapAttrs :: (String -> Any -> Any) -> AttrSet -> AttrSet + */ + mapAttrs = builtins.mapAttrs or + (f: set: + listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))); + + + /* Like `mapAttrs`, but allows the name of each attribute to be + changed in addition to the value. The applied function should + return both the new name and value as a `nameValuePair`. + + Example: + mapAttrs' (name: value: nameValuePair ("foo_" + name) ("bar-" + value)) + { x = "a"; y = "b"; } + => { foo_x = "bar-a"; foo_y = "bar-b"; } + + Type: + mapAttrs' :: (String -> Any -> { name :: String; value :: Any; }) -> AttrSet -> AttrSet + */ + mapAttrs' = + # A function, given an attribute's name and value, returns a new `nameValuePair`. + f: + # Attribute set to map over. + set: + listToAttrs (map (attr: f attr set.${attr}) (attrNames set)); + + + /* Call a function for each attribute in the given set and return + the result in a list. + + Example: + mapAttrsToList (name: value: name + value) + { x = "a"; y = "b"; } + => [ "xa" "yb" ] + + Type: + mapAttrsToList :: (String -> a -> b) -> AttrSet -> [b] + + */ + mapAttrsToList = + # A function, given an attribute's name and value, returns a new value. + f: + # Attribute set to map over. + attrs: + map (name: f name attrs.${name}) (attrNames attrs); + + + /* Like `mapAttrs`, except that it recursively applies itself to + the *leaf* attributes of a potentially-nested attribute set: + the second argument of the function will never be an attrset. + Also, the first argument of the argument function is a *list* + of the attribute names that form the path to the leaf attribute. + + For a function that gives you control over what counts as a leaf, + see `mapAttrsRecursiveCond`. + + Example: + mapAttrsRecursive (path: value: concatStringsSep "-" (path ++ [value])) + { n = { a = "A"; m = { b = "B"; c = "C"; }; }; d = "D"; } + => { n = { a = "n-a-A"; m = { b = "n-m-b-B"; c = "n-m-c-C"; }; }; d = "d-D"; } + + Type: + mapAttrsRecursive :: ([String] -> a -> b) -> AttrSet -> AttrSet + */ + mapAttrsRecursive = + # A function, given a list of attribute names and a value, returns a new value. + f: + # Set to recursively map over. + set: + mapAttrsRecursiveCond (as: true) f set; + + + /* Like `mapAttrsRecursive`, but it takes an additional predicate + function that tells it whether to recurse into an attribute + set. If it returns false, `mapAttrsRecursiveCond` does not + recurse, but does apply the map function. If it returns true, it + does recurse, and does not apply the map function. + + Example: + # To prevent recursing into derivations (which are attribute + # sets with the attribute "type" equal to "derivation"): + mapAttrsRecursiveCond + (as: !(as ? "type" && as.type == "derivation")) + (x: ... do something ...) + attrs + + Type: + mapAttrsRecursiveCond :: (AttrSet -> Bool) -> ([String] -> a -> b) -> AttrSet -> AttrSet + */ + mapAttrsRecursiveCond = + # A function, given the attribute set the recursion is currently at, determine if to recurse deeper into that attribute set. + cond: + # A function, given a list of attribute names and a value, returns a new value. + f: + # Attribute set to recursively map over. + set: + let + recurse = path: + let + g = + name: value: + if isAttrs value && cond value + then recurse (path ++ [name]) value + else f (path ++ [name]) value; + in mapAttrs g; + in recurse [] set; + + + /* Generate an attribute set by mapping a function over a list of + attribute names. + + Example: + genAttrs [ "foo" "bar" ] (name: "x_" + name) + => { foo = "x_foo"; bar = "x_bar"; } + + Type: + genAttrs :: [ String ] -> (String -> Any) -> AttrSet + */ + genAttrs = + # Names of values in the resulting attribute set. + names: + # A function, given the name of the attribute, returns the attribute's value. + f: + listToAttrs (map (n: nameValuePair n (f n)) names); + + + /* Check whether the argument is a derivation. Any set with + `{ type = "derivation"; }` counts as a derivation. + + Example: + nixpkgs = import {} + isDerivation nixpkgs.ruby + => true + isDerivation "foobar" + => false + + Type: + isDerivation :: Any -> Bool + */ + isDerivation = + # Value to check. + value: value.type or null == "derivation"; + + /* Converts a store path to a fake derivation. + + Type: + toDerivation :: Path -> Derivation + */ + toDerivation = + # A store path to convert to a derivation. + path: + let + path' = builtins.storePath path; + res = + { type = "derivation"; + name = sanitizeDerivationName (builtins.substring 33 (-1) (baseNameOf path')); + outPath = path'; + outputs = [ "out" ]; + out = res; + outputName = "out"; + }; + in res; + + + /* If `cond` is true, return the attribute set `as`, + otherwise an empty attribute set. + + Example: + optionalAttrs (true) { my = "set"; } + => { my = "set"; } + optionalAttrs (false) { my = "set"; } + => { } + + Type: + optionalAttrs :: Bool -> AttrSet -> AttrSet + */ + optionalAttrs = + # Condition under which the `as` attribute set is returned. + cond: + # The attribute set to return if `cond` is `true`. + as: + if cond then as else {}; + + + /* Merge sets of attributes and use the function `f` to merge attributes + values. + + Example: + zipAttrsWithNames ["a"] (name: vs: vs) [{a = "x";} {a = "y"; b = "z";}] + => { a = ["x" "y"]; } + + Type: + zipAttrsWithNames :: [ String ] -> (String -> [ Any ] -> Any) -> [ AttrSet ] -> AttrSet + */ + zipAttrsWithNames = + # List of attribute names to zip. + names: + # A function, accepts an attribute name, all the values, and returns a combined value. + f: + # List of values from the list of attribute sets. + sets: + listToAttrs (map (name: { + inherit name; + value = f name (catAttrs name sets); + }) names); + + + /* Merge sets of attributes and use the function f to merge attribute values. + Like `lib.attrsets.zipAttrsWithNames` with all key names are passed for `names`. + + Implementation note: Common names appear multiple times in the list of + names, hopefully this does not affect the system because the maximal + laziness avoid computing twice the same expression and `listToAttrs` does + not care about duplicated attribute names. + + Example: + zipAttrsWith (name: values: values) [{a = "x";} {a = "y"; b = "z";}] + => { a = ["x" "y"]; b = ["z"]; } + + Type: + zipAttrsWith :: (String -> [ Any ] -> Any) -> [ AttrSet ] -> AttrSet + */ + zipAttrsWith = + builtins.zipAttrsWith or (f: sets: zipAttrsWithNames (concatMap attrNames sets) f sets); + + + /* Merge sets of attributes and combine each attribute value in to a list. + + Like `lib.attrsets.zipAttrsWith` with `(name: values: values)` as the function. + + Example: + zipAttrs [{a = "x";} {a = "y"; b = "z";}] + => { a = ["x" "y"]; b = ["z"]; } + + Type: + zipAttrs :: [ AttrSet ] -> AttrSet + */ + zipAttrs = + # List of attribute sets to zip together. + sets: + zipAttrsWith (name: values: values) sets; + + + /* Does the same as the update operator '//' except that attributes are + merged until the given predicate is verified. The predicate should + accept 3 arguments which are the path to reach the attribute, a part of + the first attribute set and a part of the second attribute set. When + the predicate is satisfied, the value of the first attribute set is + replaced by the value of the second attribute set. + + Example: + recursiveUpdateUntil (path: l: r: path == ["foo"]) { + # first attribute set + foo.bar = 1; + foo.baz = 2; + bar = 3; + } { + #second attribute set + foo.bar = 1; + foo.quz = 2; + baz = 4; + } + + => { + foo.bar = 1; # 'foo.*' from the second set + foo.quz = 2; # + bar = 3; # 'bar' from the first set + baz = 4; # 'baz' from the second set + } + + Type: + recursiveUpdateUntil :: ( [ String ] -> AttrSet -> AttrSet -> Bool ) -> AttrSet -> AttrSet -> AttrSet + */ + recursiveUpdateUntil = + # Predicate, taking the path to the current attribute as a list of strings for attribute names, and the two values at that path from the original arguments. + pred: + # Left attribute set of the merge. + lhs: + # Right attribute set of the merge. + rhs: + let f = attrPath: + zipAttrsWith (n: values: + let here = attrPath ++ [n]; in + if length values == 1 + || pred here (elemAt values 1) (head values) then + head values + else + f here values + ); + in f [] [rhs lhs]; + + + /* A recursive variant of the update operator ‘//’. The recursion + stops when one of the attribute values is not an attribute set, + in which case the right hand side value takes precedence over the + left hand side value. + + Example: + recursiveUpdate { + boot.loader.grub.enable = true; + boot.loader.grub.device = "/dev/hda"; + } { + boot.loader.grub.device = ""; + } + + returns: { + boot.loader.grub.enable = true; + boot.loader.grub.device = ""; + } + + Type: + recursiveUpdate :: AttrSet -> AttrSet -> AttrSet + */ + recursiveUpdate = + # Left attribute set of the merge. + lhs: + # Right attribute set of the merge. + rhs: + recursiveUpdateUntil (path: lhs: rhs: !(isAttrs lhs && isAttrs rhs)) lhs rhs; + + + /* Returns true if the pattern is contained in the set. False otherwise. + + Example: + matchAttrs { cpu = {}; } { cpu = { bits = 64; }; } + => true + + Type: + matchAttrs :: AttrSet -> AttrSet -> Bool + */ + matchAttrs = + # Attribute set structure to match + pattern: + # Attribute set to find patterns in + attrs: + assert isAttrs pattern; + all id (attrValues (zipAttrsWithNames (attrNames pattern) (n: values: + let pat = head values; val = elemAt values 1; in + if length values == 1 then false + else if isAttrs pat then isAttrs val && matchAttrs pat val + else pat == val + ) [pattern attrs])); + + + /* Override only the attributes that are already present in the old set + useful for deep-overriding. + + Example: + overrideExisting {} { a = 1; } + => {} + overrideExisting { b = 2; } { a = 1; } + => { b = 2; } + overrideExisting { a = 3; b = 2; } { a = 1; } + => { a = 1; b = 2; } + + Type: + overrideExisting :: AttrSet -> AttrSet -> AttrSet + */ + overrideExisting = + # Original attribute set + old: + # Attribute set with attributes to override in `old`. + new: + mapAttrs (name: value: new.${name} or value) old; + + + /* Turns a list of strings into a human-readable description of those + strings represented as an attribute path. The result of this function is + not intended to be machine-readable. + Create a new attribute set with `value` set at the nested attribute location specified in `attrPath`. + + Example: + showAttrPath [ "foo" "10" "bar" ] + => "foo.\"10\".bar" + showAttrPath [] + => "" + + Type: + showAttrPath :: [String] -> String + */ + showAttrPath = + # Attribute path to render to a string + path: + if path == [] then "" + else concatMapStringsSep "." escapeNixIdentifier path; + + + /* Get a package output. + If no output is found, fallback to `.out` and then to the default. + + Example: + getOutput "dev" pkgs.openssl + => "/nix/store/9rz8gxhzf8sw4kf2j2f1grr49w8zx5vj-openssl-1.0.1r-dev" + + Type: + getOutput :: String -> Derivation -> String + */ + getOutput = output: pkg: + if ! pkg ? outputSpecified || ! pkg.outputSpecified + then pkg.${output} or pkg.out or pkg + else pkg; + + /* Get a package's `bin` output. + If the output does not exist, fallback to `.out` and then to the default. + + Example: + getBin pkgs.openssl + => "/nix/store/9rz8gxhzf8sw4kf2j2f1grr49w8zx5vj-openssl-1.0.1r" + + Type: + getBin :: Derivation -> String + */ + getBin = getOutput "bin"; + + + /* Get a package's `lib` output. + If the output does not exist, fallback to `.out` and then to the default. + + Example: + getLib pkgs.openssl + => "/nix/store/9rz8gxhzf8sw4kf2j2f1grr49w8zx5vj-openssl-1.0.1r-lib" + + Type: + getLib :: Derivation -> String + */ + getLib = getOutput "lib"; + + + /* Get a package's `dev` output. + If the output does not exist, fallback to `.out` and then to the default. + + Example: + getDev pkgs.openssl + => "/nix/store/9rz8gxhzf8sw4kf2j2f1grr49w8zx5vj-openssl-1.0.1r-dev" + + Type: + getDev :: Derivation -> String + */ + getDev = getOutput "dev"; + + + /* Get a package's `man` output. + If the output does not exist, fallback to `.out` and then to the default. + + Example: + getMan pkgs.openssl + => "/nix/store/9rz8gxhzf8sw4kf2j2f1grr49w8zx5vj-openssl-1.0.1r-man" + + Type: + getMan :: Derivation -> String + */ + getMan = getOutput "man"; + + /* Pick the outputs of packages to place in `buildInputs` + + Type: chooseDevOutputs :: [Derivation] -> [String] + + */ + chooseDevOutputs = + # List of packages to pick `dev` outputs from + drvs: + builtins.map getDev drvs; + + /* Make various Nix tools consider the contents of the resulting + attribute set when looking for what to build, find, etc. + + This function only affects a single attribute set; it does not + apply itself recursively for nested attribute sets. + + Example: + { pkgs ? import {} }: + { + myTools = pkgs.lib.recurseIntoAttrs { + inherit (pkgs) hello figlet; + }; + } + + Type: + recurseIntoAttrs :: AttrSet -> AttrSet + + */ + recurseIntoAttrs = + # An attribute set to scan for derivations. + attrs: + attrs // { recurseForDerivations = true; }; + + /* Undo the effect of recurseIntoAttrs. + + Type: + dontRecurseIntoAttrs :: AttrSet -> AttrSet + */ + dontRecurseIntoAttrs = + # An attribute set to not scan for derivations. + attrs: + attrs // { recurseForDerivations = false; }; + + /* `unionOfDisjoint x y` is equal to `x // y // z` where the + attrnames in `z` are the intersection of the attrnames in `x` and + `y`, and all values `assert` with an error message. This + operator is commutative, unlike (//). + + Type: unionOfDisjoint :: AttrSet -> AttrSet -> AttrSet + */ + unionOfDisjoint = x: y: + let + intersection = builtins.intersectAttrs x y; + collisions = lib.concatStringsSep " " (builtins.attrNames intersection); + mask = builtins.mapAttrs (name: value: builtins.throw + "unionOfDisjoint: collision on ${name}; complete list: ${collisions}") + intersection; + in + (x // y) // mask; + + # DEPRECATED + zipWithNames = zipAttrsWithNames; + + # DEPRECATED + zip = builtins.trace + "lib.zip is deprecated, use lib.zipAttrsWith instead" zipAttrsWith; +} diff --git a/cli.nix b/cli.nix new file mode 100644 index 000000000..c96d4dbb0 --- /dev/null +++ b/cli.nix @@ -0,0 +1,83 @@ +{ lib }: + +rec { + /* Automatically convert an attribute set to command-line options. + + This helps protect against malformed command lines and also to reduce + boilerplate related to command-line construction for simple use cases. + + `toGNUCommandLine` returns a list of nix strings. + `toGNUCommandLineShell` returns an escaped shell string. + + Example: + cli.toGNUCommandLine {} { + data = builtins.toJSON { id = 0; }; + X = "PUT"; + retry = 3; + retry-delay = null; + url = [ "https://example.com/foo" "https://example.com/bar" ]; + silent = false; + verbose = true; + } + => [ + "-X" "PUT" + "--data" "{\"id\":0}" + "--retry" "3" + "--url" "https://example.com/foo" + "--url" "https://example.com/bar" + "--verbose" + ] + + cli.toGNUCommandLineShell {} { + data = builtins.toJSON { id = 0; }; + X = "PUT"; + retry = 3; + retry-delay = null; + url = [ "https://example.com/foo" "https://example.com/bar" ]; + silent = false; + verbose = true; + } + => "'-X' 'PUT' '--data' '{\"id\":0}' '--retry' '3' '--url' 'https://example.com/foo' '--url' 'https://example.com/bar' '--verbose'"; + */ + toGNUCommandLineShell = + options: attrs: lib.escapeShellArgs (toGNUCommandLine options attrs); + + toGNUCommandLine = { + # how to string-format the option name; + # by default one character is a short option (`-`), + # more than one characters a long option (`--`). + mkOptionName ? + k: if builtins.stringLength k == 1 + then "-${k}" + else "--${k}", + + # how to format a boolean value to a command list; + # by default it’s a flag option + # (only the option name if true, left out completely if false). + mkBool ? k: v: lib.optional v (mkOptionName k), + + # how to format a list value to a command list; + # by default the option name is repeated for each value + # and `mkOption` is applied to the values themselves. + mkList ? k: v: lib.concatMap (mkOption k) v, + + # how to format any remaining value to a command list; + # on the toplevel, booleans and lists are handled by `mkBool` and `mkList`, + # though they can still appear as values of a list. + # By default, everything is printed verbatim and complex types + # are forbidden (lists, attrsets, functions). `null` values are omitted. + mkOption ? + k: v: if v == null + then [] + else [ (mkOptionName k) (lib.generators.mkValueStringDefault {} v) ] + }: + options: + let + render = k: v: + if builtins.isBool v then mkBool k v + else if builtins.isList v then mkList k v + else mkOption k v; + + in + builtins.concatLists (lib.mapAttrsToList render options); +} diff --git a/customisation.nix b/customisation.nix new file mode 100644 index 000000000..fe32e890f --- /dev/null +++ b/customisation.nix @@ -0,0 +1,315 @@ +{ lib }: + +rec { + + + /* `overrideDerivation drv f` takes a derivation (i.e., the result + of a call to the builtin function `derivation`) and returns a new + derivation in which the attributes of the original are overridden + according to the function `f`. The function `f` is called with + the original derivation attributes. + + `overrideDerivation` allows certain "ad-hoc" customisation + scenarios (e.g. in ~/.config/nixpkgs/config.nix). For instance, + if you want to "patch" the derivation returned by a package + function in Nixpkgs to build another version than what the + function itself provides, you can do something like this: + + mySed = overrideDerivation pkgs.gnused (oldAttrs: { + name = "sed-4.2.2-pre"; + src = fetchurl { + url = ftp://alpha.gnu.org/gnu/sed/sed-4.2.2-pre.tar.bz2; + sha256 = "11nq06d131y4wmf3drm0yk502d2xc6n5qy82cg88rb9nqd2lj41k"; + }; + patches = []; + }); + + For another application, see build-support/vm, where this + function is used to build arbitrary derivations inside a QEMU + virtual machine. + + Note that in order to preserve evaluation errors, the new derivation's + outPath depends on the old one's, which means that this function cannot + be used in circular situations when the old derivation also depends on the + new one. + + You should in general prefer `drv.overrideAttrs` over this function; + see the nixpkgs manual for more information on overriding. + */ + overrideDerivation = drv: f: + let + newDrv = derivation (drv.drvAttrs // (f drv)); + in lib.flip (extendDerivation (builtins.seq drv.drvPath true)) newDrv ( + { meta = drv.meta or {}; + passthru = if drv ? passthru then drv.passthru else {}; + } + // + (drv.passthru or {}) + // + # TODO(@Artturin): remove before release 23.05 and only have __spliced. + (lib.optionalAttrs (drv ? crossDrv && drv ? nativeDrv) { + crossDrv = overrideDerivation drv.crossDrv f; + nativeDrv = overrideDerivation drv.nativeDrv f; + }) + // + lib.optionalAttrs (drv ? __spliced) { + __spliced = {} // (lib.mapAttrs (_: sDrv: overrideDerivation sDrv f) drv.__spliced); + }); + + + /* `makeOverridable` takes a function from attribute set to attribute set and + injects `override` attribute which can be used to override arguments of + the function. + + nix-repl> x = {a, b}: { result = a + b; } + + nix-repl> y = lib.makeOverridable x { a = 1; b = 2; } + + nix-repl> y + { override = «lambda»; overrideDerivation = «lambda»; result = 3; } + + nix-repl> y.override { a = 10; } + { override = «lambda»; overrideDerivation = «lambda»; result = 12; } + + Please refer to "Nixpkgs Contributors Guide" section + ".overrideDerivation" to learn about `overrideDerivation` and caveats + related to its use. + */ + makeOverridable = f: origArgs: + let + result = f origArgs; + + # Creates a functor with the same arguments as f + copyArgs = g: lib.setFunctionArgs g (lib.functionArgs f); + # Changes the original arguments with (potentially a function that returns) a set of new attributes + overrideWith = newArgs: origArgs // (if lib.isFunction newArgs then newArgs origArgs else newArgs); + + # Re-call the function but with different arguments + overrideArgs = copyArgs (newArgs: makeOverridable f (overrideWith newArgs)); + # Change the result of the function call by applying g to it + overrideResult = g: makeOverridable (copyArgs (args: g (f args))) origArgs; + in + if builtins.isAttrs result then + result // { + override = overrideArgs; + overrideDerivation = fdrv: overrideResult (x: overrideDerivation x fdrv); + ${if result ? overrideAttrs then "overrideAttrs" else null} = fdrv: + overrideResult (x: x.overrideAttrs fdrv); + } + else if lib.isFunction result then + # Transform the result into a functor while propagating its arguments + lib.setFunctionArgs result (lib.functionArgs result) // { + override = overrideArgs; + } + else result; + + + /* Call the package function in the file `fn` with the required + arguments automatically. The function is called with the + arguments `args`, but any missing arguments are obtained from + `autoArgs`. This function is intended to be partially + parameterised, e.g., + + callPackage = callPackageWith pkgs; + pkgs = { + libfoo = callPackage ./foo.nix { }; + libbar = callPackage ./bar.nix { }; + }; + + If the `libbar` function expects an argument named `libfoo`, it is + automatically passed as an argument. Overrides or missing + arguments can be supplied in `args`, e.g. + + libbar = callPackage ./bar.nix { + libfoo = null; + enableX11 = true; + }; + */ + callPackageWith = autoArgs: fn: args: + let + f = if lib.isFunction fn then fn else import fn; + fargs = lib.functionArgs f; + + # All arguments that will be passed to the function + # This includes automatic ones and ones passed explicitly + allArgs = builtins.intersectAttrs fargs autoArgs // args; + + # A list of argument names that the function requires, but + # wouldn't be passed to it + missingArgs = lib.attrNames + # Filter out arguments that have a default value + (lib.filterAttrs (name: value: ! value) + # Filter out arguments that would be passed + (removeAttrs fargs (lib.attrNames allArgs))); + + # Get a list of suggested argument names for a given missing one + getSuggestions = arg: lib.pipe (autoArgs // args) [ + lib.attrNames + # Only use ones that are at most 2 edits away. While mork would work, + # levenshteinAtMost is only fast for 2 or less. + (lib.filter (lib.strings.levenshteinAtMost 2 arg)) + # Put strings with shorter distance first + (lib.sort (x: y: lib.strings.levenshtein x arg < lib.strings.levenshtein y arg)) + # Only take the first couple results + (lib.take 3) + # Quote all entries + (map (x: "\"" + x + "\"")) + ]; + + prettySuggestions = suggestions: + if suggestions == [] then "" + else if lib.length suggestions == 1 then ", did you mean ${lib.elemAt suggestions 0}?" + else ", did you mean ${lib.concatStringsSep ", " (lib.init suggestions)} or ${lib.last suggestions}?"; + + errorForArg = arg: + let + loc = builtins.unsafeGetAttrPos arg fargs; + # loc' can be removed once lib/minver.nix is >2.3.4, since that includes + # https://github.com/NixOS/nix/pull/3468 which makes loc be non-null + loc' = if loc != null then loc.file + ":" + toString loc.line + else if ! lib.isFunction fn then + toString fn + lib.optionalString (lib.sources.pathIsDirectory fn) "/default.nix" + else ""; + in "Function called without required argument \"${arg}\" at " + + "${loc'}${prettySuggestions (getSuggestions arg)}"; + + # Only show the error for the first missing argument + error = errorForArg (lib.head missingArgs); + + in if missingArgs == [] then makeOverridable f allArgs else abort error; + + + /* Like callPackage, but for a function that returns an attribute + set of derivations. The override function is added to the + individual attributes. */ + callPackagesWith = autoArgs: fn: args: + let + f = if lib.isFunction fn then fn else import fn; + auto = builtins.intersectAttrs (lib.functionArgs f) autoArgs; + origArgs = auto // args; + pkgs = f origArgs; + mkAttrOverridable = name: _: makeOverridable (newArgs: (f newArgs).${name}) origArgs; + in + if lib.isDerivation pkgs then throw + ("function `callPackages` was called on a *single* derivation " + + ''"${pkgs.name or ""}";'' + + " did you mean to use `callPackage` instead?") + else lib.mapAttrs mkAttrOverridable pkgs; + + + /* Add attributes to each output of a derivation without changing + the derivation itself and check a given condition when evaluating. */ + extendDerivation = condition: passthru: drv: + let + outputs = drv.outputs or [ "out" ]; + + commonAttrs = drv // (builtins.listToAttrs outputsList) // + ({ all = map (x: x.value) outputsList; }) // passthru; + + outputToAttrListElement = outputName: + { name = outputName; + value = commonAttrs // { + inherit (drv.${outputName}) type outputName; + outputSpecified = true; + drvPath = assert condition; drv.${outputName}.drvPath; + outPath = assert condition; drv.${outputName}.outPath; + } // + # TODO: give the derivation control over the outputs. + # `overrideAttrs` may not be the only attribute that needs + # updating when switching outputs. + lib.optionalAttrs (passthru?overrideAttrs) { + # TODO: also add overrideAttrs when overrideAttrs is not custom, e.g. when not splicing. + overrideAttrs = f: (passthru.overrideAttrs f).${outputName}; + }; + }; + + outputsList = map outputToAttrListElement outputs; + in commonAttrs // { + drvPath = assert condition; drv.drvPath; + outPath = assert condition; drv.outPath; + }; + + /* Strip a derivation of all non-essential attributes, returning + only those needed by hydra-eval-jobs. Also strictly evaluate the + result to ensure that there are no thunks kept alive to prevent + garbage collection. */ + hydraJob = drv: + let + outputs = drv.outputs or ["out"]; + + commonAttrs = + { inherit (drv) name system meta; inherit outputs; } + // lib.optionalAttrs (drv._hydraAggregate or false) { + _hydraAggregate = true; + constituents = map hydraJob (lib.flatten drv.constituents); + } + // (lib.listToAttrs outputsList); + + makeOutput = outputName: + let output = drv.${outputName}; in + { name = outputName; + value = commonAttrs // { + outPath = output.outPath; + drvPath = output.drvPath; + type = "derivation"; + inherit outputName; + }; + }; + + outputsList = map makeOutput outputs; + + drv' = (lib.head outputsList).value; + in if drv == null then null else + lib.deepSeq drv' drv'; + + /* Make a set of packages with a common scope. All packages called + with the provided `callPackage` will be evaluated with the same + arguments. Any package in the set may depend on any other. The + `overrideScope'` function allows subsequent modification of the package + set in a consistent way, i.e. all packages in the set will be + called with the overridden packages. The package sets may be + hierarchical: the packages in the set are called with the scope + provided by `newScope` and the set provides a `newScope` attribute + which can form the parent scope for later package sets. */ + makeScope = newScope: f: + let self = f self // { + newScope = scope: newScope (self // scope); + callPackage = self.newScope {}; + overrideScope = g: lib.warn + "`overrideScope` (from `lib.makeScope`) is deprecated. Do `overrideScope' (self: super: { … })` instead of `overrideScope (super: self: { … })`. All other overrides have the parameters in that order, including other definitions of `overrideScope`. This was the only definition violating the pattern." + (makeScope newScope (lib.fixedPoints.extends (lib.flip g) f)); + overrideScope' = g: makeScope newScope (lib.fixedPoints.extends g f); + packages = f; + }; + in self; + + /* Like the above, but aims to support cross compilation. It's still ugly, but + hopefully it helps a little bit. */ + makeScopeWithSplicing = splicePackages: newScope: otherSplices: keep: extra: f: + let + spliced0 = splicePackages { + pkgsBuildBuild = otherSplices.selfBuildBuild; + pkgsBuildHost = otherSplices.selfBuildHost; + pkgsBuildTarget = otherSplices.selfBuildTarget; + pkgsHostHost = otherSplices.selfHostHost; + pkgsHostTarget = self; # Not `otherSplices.selfHostTarget`; + pkgsTargetTarget = otherSplices.selfTargetTarget; + }; + spliced = extra spliced0 // spliced0 // keep self; + self = f self // { + newScope = scope: newScope (spliced // scope); + callPackage = newScope spliced; # == self.newScope {}; + # N.B. the other stages of the package set spliced in are *not* + # overridden. + overrideScope = g: makeScopeWithSplicing + splicePackages + newScope + otherSplices + keep + extra + (lib.fixedPoints.extends g f); + packages = f; + }; + in self; + +} diff --git a/debug.nix b/debug.nix new file mode 100644 index 000000000..a851cd747 --- /dev/null +++ b/debug.nix @@ -0,0 +1,253 @@ +/* Collection of functions useful for debugging + broken nix expressions. + + * `trace`-like functions take two values, print + the first to stderr and return the second. + * `traceVal`-like functions take one argument + which both printed and returned. + * `traceSeq`-like functions fully evaluate their + traced value before printing (not just to “weak + head normal form” like trace does by default). + * Functions that end in `-Fn` take an additional + function as their first argument, which is applied + to the traced value before it is printed. +*/ +{ lib }: +let + inherit (lib) + isInt + attrNames + isList + isAttrs + substring + addErrorContext + attrValues + concatLists + concatStringsSep + const + elem + generators + head + id + isDerivation + isFunction + mapAttrs + trace; +in + +rec { + + # -- TRACING -- + + /* Conditionally trace the supplied message, based on a predicate. + + Type: traceIf :: bool -> string -> a -> a + + Example: + traceIf true "hello" 3 + trace: hello + => 3 + */ + traceIf = + # Predicate to check + pred: + # Message that should be traced + msg: + # Value to return + x: if pred then trace msg x else x; + + /* Trace the supplied value after applying a function to it, and + return the original value. + + Type: traceValFn :: (a -> b) -> a -> a + + Example: + traceValFn (v: "mystring ${v}") "foo" + trace: mystring foo + => "foo" + */ + traceValFn = + # Function to apply + f: + # Value to trace and return + x: trace (f x) x; + + /* Trace the supplied value and return it. + + Type: traceVal :: a -> a + + Example: + traceVal 42 + # trace: 42 + => 42 + */ + traceVal = traceValFn id; + + /* `builtins.trace`, but the value is `builtins.deepSeq`ed first. + + Type: traceSeq :: a -> b -> b + + Example: + trace { a.b.c = 3; } null + trace: { a = ; } + => null + traceSeq { a.b.c = 3; } null + trace: { a = { b = { c = 3; }; }; } + => null + */ + traceSeq = + # The value to trace + x: + # The value to return + y: trace (builtins.deepSeq x x) y; + + /* Like `traceSeq`, but only evaluate down to depth n. + This is very useful because lots of `traceSeq` usages + lead to an infinite recursion. + + Example: + traceSeqN 2 { a.b.c = 3; } null + trace: { a = { b = {…}; }; } + => null + + Type: traceSeqN :: Int -> a -> b -> b + */ + traceSeqN = depth: x: y: + let snip = v: if isList v then noQuotes "[…]" v + else if isAttrs v then noQuotes "{…}" v + else v; + noQuotes = str: v: { __pretty = const str; val = v; }; + modify = n: fn: v: if (n == 0) then fn v + else if isList v then map (modify (n - 1) fn) v + else if isAttrs v then mapAttrs + (const (modify (n - 1) fn)) v + else v; + in trace (generators.toPretty { allowPrettyValues = true; } + (modify depth snip x)) y; + + /* A combination of `traceVal` and `traceSeq` that applies a + provided function to the value to be traced after `deepSeq`ing + it. + */ + traceValSeqFn = + # Function to apply + f: + # Value to trace + v: traceValFn f (builtins.deepSeq v v); + + /* A combination of `traceVal` and `traceSeq`. */ + traceValSeq = traceValSeqFn id; + + /* A combination of `traceVal` and `traceSeqN` that applies a + provided function to the value to be traced. */ + traceValSeqNFn = + # Function to apply + f: + depth: + # Value to trace + v: traceSeqN depth (f v) v; + + /* A combination of `traceVal` and `traceSeqN`. */ + traceValSeqN = traceValSeqNFn id; + + /* Trace the input and output of a function `f` named `name`, + both down to `depth`. + + This is useful for adding around a function call, + to see the before/after of values as they are transformed. + + Example: + traceFnSeqN 2 "id" (x: x) { a.b.c = 3; } + trace: { fn = "id"; from = { a.b = {…}; }; to = { a.b = {…}; }; } + => { a.b.c = 3; } + */ + traceFnSeqN = depth: name: f: v: + let res = f v; + in lib.traceSeqN + (depth + 1) + { + fn = name; + from = v; + to = res; + } + res; + + + # -- TESTING -- + + /* Evaluates a set of tests. + + A test is an attribute set `{expr, expected}`, + denoting an expression and its expected result. + + The result is a `list` of __failed tests__, each represented as + `{name, expected, result}`, + + - expected + - What was passed as `expected` + - result + - The actual `result` of the test + + Used for regression testing of the functions in lib; see + tests.nix for more examples. + + Important: Only attributes that start with `test` are executed. + + - If you want to run only a subset of the tests add the attribute `tests = ["testName"];` + + Example: + + runTests { + testAndOk = { + expr = lib.and true false; + expected = false; + }; + testAndFail = { + expr = lib.and true false; + expected = true; + }; + } + -> + [ + { + name = "testAndFail"; + expected = true; + result = false; + } + ] + + Type: + runTests :: { + tests = [ String ]; + ${testName} :: { + expr :: a; + expected :: a; + }; + } + -> + [ + { + name :: String; + expected :: a; + result :: a; + } + ] + */ + runTests = + # Tests to run + tests: concatLists (attrValues (mapAttrs (name: test: + let testsToRun = if tests ? tests then tests.tests else []; + in if (substring 0 4 name == "test" || elem name testsToRun) + && ((testsToRun == []) || elem name tests.tests) + && (test.expr != test.expected) + + then [ { inherit name; expected = test.expected; result = test.expr; } ] + else [] ) tests)); + + /* Create a test assuming that list elements are `true`. + + Example: + { testX = allTrue [ true ]; } + */ + testAllTrue = expr: { inherit expr; expected = map (x: true) expr; }; +} diff --git a/default.nix b/default.nix new file mode 100644 index 000000000..7f7f73168 --- /dev/null +++ b/default.nix @@ -0,0 +1,167 @@ +/* Library of low-level helper functions for nix expressions. + * + * Please implement (mostly) exhaustive unit tests + * for new functions in `./tests.nix`. + */ +let + + inherit (import ./fixed-points.nix { inherit lib; }) makeExtensible; + + lib = makeExtensible (self: let + callLibs = file: import file { lib = self; }; + in { + + # often used, or depending on very little + trivial = callLibs ./trivial.nix; + fixedPoints = callLibs ./fixed-points.nix; + + # datatypes + attrsets = callLibs ./attrsets.nix; + lists = callLibs ./lists.nix; + strings = callLibs ./strings.nix; + stringsWithDeps = callLibs ./strings-with-deps.nix; + + # packaging + customisation = callLibs ./customisation.nix; + derivations = callLibs ./derivations.nix; + maintainers = import ../maintainers/maintainer-list.nix; + teams = callLibs ../maintainers/team-list.nix; + meta = callLibs ./meta.nix; + versions = callLibs ./versions.nix; + + # module system + modules = callLibs ./modules.nix; + options = callLibs ./options.nix; + types = callLibs ./types.nix; + + # constants + licenses = callLibs ./licenses.nix; + sourceTypes = callLibs ./source-types.nix; + systems = callLibs ./systems; + + # serialization + cli = callLibs ./cli.nix; + generators = callLibs ./generators.nix; + + # misc + asserts = callLibs ./asserts.nix; + debug = callLibs ./debug.nix; + misc = callLibs ./deprecated.nix; + + # domain-specific + fetchers = callLibs ./fetchers.nix; + + # Eval-time filesystem handling + path = callLibs ./path; + filesystem = callLibs ./filesystem.nix; + fileset = callLibs ./fileset.nix; + sources = callLibs ./sources.nix; + + # back-compat aliases + platforms = self.systems.doubles; + + # linux kernel configuration + kernel = callLibs ./kernel.nix; + + inherit (builtins) add addErrorContext attrNames concatLists + deepSeq elem elemAt filter genericClosure genList getAttr + hasAttr head isAttrs isBool isInt isList isPath isString length + lessThan listToAttrs pathExists readFile replaceStrings seq + stringLength sub substring tail trace; + inherit (self.trivial) id const pipe concat or and bitAnd bitOr bitXor + bitNot boolToString mergeAttrs flip mapNullable inNixShell isFloat min max + importJSON importTOML warn warnIf warnIfNot throwIf throwIfNot checkListOfEnum + info showWarnings nixpkgsVersion version isInOldestRelease + mod compare splitByAndCompare + functionArgs setFunctionArgs isFunction toFunction + toHexString toBaseDigits inPureEvalMode; + inherit (self.fixedPoints) fix fix' converge extends composeExtensions + composeManyExtensions makeExtensible makeExtensibleWithCustomName; + inherit (self.attrsets) attrByPath hasAttrByPath setAttrByPath + getAttrFromPath attrVals attrValues getAttrs catAttrs filterAttrs + filterAttrsRecursive foldlAttrs foldAttrs collect nameValuePair mapAttrs + mapAttrs' mapAttrsToList concatMapAttrs mapAttrsRecursive mapAttrsRecursiveCond + genAttrs isDerivation toDerivation optionalAttrs + zipAttrsWithNames zipAttrsWith zipAttrs recursiveUpdateUntil + recursiveUpdate matchAttrs overrideExisting showAttrPath getOutput getBin + getLib getDev getMan chooseDevOutputs zipWithNames zip + recurseIntoAttrs dontRecurseIntoAttrs cartesianProductOfSets + updateManyAttrsByPath; + inherit (self.lists) singleton forEach foldr fold foldl foldl' imap0 imap1 + concatMap flatten remove findSingle findFirst any all count + optional optionals toList range replicate partition zipListsWith zipLists + reverseList listDfs toposort sort naturalSort compareLists take + drop sublist last init crossLists unique intersectLists + subtractLists mutuallyExclusive groupBy groupBy'; + inherit (self.strings) concatStrings concatMapStrings concatImapStrings + intersperse concatStringsSep concatMapStringsSep + concatImapStringsSep concatLines makeSearchPath makeSearchPathOutput + makeLibraryPath makeBinPath optionalString + hasInfix hasPrefix hasSuffix stringToCharacters stringAsChars escape + escapeShellArg escapeShellArgs + isStorePath isStringLike + isValidPosixName toShellVar toShellVars + escapeRegex escapeURL escapeXML replaceChars lowerChars + upperChars toLower toUpper addContextFrom splitString + removePrefix removeSuffix versionOlder versionAtLeast + getName getVersion + mesonOption mesonBool mesonEnable + nameFromURL enableFeature enableFeatureAs withFeature + withFeatureAs fixedWidthString fixedWidthNumber + toInt toIntBase10 readPathsFromFile fileContents; + inherit (self.stringsWithDeps) textClosureList textClosureMap + noDepEntry fullDepEntry packEntry stringAfter; + inherit (self.customisation) overrideDerivation makeOverridable + callPackageWith callPackagesWith extendDerivation hydraJob + makeScope makeScopeWithSplicing; + inherit (self.derivations) lazyDerivation; + inherit (self.meta) addMetaAttrs dontDistribute setName updateName + appendToName mapDerivationAttrset setPrio lowPrio lowPrioSet hiPrio + hiPrioSet getLicenseFromSpdxId getExe; + inherit (self.filesystem) pathType pathIsDirectory pathIsRegularFile; + inherit (self.sources) cleanSourceFilter + cleanSource sourceByRegex sourceFilesBySuffices + commitIdFromGitRepo cleanSourceWith pathHasContext + canCleanSource pathIsGitRepo; + inherit (self.modules) evalModules setDefaultModuleLocation + unifyModuleSyntax applyModuleArgsIfFunction mergeModules + mergeModules' mergeOptionDecls evalOptionValue mergeDefinitions + pushDownProperties dischargeProperties filterOverrides + sortProperties fixupOptionType mkIf mkAssert mkMerge mkOverride + mkOptionDefault mkDefault mkImageMediaOverride mkForce mkVMOverride + mkFixStrictness mkOrder mkBefore mkAfter mkAliasDefinitions + mkAliasAndWrapDefinitions fixMergeModules mkRemovedOptionModule + mkRenamedOptionModule mkRenamedOptionModuleWith + mkMergedOptionModule mkChangedOptionModule + mkAliasOptionModule mkDerivedConfig doRename + mkAliasOptionModuleMD; + inherit (self.options) isOption mkEnableOption mkSinkUndeclaredOptions + mergeDefaultOption mergeOneOption mergeEqualOption mergeUniqueOption + getValues getFiles + optionAttrSetToDocList optionAttrSetToDocList' + scrubOptionValue literalExpression literalExample literalDocBook + showOption showOptionWithDefLocs showFiles + unknownModule mkOption mkPackageOption mkPackageOptionMD + mdDoc literalMD; + inherit (self.types) isType setType defaultTypeMerge defaultFunctor + isOptionType mkOptionType; + inherit (self.asserts) + assertMsg assertOneOf; + inherit (self.debug) traceIf traceVal traceValFn + traceSeq traceSeqN traceValSeq + traceValSeqFn traceValSeqN traceValSeqNFn traceFnSeqN + runTests testAllTrue; + inherit (self.misc) maybeEnv defaultMergeArg defaultMerge foldArgs + maybeAttrNullable maybeAttr ifEnable checkFlag getValue + checkReqs uniqList uniqListExt condConcat lazyGenericClosure + innerModifySumArgs modifySumArgs innerClosePropagation + closePropagation mapAttrsFlatten nvs setAttr setAttrMerge + mergeAttrsWithFunc mergeAttrsConcatenateValues + mergeAttrsNoOverride mergeAttrByFunc mergeAttrsByFuncDefaults + mergeAttrsByFuncDefaultsClean mergeAttrBy + fakeHash fakeSha256 fakeSha512 + nixType imap; + inherit (self.versions) + splitVersion; + }); +in lib diff --git a/deprecated.nix b/deprecated.nix new file mode 100644 index 000000000..ed14e04bb --- /dev/null +++ b/deprecated.nix @@ -0,0 +1,307 @@ +{ lib }: +let + inherit (builtins) head tail isList isAttrs isInt attrNames; + +in + +with lib.lists; +with lib.attrsets; +with lib.strings; + +rec { + + # returns default if env var is not set + maybeEnv = name: default: + let value = builtins.getEnv name; in + if value == "" then default else value; + + defaultMergeArg = x : y: if builtins.isAttrs y then + y + else + (y x); + defaultMerge = x: y: x // (defaultMergeArg x y); + foldArgs = merger: f: init: x: + let arg = (merger init (defaultMergeArg init x)); + # now add the function with composed args already applied to the final attrs + base = (setAttrMerge "passthru" {} (f arg) + ( z: z // { + function = foldArgs merger f arg; + args = (lib.attrByPath ["passthru" "args"] {} z) // x; + } )); + withStdOverrides = base // { + override = base.passthru.function; + }; + in + withStdOverrides; + + + # shortcut for attrByPath ["name"] default attrs + maybeAttrNullable = maybeAttr; + + # shortcut for attrByPath ["name"] default attrs + maybeAttr = name: default: attrs: attrs.${name} or default; + + + # Return the second argument if the first one is true or the empty version + # of the second argument. + ifEnable = cond: val: + if cond then val + else if builtins.isList val then [] + else if builtins.isAttrs val then {} + # else if builtins.isString val then "" + else if val == true || val == false then false + else null; + + + # Return true only if there is an attribute and it is true. + checkFlag = attrSet: name: + if name == "true" then true else + if name == "false" then false else + if (elem name (attrByPath ["flags"] [] attrSet)) then true else + attrByPath [name] false attrSet ; + + + # Input : attrSet, [ [name default] ... ], name + # Output : its value or default. + getValue = attrSet: argList: name: + ( attrByPath [name] (if checkFlag attrSet name then true else + if argList == [] then null else + let x = builtins.head argList; in + if (head x) == name then + (head (tail x)) + else (getValue attrSet + (tail argList) name)) attrSet ); + + + # Input : attrSet, [[name default] ...], [ [flagname reqs..] ... ] + # Output : are reqs satisfied? It's asserted. + checkReqs = attrSet: argList: condList: + ( + foldr lib.and true + (map (x: let name = (head x); in + + ((checkFlag attrSet name) -> + (foldr lib.and true + (map (y: let val=(getValue attrSet argList y); in + (val!=null) && (val!=false)) + (tail x))))) condList)); + + + # This function has O(n^2) performance. + uniqList = { inputList, acc ? [] }: + let go = xs: acc: + if xs == [] + then [] + else let x = head xs; + y = if elem x acc then [] else [x]; + in y ++ go (tail xs) (y ++ acc); + in go inputList acc; + + uniqListExt = { inputList, + outputList ? [], + getter ? (x: x), + compare ? (x: y: x==y) }: + if inputList == [] then outputList else + let x = head inputList; + isX = y: (compare (getter y) (getter x)); + newOutputList = outputList ++ + (if any isX outputList then [] else [x]); + in uniqListExt { outputList = newOutputList; + inputList = (tail inputList); + inherit getter compare; + }; + + condConcat = name: list: checker: + if list == [] then name else + if checker (head list) then + condConcat + (name + (head (tail list))) + (tail (tail list)) + checker + else condConcat + name (tail (tail list)) checker; + + lazyGenericClosure = {startSet, operator}: + let + work = list: doneKeys: result: + if list == [] then + result + else + let x = head list; key = x.key; in + if elem key doneKeys then + work (tail list) doneKeys result + else + work (tail list ++ operator x) ([key] ++ doneKeys) ([x] ++ result); + in + work startSet [] []; + + innerModifySumArgs = f: x: a: b: if b == null then (f a b) // x else + innerModifySumArgs f x (a // b); + modifySumArgs = f: x: innerModifySumArgs f x {}; + + + innerClosePropagation = acc: xs: + if xs == [] + then acc + else let y = head xs; + ys = tail xs; + in if ! isAttrs y + then innerClosePropagation acc ys + else let acc' = [y] ++ acc; + in innerClosePropagation + acc' + (uniqList { inputList = (maybeAttrNullable "propagatedBuildInputs" [] y) + ++ (maybeAttrNullable "propagatedNativeBuildInputs" [] y) + ++ ys; + acc = acc'; + } + ); + + closePropagationSlow = list: (uniqList {inputList = (innerClosePropagation [] list);}); + + # This is an optimisation of lib.closePropagation which avoids the O(n^2) behavior + # Using a list of derivations, it generates the full closure of the propagatedXXXBuildInputs + # The ordering / sorting / comparison is done based on the `outPath` + # attribute of each derivation. + # On some benchmarks, it performs up to 15 times faster than lib.closePropagation. + # See https://github.com/NixOS/nixpkgs/pull/194391 for details. + closePropagationFast = list: + builtins.map (x: x.val) (builtins.genericClosure { + startSet = builtins.map (x: { + key = x.outPath; + val = x; + }) (builtins.filter (x: x != null) list); + operator = item: + if !builtins.isAttrs item.val then + [ ] + else + builtins.concatMap (x: + if x != null then [{ + key = x.outPath; + val = x; + }] else + [ ]) ((item.val.propagatedBuildInputs or [ ]) + ++ (item.val.propagatedNativeBuildInputs or [ ])); + }); + + closePropagation = if builtins ? genericClosure + then closePropagationFast + else closePropagationSlow; + + # calls a function (f attr value ) for each record item. returns a list + mapAttrsFlatten = f: r: map (attr: f attr r.${attr}) (attrNames r); + + # attribute set containing one attribute + nvs = name: value: listToAttrs [ (nameValuePair name value) ]; + # adds / replaces an attribute of an attribute set + setAttr = set: name: v: set // (nvs name v); + + # setAttrMerge (similar to mergeAttrsWithFunc but only merges the values of a particular name) + # setAttrMerge "a" [] { a = [2];} (x: x ++ [3]) -> { a = [2 3]; } + # setAttrMerge "a" [] { } (x: x ++ [3]) -> { a = [ 3]; } + setAttrMerge = name: default: attrs: f: + setAttr attrs name (f (maybeAttr name default attrs)); + + # Using f = a: b = b the result is similar to // + # merge attributes with custom function handling the case that the attribute + # exists in both sets + mergeAttrsWithFunc = f: set1: set2: + foldr (n: set: if set ? ${n} + then setAttr set n (f set.${n} set2.${n}) + else set ) + (set2 // set1) (attrNames set2); + + # merging two attribute set concatenating the values of same attribute names + # eg { a = 7; } { a = [ 2 3 ]; } becomes { a = [ 7 2 3 ]; } + mergeAttrsConcatenateValues = mergeAttrsWithFunc ( a: b: (toList a) ++ (toList b) ); + + # merges attributes using //, if a name exists in both attributes + # an error will be triggered unless its listed in mergeLists + # so you can mergeAttrsNoOverride { buildInputs = [a]; } { buildInputs = [a]; } {} to get + # { buildInputs = [a b]; } + # merging buildPhase doesn't really make sense. The cases will be rare where appending /prefixing will fit your needs? + # in these cases the first buildPhase will override the second one + # ! deprecated, use mergeAttrByFunc instead + mergeAttrsNoOverride = { mergeLists ? ["buildInputs" "propagatedBuildInputs"], + overrideSnd ? [ "buildPhase" ] + }: attrs1: attrs2: + foldr (n: set: + setAttr set n ( if set ? ${n} + then # merge + if elem n mergeLists # attribute contains list, merge them by concatenating + then attrs2.${n} ++ attrs1.${n} + else if elem n overrideSnd + then attrs1.${n} + else throw "error mergeAttrsNoOverride, attribute ${n} given in both attributes - no merge func defined" + else attrs2.${n} # add attribute not existing in attr1 + )) attrs1 (attrNames attrs2); + + + # example usage: + # mergeAttrByFunc { + # inherit mergeAttrBy; # defined below + # buildInputs = [ a b ]; + # } { + # buildInputs = [ c d ]; + # }; + # will result in + # { mergeAttrsBy = [...]; buildInputs = [ a b c d ]; } + # is used by defaultOverridableDelayableArgs and can be used when composing using + # foldArgs, composedArgsAndFun or applyAndFun. Example: composableDerivation in all-packages.nix + mergeAttrByFunc = x: y: + let + mergeAttrBy2 = { mergeAttrBy = lib.mergeAttrs; } + // (maybeAttr "mergeAttrBy" {} x) + // (maybeAttr "mergeAttrBy" {} y); in + foldr lib.mergeAttrs {} [ + x y + (mapAttrs ( a: v: # merge special names using given functions + if x ? ${a} + then if y ? ${a} + then v x.${a} y.${a} # both have attr, use merge func + else x.${a} # only x has attr + else y.${a} # only y has attr) + ) (removeAttrs mergeAttrBy2 + # don't merge attrs which are neither in x nor y + (filter (a: ! x ? ${a} && ! y ? ${a}) + (attrNames mergeAttrBy2)) + ) + ) + ]; + mergeAttrsByFuncDefaults = foldl mergeAttrByFunc { inherit mergeAttrBy; }; + mergeAttrsByFuncDefaultsClean = list: removeAttrs (mergeAttrsByFuncDefaults list) ["mergeAttrBy"]; + + # sane defaults (same name as attr name so that inherit can be used) + mergeAttrBy = # { buildInputs = concatList; [...]; passthru = mergeAttr; [..]; } + listToAttrs (map (n: nameValuePair n lib.concat) + [ "nativeBuildInputs" "buildInputs" "propagatedBuildInputs" "configureFlags" "prePhases" "postAll" "patches" ]) + // listToAttrs (map (n: nameValuePair n lib.mergeAttrs) [ "passthru" "meta" "cfg" "flags" ]) + // listToAttrs (map (n: nameValuePair n (a: b: "${a}\n${b}") ) [ "preConfigure" "postInstall" ]) + ; + + nixType = x: + if isAttrs x then + if x ? outPath then "derivation" + else "attrs" + else if lib.isFunction x then "function" + else if isList x then "list" + else if x == true then "bool" + else if x == false then "bool" + else if x == null then "null" + else if isInt x then "int" + else "string"; + + /* deprecated: + + For historical reasons, imap has an index starting at 1. + + But for consistency with the rest of the library we want an index + starting at zero. + */ + imap = imap1; + + # Fake hashes. Can be used as hash placeholders, when computing hash ahead isn't trivial + fakeHash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + fakeSha256 = "0000000000000000000000000000000000000000000000000000000000000000"; + fakeSha512 = "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"; +} diff --git a/derivations.nix b/derivations.nix new file mode 100644 index 000000000..5b7ed1868 --- /dev/null +++ b/derivations.nix @@ -0,0 +1,101 @@ +{ lib }: + +let + inherit (lib) throwIfNot; +in +{ + /* + Restrict a derivation to a predictable set of attribute names, so + that the returned attrset is not strict in the actual derivation, + saving a lot of computation when the derivation is non-trivial. + + This is useful in situations where a derivation might only be used for its + passthru attributes, improving evaluation performance. + + The returned attribute set is lazy in `derivation`. Specifically, this + means that the derivation will not be evaluated in at least the + situations below. + + For illustration and/or testing, we define derivation such that its + evaluation is very noticeable. + + let derivation = throw "This won't be evaluated."; + + In the following expressions, `derivation` will _not_ be evaluated: + + (lazyDerivation { inherit derivation; }).type + + attrNames (lazyDerivation { inherit derivation; }) + + (lazyDerivation { inherit derivation; } // { foo = true; }).foo + + (lazyDerivation { inherit derivation; meta.foo = true; }).meta + + In these expressions, `derivation` _will_ be evaluated: + + "${lazyDerivation { inherit derivation }}" + + (lazyDerivation { inherit derivation }).outPath + + (lazyDerivation { inherit derivation }).meta + + And the following expressions are not valid, because the refer to + implementation details and/or attributes that may not be present on + some derivations: + + (lazyDerivation { inherit derivation }).buildInputs + + (lazyDerivation { inherit derivation }).passthru + + (lazyDerivation { inherit derivation }).pythonPath + + */ + lazyDerivation = + args@{ + # The derivation to be wrapped. + derivation + , # Optional meta attribute. + # + # While this function is primarily about derivations, it can improve + # the `meta` package attribute, which is usually specified through + # `mkDerivation`. + meta ? null + , # Optional extra values to add to the returned attrset. + # + # This can be used for adding package attributes, such as `tests`. + passthru ? { } + }: + let + # These checks are strict in `drv` and some `drv` attributes, but the + # attrset spine returned by lazyDerivation does not depend on it. + # Instead, the individual derivation attributes do depend on it. + checked = + throwIfNot (derivation.type or null == "derivation") + "lazySimpleDerivation: input must be a derivation." + throwIfNot + (derivation.outputs == [ "out" ]) + # Supporting multiple outputs should be a matter of inheriting more attrs. + "The derivation ${derivation.name or ""} has multiple outputs. This is not supported by lazySimpleDerivation yet. Support could be added, and be useful as long as the set of outputs is known in advance, without evaluating the actual derivation." + derivation; + in + { + # Hardcoded `type` + # + # `lazyDerivation` requires its `derivation` argument to be a derivation, + # so if it is not, that is a programming error by the caller and not + # something that `lazyDerivation` consumers should be able to correct + # for after the fact. + # So, to improve laziness, we assume correctness here and check it only + # when actual derivation values are accessed later. + type = "derivation"; + + # A fixed set of derivation values, so that `lazyDerivation` can return + # its attrset before evaluating `derivation`. + # This must only list attributes that are available on _all_ derivations. + inherit (checked) outputs out outPath outputName drvPath name system; + + # The meta attribute can either be taken from the derivation, or if the + # `lazyDerivation` caller knew a shortcut, be taken from there. + meta = args.meta or checked.meta; + } // passthru; +} diff --git a/fetchers.nix b/fetchers.nix new file mode 100644 index 000000000..1107353b5 --- /dev/null +++ b/fetchers.nix @@ -0,0 +1,13 @@ +# snippets that can be shared by multiple fetchers (pkgs/build-support) +{ lib }: +{ + + proxyImpureEnvVars = [ + # We borrow these environment variables from the caller to allow + # easy proxy configuration. This is impure, but a fixed-output + # derivation like fetchurl is allowed to do so since its result is + # by definition pure. + "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy" + ]; + +} diff --git a/fileset.nix b/fileset.nix new file mode 100644 index 000000000..3eb909162 --- /dev/null +++ b/fileset.nix @@ -0,0 +1,993 @@ +# Add this to docs somewhere: If you pass the same arguments to the same functions, you will get the same result +{ lib }: +let + # TODO: Add builtins.traceVerbose or so to all the operations for debugging + # TODO: Document limitation that empty directories won't be included + # TODO: Point out that equality comparison using `==` doesn't quite work because there's multiple representations for all files in a directory: "directory" and `readDir path`. + # TODO: subset and superset check functions. Can easily be implemented with `difference` and `isEmpty` + # TODO: Write down complexity of each operation, most should be O(1)! + # TODO: Implement an operation for optionally including a file if it exists. + # TODO: Derive property tests from https://en.wikipedia.org/wiki/Algebra_of_sets + + inherit (builtins) + isAttrs + isString + isPath + isList + typeOf + readDir + match + pathExists + seq + ; + + inherit (lib.trivial) + mapNullable + ; + + inherit (lib.lists) + head + tail + foldl' + range + length + elemAt + all + imap0 + drop + commonPrefix + ; + + inherit (lib.strings) + isCoercibleToString + ; + + inherit (lib.attrsets) + mapAttrs + attrNames + attrValues + ; + + inherit (lib.filesystem) + pathType + ; + + inherit (lib.sources) + cleanSourceWith + ; + + inherit (lib.path) + append + deconstruct + construct + hasPrefix + removePrefix + ; + + inherit (lib.path.components) + fromSubpath + ; + + # Internal file set structure: + # + # # A set of files + # = { + # _type = "fileset"; + # + # # The base path, only files under this path can be represented + # # Always a directory + # _base = ; + # + # # A tree representation of all included files + # _tree = ; + # }; + # + # # A directory entry value + # = + # # A nested directory + # + # + # # A nested file + # | + # + # # A removed file or directory + # # This is represented like this instead of removing the entry from the attribute set because: + # # - It improves laziness + # # - It allows keeping the attribute set as a `builtins.readDir` cache + # | null + # + # # A directory + # = + # # The inclusion state for every directory entry + # { = ; } + # + # # All files in a directory, recursively. + # # Semantically this is equivalent to `builtins.readDir path`, but lazier, because + # # operations that don't require the entry listing can avoid it. + # # This string is chosen to be compatible with `builtins.readDir` for a simpler implementation + # "directory"; + # + # # A file + # = + # # A file with this filetype + # # These strings match `builtins.readDir` for a simpler implementation + # "regular" | "symlink" | "unknown" + + # Create a fileset structure + # Type: Path -> -> + _create = base: tree: { + _type = "fileset"; + # All attributes are internal + _base = base; + _tree = tree; + # Double __ to make it be evaluated and ordered first + __noEval = throw '' + File sets are not intended to be directly inspected or evaluated. Instead prefer: + - If you want to print a file set, use the `lib.fileset.trace` or `lib.fileset.pretty` function. + - If you want to check file sets for equality, use the `lib.fileset.equals` function. + ''; + }; + + # Create a file set from a path + # Type: Path -> + _singleton = path: + let + type = pathType path; + in + if type == "directory" then + _create path type + else + # Always coerce to a directory + # If we don't do this we run into problems like: + # - What should `toSource { base = ./default.nix; fileset = difference ./default.nix ./default.nix; }` do? + # - Importing an empty directory wouldn't make much sense because our `base` is a file + # - Neither can we create a store path containing nothing at all + # - The only option is to throw an error that `base` should be a directory + # - Should `fileFilter (file: file.name == "default.nix") ./default.nix` run the predicate on the ./default.nix file? + # - If no, should the result include or exclude ./default.nix? In any case, it would be confusing and inconsistent + # - If yes, it needs to consider ./. to have influence the filesystem result, because file names are part of the parent directory, so filter would change the necessary base + _create (dirOf path) + (_nestTree + (dirOf path) + [ (baseNameOf path) ] + type + ); + + # Turn a builtins.filterSource-based source filter on a root path into a file set containing only files included by the filter + # Type: Path -> (String -> String -> Bool) -> + _fromSource = root: filter: + let + recurse = focusPath: type: + # FIXME: Generally we shouldn't use toString on paths, though it might be correct + # here since we're trying to mimic the impure behavior of `builtins.filterPath` + if ! filter (toString focusPath) type then + null + else if type == "directory" then + mapAttrs + (name: recurse (append focusPath name)) + (readDir focusPath) + else + type; + + + rootPathType = pathType root; + tree = + if rootPathType == "directory" then + recurse root rootPathType + else + rootPathType; + in + _create root tree; + + # Coerce a value to a fileset + # Type: String -> String -> Any -> + _coerce = function: context: value: + if value._type or "" == "fileset" then + value + else if ! isPath value then + if value._isLibCleanSourceWith or false then + throw '' + lib.fileset.${function}: Expected ${context} to be a path, but it's a value produced by `lib.sources` instead. + Such a value is only supported when converted to a file set using `lib.fileset.fromSource`.'' + else if isCoercibleToString value then + throw '' + lib.fileset.${function}: Expected ${context} to be a path, but it's a string-coercible value instead, possibly a Nix store path. + Such a value is not supported, `lib.fileset` only supports local file filtering.'' + else + throw "lib.fileset.${function}: Expected ${context} to be a path, but got a ${typeOf value}." + else if ! pathExists value then + throw "lib.fileset.${function}: Expected ${context} \"${toString value}\" to be a path that exists, but it doesn't." + else + _singleton value; + + # Nest a tree under some further components + # Type: Path -> [ String ] -> -> + _nestTree = targetBase: extraComponents: tree: + let + recurse = index: focusPath: + if index == length extraComponents then + tree + else + let + focusedName = elemAt extraComponents index; + in + mapAttrs + (name: _: + if name == focusedName then + recurse (index + 1) (append focusPath name) + else + null + ) + (readDir focusPath); + in + recurse 0 targetBase; + + # Expand "directory" to { = ; } + # Type: Path -> -> { = ; } + _directoryEntries = path: value: + if isAttrs value then + value + else + readDir path; + + # The following tables are a bit complicated, but they nicely explain the + # corresponding implementations, here's the legend: + # + # lhs\rhs: The values for the left hand side and right hand side arguments + # null: null, an excluded file/directory + # attrs: satisfies `isAttrs value`, an explicitly listed directory containing nested trees + # dir: "directory", a recursively included directory + # str: "regular", "symlink" or "unknown", a filetype string + # rec: A result computed by recursing + # -: Can't occur because one argument is a directory while the other is a file + # : Indicates that the result is computed by the branch with that number + + # The union of two 's + # Type: -> -> + # + # lhs\rhs | null | attrs | dir | str | + # ------- | ------- | ------- | ----- | ----- | + # null | 2 null | 2 attrs | 2 dir | 2 str | + # attrs | 3 attrs | 1 rec | 2 dir | - | + # dir | 3 dir | 3 dir | 2 dir | - | + # str | 3 str | - | - | 2 str | + _unionTree = lhs: rhs: + # Branch 1 + if isAttrs lhs && isAttrs rhs then + mapAttrs (name: _unionTree lhs.${name}) rhs + # Branch 2 + else if lhs == null || isString rhs then + rhs + # Branch 3 + else + lhs; + + # The intersection of two 's + # Type: -> -> + # + # lhs\rhs | null | attrs | dir | str | + # ------- | ------- | ------- | ------- | ------ | + # null | 2 null | 2 null | 2 null | 2 null | + # attrs | 3 null | 1 rec | 2 attrs | - | + # dir | 3 null | 3 attrs | 2 dir | - | + # str | 3 null | - | - | 2 str | + _intersectTree = lhs: rhs: + # Branch 1 + if isAttrs lhs && isAttrs rhs then + mapAttrs (name: _intersectTree lhs.${name}) rhs + # Branch 2 + else if lhs == null || isString rhs then + lhs + # Branch 3 + else + rhs; + + # The difference between two 's + # Type: Path -> -> -> + # + # lhs\rhs | null | attrs | dir | str | + # ------- | ------- | ------- | ------ | ------ | + # null | 1 null | 1 null | 1 null | 1 null | + # attrs | 2 attrs | 3 rec | 1 null | - | + # dir | 2 dir | 3 rec | 1 null | - | + # str | 2 str | - | - | 1 null | + _differenceTree = path: lhs: rhs: + # Branch 1 + if isString rhs || lhs == null then + null + # Branch 2 + else if rhs == null then + lhs + # Branch 3 + else + mapAttrs (name: lhsValue: + _differenceTree (append path name) lhsValue rhs.${name} + ) (_directoryEntries path lhs); + + # Whether two 's are equal + # Type: Path -> -> -> + # + # | lhs\rhs | null | attrs | dir | str | + # | ------- | ------- | ------- | ------ | ------- | + # | null | 1 true | 1 rec | 1 rec | 1 false | + # | attrs | 2 rec | 3 rec | 3 rec | - | + # | dir | 2 rec | 3 rec | 4 true | - | + # | str | 2 false | - | - | 4 true | + _equalsTree = path: lhs: rhs: + # Branch 1 + if lhs == null then + _isEmptyTree path rhs + # Branch 2 + else if rhs == null then + _isEmptyTree path lhs + # Branch 3 + else if isAttrs lhs || isAttrs rhs then + let + lhs' = _directoryEntries path lhs; + rhs' = _directoryEntries path rhs; + in + all (name: + _equalsTree (append path name) lhs'.${name} rhs'.${name} + ) (attrNames lhs') + # Branch 4 + else + true; + + # Whether a tree is empty, containing no files + # Type: Path -> -> Bool + _isEmptyTree = path: tree: + if isAttrs tree || tree == "directory" then + let + entries = _directoryEntries path tree; + in + all (name: _isEmptyTree (append path name) entries.${name}) (attrNames entries) + else + tree == null; + + # Simplifies a tree, optionally expanding all "directory"'s into complete listings + # Type: Bool -> Path -> -> + _simplifyTree = expand: base: tree: + let + recurse = focusPath: tree: + if tree == "directory" && expand || isAttrs tree then + let + expanded = _directoryEntries focusPath tree; + transformedSubtrees = mapAttrs (name: recurse (append focusPath name)) expanded; + values = attrValues transformedSubtrees; + in + if all (value: value == "emptyDir") values then + "emptyDir" + else if all (value: isNull value || value == "emptyDir") values then + null + else if !expand && all (value: isString value || value == "emptyDir") values then + "directory" + else + mapAttrs (name: value: if value == "emptyDir" then null else value) transformedSubtrees + else + tree; + result = recurse base tree; + in + if result == "emptyDir" then + null + else + result; + + _prettyTreeSuffix = tree: + if isAttrs tree then + "" + else if tree == "directory" then + " (recursive directory)" + else + " (${tree})"; + + # Pretty-print all files included in the file set. + # Type: (b -> String -> b) -> b -> Path -> FileSet -> b + _prettyFoldl' = f: start: base: tree: + let + traceTreeAttrs = start: indent: tree: + # Nix should really be evaluating foldl''s second argument before starting the iteration + # See the same problem in Haskell: + # - https://stackoverflow.com/a/14282642 + # - https://gitlab.haskell.org/ghc/ghc/-/issues/12173 + # - https://well-typed.com/blog/90/#a-postscript-which-foldl + # - https://old.reddit.com/r/haskell/comments/21wvk7/foldl_is_broken/ + seq start + (foldl' (prev: name: + let + subtree = tree.${name}; + + intermediate = + f prev "${indent}- ${name}${_prettyTreeSuffix subtree}"; + in + if subtree == null then + # Don't print anything at all if this subtree is empty + prev + else if isAttrs subtree then + # A directory with explicit entries + # Do print this node, but also recurse + traceTreeAttrs intermediate "${indent} " subtree + else + # Either a file, or a recursively included directory + # Do print this node but no further recursion needed + intermediate + ) start (attrNames tree)); + + intermediate = + if tree == null then + f start "${toString base} (empty)" + else + f start "${toString base}${_prettyTreeSuffix tree}"; + in + if isAttrs tree then + traceTreeAttrs intermediate "" tree + else + intermediate; + + # Coerce and normalise the bases of multiple file set values passed to user-facing functions + # Type: String -> [ { context :: String, value :: Any } ] -> { commonBase :: Path, trees :: [ ] } + _normaliseBase = function: list: + let + processed = map ({ context, value }: + let + fileset = _coerce function context value; + in { + inherit fileset context; + baseParts = deconstruct fileset._base; + } + ) list; + + first = head processed; + + commonComponents = foldl' (components: el: + if first.baseParts.root != el.baseParts.root then + throw "lib.fileset.${function}: Expected file sets to have the same filesystem root, but ${first.context} has root \"${toString first.baseParts.root}\" while ${el.context} has root \"${toString el.baseParts.root}\"." + else + commonPrefix components el.baseParts.components + ) first.baseParts.components (tail processed); + + commonBase = construct { + root = first.baseParts.root; + components = commonComponents; + }; + + commonComponentsLength = length commonComponents; + + trees = map (value: + _nestTree + commonBase + (drop commonComponentsLength value.baseParts.components) + value.fileset._tree + ) processed; + in + { + inherit commonBase trees; + }; + +in { + + /* + Import a file set into the Nix store, making it usable inside derivations. + Return a source-like value that can be coerced to a Nix store path. + + This function takes an attribute set with these attributes as an argument: + + - `root` (required): The local path that should be the root of the result. + `fileset` must not be influenceable by paths outside `root`, meaning `lib.fileset.getInfluenceBase fileset` must be under `root`. + + Warning: Setting `root` to `lib.fileset.getInfluenceBase fileset` directly would make the resulting Nix store path file structure dependent on how `fileset` is declared. + This makes it non-trivial to predict where specific paths are located in the result. + + - `fileset` (required): The set of files to import into the Nix store. + Use the other `lib.fileset` functions to define `fileset`. + Only directories containing at least one file are included in the result, unless `extraExistingDirs` is used to ensure the existence of specific directories even without any files. + + - `extraExistingDirs` (optional, default `[]`): Additionally ensure the existence of these directory paths in the result, even they don't contain any files in `fileset`. + + Type: + toSource :: { + root :: Path, + fileset :: FileSet, + extraExistingDirs :: [ Path ] ? [ ], + } -> SourceLike + */ + toSource = { root, fileset, extraExistingDirs ? [ ] }: + let + maybeFileset = fileset; + in + let + fileset = _coerce "toSource" "`fileset` attribute" maybeFileset; + + # Directories that recursively have no files in them will always be `null` + sparseTree = + let + recurse = focusPath: tree: + if tree == "directory" || isAttrs tree then + let + entries = _directoryEntries focusPath tree; + sparseSubtrees = mapAttrs (name: recurse (append focusPath name)) entries; + values = attrValues sparseSubtrees; + in + if all isNull values then + null + else if all isString values then + "directory" + else + sparseSubtrees + else + tree; + resultingTree = recurse fileset._base fileset._tree; + # The fileset's _base might be below the root of the `toSource`, so we need to lift the tree up to `root` + extraRootNesting = removePrefix root fileset._base; + in _nestTree root extraRootNesting resultingTree; + + sparseExtendedTree = + if ! isList extraExistingDirs then + throw "lib.fileset.toSource: Expected the `extraExistingDirs` attribute to be a list, but it's a ${typeOf extraExistingDirs} instead." + else + lib.foldl' (tree: i: + let + dir = elemAt extraExistingDirs i; + + # We're slightly abusing the internal functions and structure to ensure that the extra directory is represented in the sparse tree. + value = mapAttrs (name: value: null) (readDir dir); + extraTree = _nestTree root (removePrefix root dir) value; + result = _unionTree tree extraTree; + in + if ! isPath dir then + throw "lib.fileset.toSource: Expected all elements of the `extraExistingDirs` attribute to be paths, but element at index ${toString i} is a ${typeOf dir} instead." + else if ! pathExists dir then + throw "lib.fileset.toSource: Expected all elements of the `extraExistingDirs` attribute to be paths that exist, but the path at index ${toString i} \"${toString dir}\" does not." + else if pathType dir != "directory" then + throw "lib.fileset.toSource: Expected all elements of the `extraExistingDirs` attribute to be paths pointing to directories, but the path at index ${toString i} \"${toString dir}\" points to a file instead." + else if ! hasPrefix root dir then + throw "lib.fileset.toSource: Expected all elements of the `extraExistingDirs` attribute to be paths under the `root` attribute \"${toString root}\", but the path at index ${toString i} \"${toString dir}\" is not." + else + result + ) sparseTree (range 0 (length extraExistingDirs - 1)); + + rootComponentsLength = length (deconstruct root).components; + + # This function is called often for the filter, so it should be fast + inSet = components: + let + recurse = index: localTree: + if index == length components then + localTree != null + else if localTree ? ${elemAt components index} then + recurse (index + 1) localTree.${elemAt components index} + else + localTree == "directory"; + in recurse rootComponentsLength sparseExtendedTree; + + in + if ! isPath root then + if root._isLibCleanSourceWith or false then + throw '' + lib.fileset.toSource: Expected attribute `root` to be a path, but it's a value produced by `lib.sources` instead. + Such a value is only supported when converted to a file set using `lib.fileset.fromSource` and passed to the `fileset` attribute, where it may also be combined using other functions from `lib.fileset`.'' + else if isCoercibleToString root then + throw '' + lib.fileset.toSource: Expected attribute `root` to be a path, but it's a string-like value instead, possibly a Nix store path. + Such a value is not supported, `lib.fileset` only supports local file filtering.'' + else + throw "lib.fileset.toSource: Expected attribute `root` to be a path, but it's a ${typeOf root} instead." + else if ! pathExists root then + throw "lib.fileset.toSource: Expected attribute `root` \"${toString root}\" to be a path that exists, but it doesn't." + else if pathType root != "directory" then + throw "lib.fileset.toSource: Expected attribute `root` \"${toString root}\" to be a path pointing to a directory, but it's pointing to a file instead." + else if ! hasPrefix root fileset._base then + throw "lib.fileset.toSource: Expected attribute `fileset` to not be influenceable by any paths outside `root`, but `lib.fileset.getInfluenceBase fileset` \"${toString fileset._base}\" is outside `root`." + else + cleanSourceWith { + name = "source"; + src = root; + filter = pathString: _: inSet (fromSubpath "./${pathString}"); + }; + + /* + Create a file set from a filtered local source as produced by the `lib.sources` functions. + This does not import anything into the store. + + Type: + fromSource :: SourceLike -> FileSet + + Example: + fromSource (lib.sources.cleanSource ./.) + */ + fromSource = source: + if ! source._isLibCleanSourceWith or false || ! source ? origSrc || ! source ? filter then + throw "lib.fileset.fromSource: Expected the argument to be a value produced from `lib.sources`, but got a ${typeOf source} instead." + else if ! isPath source.origSrc then + throw "lib.fileset.fromSource: Expected the argument to be source-like value of a local path." + else + _fromSource source.origSrc source.filter; + + /* + Coerce a value to a file set: + + - If the value is a file set already, return it directly + + - If the value is a path pointing to a file, return a file set with that single file + + - If the value is a path pointing to a directory, return a file set with all files contained in that directory + + This function is mostly not needed because all functions in `lib.fileset` will implicitly apply it for arguments that are expected to be a file set. + + Type: + coerce :: Any -> FileSet + */ + coerce = value: _coerce "coerce" "argument" value; + + + /* + Create a file set containing all files contained in a path (see `coerce`), or no files if the path doesn't exist. + + This is useful when you want to include a file only if it actually exists. + + Type: + optional :: Path -> FileSet + */ + optional = path: + if ! isPath path then + throw "lib.fileset.optional: Expected argument to be a path, but got a ${typeOf path}." + else if pathExists path then + _singleton path + else + _create path null; + + /* + Return the common ancestor directory of all file set operations used to construct this file set, meaning that nothing outside the this directory can influence the set of files included. + + Type: + getInfluenceBase :: FileSet -> Path + + Example: + getInfluenceBase ./Makefile + => ./. + + getInfluenceBase ./src + => ./src + + getInfluenceBase (fileFilter (file: false) ./.) + => ./. + + getInfluenceBase (union ./Makefile ../default.nix) + => ../. + */ + getInfluenceBase = maybeFileset: + let + fileset = _coerce "getInfluenceBase" "argument" maybeFileset; + in + fileset._base; + + /* + Incrementally evaluate and trace a file set in a pretty way. + Functionally this is the same as splitting the result from `lib.fileset.pretty` into lines and tracing those. + However this function can do the same thing incrementally, so it can already start printing the result as the first lines are known. + + The `expand` argument (false by default) controls whether all files should be printed individually. + + Type: + trace :: { expand :: Bool ? false } -> FileSet -> Any -> Any + + Example: + trace {} (unions [ ./foo.nix ./bar/baz.c ./qux ]) + => + trace: /home/user/src/myProject + trace: - bar + trace: - baz.c (regular) + trace: - foo.nix (regular) + trace: - qux (recursive directory) + null + + trace { expand = true; } (unions [ ./foo.nix ./bar/baz.c ./qux ]) + => + trace: /home/user/src/myProject + trace: - bar + trace: - baz.c (regular) + trace: - foo.nix (regular) + trace: - qux + trace: - florp.c (regular) + trace: - florp.h (regular) + null + */ + trace = { expand ? false }: maybeFileset: + let + fileset = _coerce "trace" "second argument" maybeFileset; + simpleTree = _simplifyTree expand fileset._base fileset._tree; + in + _prettyFoldl' (acc: el: builtins.trace el acc) (x: x) + fileset._base + simpleTree; + + /* + The same as `lib.fileset.trace`, but instead of taking an argument for the value to return, the given file set is returned instead. + + Type: + traceVal :: { expand :: Bool ? false } -> FileSet -> FileSet + */ + traceVal = { expand ? false }: maybeFileset: + let + fileset = _coerce "traceVal" "second argument" maybeFileset; + simpleTree = _simplifyTree expand fileset._base fileset._tree; + in + _prettyFoldl' (acc: el: builtins.trace el acc) fileset + fileset._base + simpleTree; + + /* + The same as `lib.fileset.trace`, but instead of tracing each line, the result is returned as a string. + + Type: + pretty :: { expand :: Bool ? false } -> FileSet -> String + */ + pretty = { expand ? false }: maybeFileset: + let + fileset = _coerce "pretty" "second argument" maybeFileset; + simpleTree = _simplifyTree expand fileset._base fileset._tree; + in + _prettyFoldl' (acc: el: "${acc}\n${el}") "" + fileset._base + simpleTree; + + /* + The file set containing all files that are in either of two given file sets. + Recursively, the first argument is evaluated first, only evaluating the second argument if necessary. + + union a b = a ⋃ b + + Type: + union :: FileSet -> FileSet -> FileSet + */ + union = lhs: rhs: + let + normalised = _normaliseBase "union" [ + { + context = "first argument"; + value = lhs; + } + { + context = "second argument"; + value = rhs; + } + ]; + in + _create normalised.commonBase + (_unionTree + (elemAt normalised.trees 0) + (elemAt normalised.trees 1) + ); + + /* + The file containing all files from that are in any of the given file sets. + Recursively, the elements are evaluated from left to right, only evaluating arguments on the right if necessary. + + Type: + unions :: [FileSet] -> FileSet + */ + unions = list: + let + annotated = imap0 (i: el: { + context = "element ${toString i} of the argument"; + value = el; + }) list; + + normalised = _normaliseBase "unions" annotated; + + tree = foldl' _unionTree (head normalised.trees) (tail normalised.trees); + in + if ! isList list then + throw "lib.fileset.unions: Expected argument to be a list, but got a ${typeOf list}." + else if length list == 0 then + throw "lib.fileset.unions: Expected argument to be a list with at least one element, but it contains no elements." + else + _create normalised.commonBase tree; + + /* + The file set containing all files that are in both given sets. + Recursively, the first argument is evaluated first, only evaluating the second argument if necessary. + + intersect a b == a ⋂ b + + Type: + intersect :: FileSet -> FileSet -> FileSet + */ + intersect = lhs: rhs: + let + normalised = _normaliseBase "intersect" [ + { + context = "first argument"; + value = lhs; + } + { + context = "second argument"; + value = rhs; + } + ]; + in + _create normalised.commonBase + (_intersectTree + (elemAt normalised.trees 0) + (elemAt normalised.trees 1) + ); + + /* + The file set containing all files that are in all the given sets. + Recursively, the elements are evaluated from left to right, only evaluating arguments on the right if necessary. + + Type: + intersects :: [FileSet] -> FileSet + */ + intersects = list: + let + annotated = imap0 (i: el: { + context = "element ${toString i} of the argument"; + value = el; + }) list; + + normalised = _normaliseBase "intersects" annotated; + + tree = foldl' _intersectTree (head normalised.trees) (tail normalised.trees); + in + if ! isList list then + throw "lib.fileset.intersects: Expected argument to be a list, but got a ${typeOf list}." + else if length list == 0 then + throw "lib.fileset.intersects: Expected argument to be a list with at least one element, but it contains no elements." + else + _create normalised.commonBase tree; + + /* + The file set containing all files that are in the first file set but not in the second. + Recursively, the second argument is evaluated first, only evaluating the first argument if necessary. + + difference a b == a ∖ b + + Type: + difference :: FileSet -> FileSet -> FileSet + */ + difference = lhs: rhs: + let + normalised = _normaliseBase "difference" [ + { + context = "first argument"; + value = lhs; + } + { + context = "second argument"; + value = rhs; + } + ]; + in + _create normalised.commonBase + (_differenceTree normalised.commonBase + (elemAt normalised.trees 0) + (elemAt normalised.trees 1) + ); + + /* + Filter a file set to only contain files matching some predicate. + + The predicate is called with an attribute set containing these attributes: + + - `name`: The filename + + - `type`: The type of the file, either "regular", "symlink" or "unknown" + + - `ext`: The file extension or `null` if the file has none. + + More formally: + - `ext` contains no `.` + - `.${ext}` is a suffix of the `name` + + - Potentially other attributes in the future + + Type: + fileFilter :: + ({ + name :: String, + type :: String, + ext :: String | Null, + ... + } -> Bool) + -> FileSet + -> FileSet + */ + fileFilter = predicate: maybeFileset: + let + fileset = _coerce "fileFilter" "second argument" maybeFileset; + recurse = focusPath: tree: + mapAttrs (name: subtree: + if isAttrs subtree || subtree == "directory" then + recurse (append focusPath name) subtree + else if + predicate { + inherit name; + type = subtree; + ext = mapNullable head (match ".*\\.(.*)" name); + # To ensure forwards compatibility with more arguments being added in the future, + # adding an attribute which can't be deconstructed :) + "This attribute is passed to prevent `lib.fileset.fileFilter` predicate functions from breaking when more attributes are added in the future. Please add `...` to the function to handle this and future additional arguments." = null; + } + then + subtree + else + null + ) (_directoryEntries focusPath tree); + in + _create fileset._base (recurse fileset._base fileset._tree); + + /* + A file set containing all files that are contained in a directory whose name satisfies the given predicate. + Only directories under the given path are checked, this is to ensure that components outside of the given path cannot influence the result. + Consequently this function does not accept a file set as an argument. + If you need to filter files in a file set based on components, use `intersect myFileSet (directoryFilter myPredicate myPath)` instead. + + Type: + directoryFilter :: (String -> Bool) -> Path -> FileSet + + Example: + # Select all files in hidden directories within ./. + directoryFilter (hasPrefix ".") ./. + + # Select all files in directories named `build` within ./src + directoryFilter (name: name == "build") ./src + */ + directoryFilter = predicate: path: + let + recurse = focusPath: + mapAttrs (name: type: + if type == "directory" then + if predicate name then + type + else + recurse (append focusPath name) + else + null + ) (readDir focusPath); + in + if path._type or null == "fileset" then + throw '' + lib.fileset.directoryFilter: Expected second argument to be a path, but it's a file set. + If you need to filter files in a file set, use `intersect myFileSet (directoryFilter myPredicate myPath)` instead.'' + else if ! isPath path then + throw "lib.fileset.directoryFilter: Expected second argument to be a path, but got a ${typeOf path}." + else if pathType path != "directory" then + throw "lib.fileset.directoryFilter: Expected second argument \"${toString path}\" to be a directory, but it's not." + else + _create path (recurse path); + + /* + Check whether two file sets contain the same files. + + Type: + equals :: FileSet -> FileSet -> Bool + */ + equals = lhs: rhs: + let + normalised = _normaliseBase "equals" [ + { + context = "first argument"; + value = lhs; + } + { + context = "second argument"; + value = rhs; + } + ]; + in + _equalsTree normalised.commonBase + (elemAt normalised.trees 0) + (elemAt normalised.trees 1); + + /* + Check whether a file set contains no files. + + Type: + isEmpty :: FileSet -> Bool + */ + isEmpty = maybeFileset: + let + fileset = _coerce "isEmpty" "argument" maybeFileset; + in + _isEmptyTree fileset._base fileset._tree; +} diff --git a/filesystem.nix b/filesystem.nix new file mode 100644 index 000000000..5d048c0a1 --- /dev/null +++ b/filesystem.nix @@ -0,0 +1,194 @@ +# Functions for querying information about the filesystem +# without copying any files to the Nix store. +{ lib }: + +# Tested in lib/tests/filesystem.sh +let + inherit (builtins) + readDir + pathExists + storeDir + ; + + inherit (lib.trivial) + inPureEvalMode + ; + + inherit (lib.path) + deconstruct + ; + + inherit (lib.path.components) + fromSubpath + ; + + inherit (lib.lists) + take + length + ; + + inherit (lib.filesystem) + pathType + ; +in + +{ + + /* + The type of a path. The path needs to exist and be accessible. + The result is either "directory" for a directory, "regular" for a regular file, "symlink" for a symlink, or "unknown" for anything else. + + Type: + pathType :: Path -> String + + Example: + pathType /. + => "directory" + + pathType /some/file.nix + => "regular" + */ + pathType = + let + storeDirComponents = fromSubpath "./${storeDir}"; + + legacy = path: + if ! pathExists path + # Fail irrecoverably to mimic the historic behavior of this function and + # the new builtins.readFileType + then abort "lib.filesystem.pathType: Path ${toString path} does not exist." + # The filesystem root is the only path where `dirOf / == /` and + # `baseNameOf /` is not valid. We can detect this and directly return + # "directory", since we know the filesystem root can't be anything else. + else if dirOf path == path + then "directory" + else (readDir (dirOf path)).${baseNameOf path}; + + legacyPureEval = path: + let + # This is a workaround for the fact that `lib.filesystem.pathType` doesn't work correctly on /nix/store/...-name paths in pure evaluation mode. For file set combinators it's safe to assume that + pathParts = deconstruct path; + assumeDirectory = + storeDirComponents == take (length pathParts.components - 1) pathParts.components; + in + if assumeDirectory then + "directory" + else + legacy path; + + in + if builtins ? readFileType then + builtins.readFileType + # Nix <2.14 compatibility shim + else if inPureEvalMode then + legacyPureEval + else + legacy; + + /* + Whether a path exists and is a directory. + + Type: + pathIsDirectory :: Path -> Bool + + Example: + pathIsDirectory /. + => true + + pathIsDirectory /this/does/not/exist + => false + + pathIsDirectory /some/file.nix + => false + */ + pathIsDirectory = path: + pathExists path && pathType path == "directory"; + + /* + Whether a path exists and is a regular file, meaning not a symlink or any other special file type. + + Type: + pathIsRegularFile :: Path -> Bool + + Example: + pathIsRegularFile /. + => false + + pathIsRegularFile /this/does/not/exist + => false + + pathIsRegularFile /some/file.nix + => true + */ + pathIsRegularFile = path: + pathExists path && pathType path == "regular"; + + /* + A map of all haskell packages defined in the given path, + identified by having a cabal file with the same name as the + directory itself. + + Type: Path -> Map String Path + */ + haskellPathsInDir = + # The directory within to search + root: + let # Files in the root + root-files = builtins.attrNames (builtins.readDir root); + # Files with their full paths + root-files-with-paths = + map (file: + { name = file; value = root + "/${file}"; } + ) root-files; + # Subdirectories of the root with a cabal file. + cabal-subdirs = + builtins.filter ({ name, value }: + builtins.pathExists (value + "/${name}.cabal") + ) root-files-with-paths; + in builtins.listToAttrs cabal-subdirs; + /* + Find the first directory containing a file matching 'pattern' + upward from a given 'file'. + Returns 'null' if no directories contain a file matching 'pattern'. + + Type: RegExp -> Path -> Nullable { path : Path; matches : [ MatchResults ]; } + */ + locateDominatingFile = + # The pattern to search for + pattern: + # The file to start searching upward from + file: + let go = path: + let files = builtins.attrNames (builtins.readDir path); + matches = builtins.filter (match: match != null) + (map (builtins.match pattern) files); + in + if builtins.length matches != 0 + then { inherit path matches; } + else if path == /. + then null + else go (dirOf path); + parent = dirOf file; + isDir = + let base = baseNameOf file; + type = (builtins.readDir parent).${base} or null; + in file == /. || type == "directory"; + in go (if isDir then file else parent); + + + /* + Given a directory, return a flattened list of all files within it recursively. + + Type: Path -> [ Path ] + */ + listFilesRecursive = + # The path to recursively list + dir: + lib.flatten (lib.mapAttrsToList (name: type: + if type == "directory" then + lib.filesystem.listFilesRecursive (dir + "/${name}") + else + dir + "/${name}" + ) (builtins.readDir dir)); + +} diff --git a/fixed-points.nix b/fixed-points.nix new file mode 100644 index 000000000..926428293 --- /dev/null +++ b/fixed-points.nix @@ -0,0 +1,113 @@ +{ lib, ... }: +rec { + # Compute the fixed point of the given function `f`, which is usually an + # attribute set that expects its final, non-recursive representation as an + # argument: + # + # f = self: { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; } + # + # Nix evaluates this recursion until all references to `self` have been + # resolved. At that point, the final result is returned and `f x = x` holds: + # + # nix-repl> fix f + # { bar = "bar"; foo = "foo"; foobar = "foobar"; } + # + # Type: fix :: (a -> a) -> a + # + # See https://en.wikipedia.org/wiki/Fixed-point_combinator for further + # details. + fix = f: let x = f x; in x; + + # A variant of `fix` that records the original recursive attribute set in the + # result. This is useful in combination with the `extends` function to + # implement deep overriding. See pkgs/development/haskell-modules/default.nix + # for a concrete example. + fix' = f: let x = f x // { __unfix__ = f; }; in x; + + # Return the fixpoint that `f` converges to when called recursively, starting + # with the input `x`. + # + # nix-repl> converge (x: x / 2) 16 + # 0 + converge = f: x: + let + x' = f x; + in + if x' == x + then x + else converge f x'; + + # Modify the contents of an explicitly recursive attribute set in a way that + # honors `self`-references. This is accomplished with a function + # + # g = self: super: { foo = super.foo + " + "; } + # + # that has access to the unmodified input (`super`) as well as the final + # non-recursive representation of the attribute set (`self`). `extends` + # differs from the native `//` operator insofar as that it's applied *before* + # references to `self` are resolved: + # + # nix-repl> fix (extends g f) + # { bar = "bar"; foo = "foo + "; foobar = "foo + bar"; } + # + # The name of the function is inspired by object-oriented inheritance, i.e. + # think of it as an infix operator `g extends f` that mimics the syntax from + # Java. It may seem counter-intuitive to have the "base class" as the second + # argument, but it's nice this way if several uses of `extends` are cascaded. + # + # To get a better understanding how `extends` turns a function with a fix + # point (the package set we start with) into a new function with a different fix + # point (the desired packages set) lets just see, how `extends g f` + # unfolds with `g` and `f` defined above: + # + # extends g f = self: let super = f self; in super // g self super; + # = self: let super = { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; }; in super // g self super + # = self: { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; } // g self { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; } + # = self: { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; } // { foo = "foo" + " + "; } + # = self: { foo = "foo + "; bar = "bar"; foobar = self.foo + self.bar; } + # + extends = f: rattrs: self: let super = rattrs self; in super // f self super; + + # Compose two extending functions of the type expected by 'extends' + # into one where changes made in the first are available in the + # 'super' of the second + composeExtensions = + f: g: final: prev: + let fApplied = f final prev; + prev' = prev // fApplied; + in fApplied // g final prev'; + + # Compose several extending functions of the type expected by 'extends' into + # one where changes made in preceding functions are made available to + # subsequent ones. + # + # composeManyExtensions : [packageSet -> packageSet -> packageSet] -> packageSet -> packageSet -> packageSet + # ^final ^prev ^overrides ^final ^prev ^overrides + composeManyExtensions = + lib.foldr (x: y: composeExtensions x y) (final: prev: {}); + + # Create an overridable, recursive attribute set. For example: + # + # nix-repl> obj = makeExtensible (self: { }) + # + # nix-repl> obj + # { __unfix__ = «lambda»; extend = «lambda»; } + # + # nix-repl> obj = obj.extend (self: super: { foo = "foo"; }) + # + # nix-repl> obj + # { __unfix__ = «lambda»; extend = «lambda»; foo = "foo"; } + # + # nix-repl> obj = obj.extend (self: super: { foo = super.foo + " + "; bar = "bar"; foobar = self.foo + self.bar; }) + # + # nix-repl> obj + # { __unfix__ = «lambda»; bar = "bar"; extend = «lambda»; foo = "foo + "; foobar = "foo + bar"; } + makeExtensible = makeExtensibleWithCustomName "extend"; + + # Same as `makeExtensible` but the name of the extending attribute is + # customized. + makeExtensibleWithCustomName = extenderName: rattrs: + fix' (self: (rattrs self) // { + ${extenderName} = f: makeExtensibleWithCustomName extenderName (extends f rattrs); + }); +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..0b5e54d54 --- /dev/null +++ b/flake.nix @@ -0,0 +1,5 @@ +{ + description = "Library of low-level helper functions for nix expressions."; + + outputs = { self }: { lib = import ./.; }; +} diff --git a/generators.nix b/generators.nix new file mode 100644 index 000000000..496845fc9 --- /dev/null +++ b/generators.nix @@ -0,0 +1,526 @@ +/* Functions that generate widespread file + * formats from nix data structures. + * + * They all follow a similar interface: + * generator { config-attrs } data + * + * `config-attrs` are “holes” in the generators + * with sensible default implementations that + * can be overwritten. The default implementations + * are mostly generators themselves, called with + * their respective default values; they can be reused. + * + * Tests can be found in ./tests/misc.nix + * Documentation in the manual, #sec-generators + */ +{ lib }: +with (lib).trivial; +let + libStr = lib.strings; + libAttr = lib.attrsets; + + inherit (lib) isFunction; +in + +rec { + + ## -- HELPER FUNCTIONS & DEFAULTS -- + + /* Convert a value to a sensible default string representation. + * The builtin `toString` function has some strange defaults, + * suitable for bash scripts but not much else. + */ + mkValueStringDefault = {}: v: with builtins; + let err = t: v: abort + ("generators.mkValueStringDefault: " + + "${t} not supported: ${toPretty {} v}"); + in if isInt v then toString v + # convert derivations to store paths + else if lib.isDerivation v then toString v + # we default to not quoting strings + else if isString v then v + # isString returns "1", which is not a good default + else if true == v then "true" + # here it returns to "", which is even less of a good default + else if false == v then "false" + else if null == v then "null" + # if you have lists you probably want to replace this + else if isList v then err "lists" v + # same as for lists, might want to replace + else if isAttrs v then err "attrsets" v + # functions can’t be printed of course + else if isFunction v then err "functions" v + # Floats currently can't be converted to precise strings, + # condition warning on nix version once this isn't a problem anymore + # See https://github.com/NixOS/nix/pull/3480 + else if isFloat v then libStr.floatToString v + else err "this value is" (toString v); + + + /* Generate a line of key k and value v, separated by + * character sep. If sep appears in k, it is escaped. + * Helper for synaxes with different separators. + * + * mkValueString specifies how values should be formatted. + * + * mkKeyValueDefault {} ":" "f:oo" "bar" + * > "f\:oo:bar" + */ + mkKeyValueDefault = { + mkValueString ? mkValueStringDefault {} + }: sep: k: v: + "${libStr.escape [sep] k}${sep}${mkValueString v}"; + + + ## -- FILE FORMAT GENERATORS -- + + + /* Generate a key-value-style config file from an attrset. + * + * mkKeyValue is the same as in toINI. + */ + toKeyValue = { + mkKeyValue ? mkKeyValueDefault {} "=", + listsAsDuplicateKeys ? false + }: + let mkLine = k: v: mkKeyValue k v + "\n"; + mkLines = if listsAsDuplicateKeys + then k: v: map (mkLine k) (if lib.isList v then v else [v]) + else k: v: [ (mkLine k v) ]; + in attrs: libStr.concatStrings (lib.concatLists (libAttr.mapAttrsToList mkLines attrs)); + + + /* Generate an INI-style config file from an + * attrset of sections to an attrset of key-value pairs. + * + * generators.toINI {} { + * foo = { hi = "${pkgs.hello}"; ciao = "bar"; }; + * baz = { "also, integers" = 42; }; + * } + * + *> [baz] + *> also, integers=42 + *> + *> [foo] + *> ciao=bar + *> hi=/nix/store/y93qql1p5ggfnaqjjqhxcw0vqw95rlz0-hello-2.10 + * + * The mk* configuration attributes can generically change + * the way sections and key-value strings are generated. + * + * For more examples see the test cases in ./tests/misc.nix. + */ + toINI = { + # apply transformations (e.g. escapes) to section names + mkSectionName ? (name: libStr.escape [ "[" "]" ] name), + # format a setting line from key and value + mkKeyValue ? mkKeyValueDefault {} "=", + # allow lists as values for duplicate keys + listsAsDuplicateKeys ? false + }: attrsOfAttrs: + let + # map function to string for each key val + mapAttrsToStringsSep = sep: mapFn: attrs: + libStr.concatStringsSep sep + (libAttr.mapAttrsToList mapFn attrs); + mkSection = sectName: sectValues: '' + [${mkSectionName sectName}] + '' + toKeyValue { inherit mkKeyValue listsAsDuplicateKeys; } sectValues; + in + # map input to ini sections + mapAttrsToStringsSep "\n" mkSection attrsOfAttrs; + + /* Generate an INI-style config file from an attrset + * specifying the global section (no header), and an + * attrset of sections to an attrset of key-value pairs. + * + * generators.toINIWithGlobalSection {} { + * globalSection = { + * someGlobalKey = "hi"; + * }; + * sections = { + * foo = { hi = "${pkgs.hello}"; ciao = "bar"; }; + * baz = { "also, integers" = 42; }; + * } + * + *> someGlobalKey=hi + *> + *> [baz] + *> also, integers=42 + *> + *> [foo] + *> ciao=bar + *> hi=/nix/store/y93qql1p5ggfnaqjjqhxcw0vqw95rlz0-hello-2.10 + * + * The mk* configuration attributes can generically change + * the way sections and key-value strings are generated. + * + * For more examples see the test cases in ./tests/misc.nix. + * + * If you don’t need a global section, you can also use + * `generators.toINI` directly, which only takes + * the part in `sections`. + */ + toINIWithGlobalSection = { + # apply transformations (e.g. escapes) to section names + mkSectionName ? (name: libStr.escape [ "[" "]" ] name), + # format a setting line from key and value + mkKeyValue ? mkKeyValueDefault {} "=", + # allow lists as values for duplicate keys + listsAsDuplicateKeys ? false + }: { globalSection, sections }: + ( if globalSection == {} + then "" + else (toKeyValue { inherit mkKeyValue listsAsDuplicateKeys; } globalSection) + + "\n") + + (toINI { inherit mkSectionName mkKeyValue listsAsDuplicateKeys; } sections); + + /* Generate a git-config file from an attrset. + * + * It has two major differences from the regular INI format: + * + * 1. values are indented with tabs + * 2. sections can have sub-sections + * + * generators.toGitINI { + * url."ssh://git@github.com/".insteadOf = "https://github.com"; + * user.name = "edolstra"; + * } + * + *> [url "ssh://git@github.com/"] + *> insteadOf = https://github.com/ + *> + *> [user] + *> name = edolstra + */ + toGitINI = attrs: + with builtins; + let + mkSectionName = name: + let + containsQuote = libStr.hasInfix ''"'' name; + sections = libStr.splitString "." name; + section = head sections; + subsections = tail sections; + subsection = concatStringsSep "." subsections; + in if containsQuote || subsections == [ ] then + name + else + ''${section} "${subsection}"''; + + # generation for multiple ini values + mkKeyValue = k: v: + let mkKeyValue = mkKeyValueDefault { } " = " k; + in concatStringsSep "\n" (map (kv: "\t" + mkKeyValue kv) (lib.toList v)); + + # converts { a.b.c = 5; } to { "a.b".c = 5; } for toINI + gitFlattenAttrs = let + recurse = path: value: + if isAttrs value && !lib.isDerivation value then + lib.mapAttrsToList (name: value: recurse ([ name ] ++ path) value) value + else if length path > 1 then { + ${concatStringsSep "." (lib.reverseList (tail path))}.${head path} = value; + } else { + ${head path} = value; + }; + in attrs: lib.foldl lib.recursiveUpdate { } (lib.flatten (recurse [ ] attrs)); + + toINI_ = toINI { inherit mkKeyValue mkSectionName; }; + in + toINI_ (gitFlattenAttrs attrs); + + /* Generates JSON from an arbitrary (non-function) value. + * For more information see the documentation of the builtin. + */ + toJSON = {}: builtins.toJSON; + + + /* YAML has been a strict superset of JSON since 1.2, so we + * use toJSON. Before it only had a few differences referring + * to implicit typing rules, so it should work with older + * parsers as well. + */ + toYAML = toJSON; + + withRecursion = + { + /* If this option is not null, the given value will stop evaluating at a certain depth */ + depthLimit + /* If this option is true, an error will be thrown, if a certain given depth is exceeded */ + , throwOnDepthLimit ? true + }: + assert builtins.isInt depthLimit; + let + specialAttrs = [ + "__functor" + "__functionArgs" + "__toString" + "__pretty" + ]; + stepIntoAttr = evalNext: name: + if builtins.elem name specialAttrs + then id + else evalNext; + transform = depth: + if depthLimit != null && depth > depthLimit then + if throwOnDepthLimit + then throw "Exceeded maximum eval-depth limit of ${toString depthLimit} while trying to evaluate with `generators.withRecursion'!" + else const "" + else id; + mapAny = with builtins; depth: v: + let + evalNext = x: mapAny (depth + 1) (transform (depth + 1) x); + in + if isAttrs v then mapAttrs (stepIntoAttr evalNext) v + else if isList v then map evalNext v + else transform (depth + 1) v; + in + mapAny 0; + + /* Pretty print a value, akin to `builtins.trace`. + * Should probably be a builtin as well. + * The pretty-printed string should be suitable for rendering default values + * in the NixOS manual. In particular, it should be as close to a valid Nix expression + * as possible. + */ + toPretty = { + /* If this option is true, attrsets like { __pretty = fn; val = …; } + will use fn to convert val to a pretty printed representation. + (This means fn is type Val -> String.) */ + allowPrettyValues ? false, + /* If this option is true, the output is indented with newlines for attribute sets and lists */ + multiline ? true, + /* Initial indentation level */ + indent ? "" + }: + let + go = indent: v: with builtins; + let isPath = v: typeOf v == "path"; + introSpace = if multiline then "\n${indent} " else " "; + outroSpace = if multiline then "\n${indent}" else " "; + in if isInt v then toString v + # toString loses precision on floats, so we use toJSON instead. This isn't perfect + # as the resulting string may not parse back as a float (e.g. 42, 1e-06), but for + # pretty-printing purposes this is acceptable. + else if isFloat v then builtins.toJSON v + else if isString v then + let + lines = filter (v: ! isList v) (builtins.split "\n" v); + escapeSingleline = libStr.escape [ "\\" "\"" "\${" ]; + escapeMultiline = libStr.replaceStrings [ "\${" "''" ] [ "''\${" "'''" ]; + singlelineResult = "\"" + concatStringsSep "\\n" (map escapeSingleline lines) + "\""; + multilineResult = let + escapedLines = map escapeMultiline lines; + # The last line gets a special treatment: if it's empty, '' is on its own line at the "outer" + # indentation level. Otherwise, '' is appended to the last line. + lastLine = lib.last escapedLines; + in "''" + introSpace + concatStringsSep introSpace (lib.init escapedLines) + + (if lastLine == "" then outroSpace else introSpace + lastLine) + "''"; + in + if multiline && length lines > 1 then multilineResult else singlelineResult + else if true == v then "true" + else if false == v then "false" + else if null == v then "null" + else if isPath v then toString v + else if isList v then + if v == [] then "[ ]" + else "[" + introSpace + + libStr.concatMapStringsSep introSpace (go (indent + " ")) v + + outroSpace + "]" + else if isFunction v then + let fna = lib.functionArgs v; + showFnas = concatStringsSep ", " (libAttr.mapAttrsToList + (name: hasDefVal: if hasDefVal then name + "?" else name) + fna); + in if fna == {} then "" + else "" + else if isAttrs v then + # apply pretty values if allowed + if allowPrettyValues && v ? __pretty && v ? val + then v.__pretty v.val + else if v == {} then "{ }" + else if v ? type && v.type == "derivation" then + "" + else "{" + introSpace + + libStr.concatStringsSep introSpace (libAttr.mapAttrsToList + (name: value: + "${libStr.escapeNixIdentifier name} = ${ + builtins.addErrorContext "while evaluating an attribute `${name}`" + (go (indent + " ") value) + };") v) + + outroSpace + "}" + else abort "generators.toPretty: should never happen (v = ${v})"; + in go indent; + + # PLIST handling + toPlist = {}: v: let + isFloat = builtins.isFloat or (x: false); + isPath = x: builtins.typeOf x == "path"; + expr = ind: x: with builtins; + if x == null then "" else + if isBool x then bool ind x else + if isInt x then int ind x else + if isString x then str ind x else + if isList x then list ind x else + if isAttrs x then attrs ind x else + if isPath x then str ind (toString x) else + if isFloat x then float ind x else + abort "generators.toPlist: should never happen (v = ${v})"; + + literal = ind: x: ind + x; + + bool = ind: x: literal ind (if x then "" else ""); + int = ind: x: literal ind "${toString x}"; + str = ind: x: literal ind "${x}"; + key = ind: x: literal ind "${x}"; + float = ind: x: literal ind "${toString x}"; + + indent = ind: expr "\t${ind}"; + + item = ind: libStr.concatMapStringsSep "\n" (indent ind); + + list = ind: x: libStr.concatStringsSep "\n" [ + (literal ind "") + (item ind x) + (literal ind "") + ]; + + attrs = ind: x: libStr.concatStringsSep "\n" [ + (literal ind "") + (attr ind x) + (literal ind "") + ]; + + attr = let attrFilter = name: value: name != "_module" && value != null; + in ind: x: libStr.concatStringsSep "\n" (lib.flatten (lib.mapAttrsToList + (name: value: lib.optionals (attrFilter name value) [ + (key "\t${ind}" name) + (expr "\t${ind}" value) + ]) x)); + + in '' + + +${expr "" v} +''; + + /* Translate a simple Nix expression to Dhall notation. + * Note that integers are translated to Integer and never + * the Natural type. + */ + toDhall = { }@args: v: + with builtins; + let concatItems = lib.strings.concatStringsSep ", "; + in if isAttrs v then + "{ ${ + concatItems (lib.attrsets.mapAttrsToList + (key: value: "${key} = ${toDhall args value}") v) + } }" + else if isList v then + "[ ${concatItems (map (toDhall args) v)} ]" + else if isInt v then + "${if v < 0 then "" else "+"}${toString v}" + else if isBool v then + (if v then "True" else "False") + else if isFunction v then + abort "generators.toDhall: cannot convert a function to Dhall" + else if v == null then + abort "generators.toDhall: cannot convert a null to Dhall" + else + builtins.toJSON v; + + /* + Translate a simple Nix expression to Lua representation with occasional + Lua-inlines that can be constructed by mkLuaInline function. + + Configuration: + * multiline - by default is true which results in indented block-like view. + * indent - initial indent. + * asBindings - by default generate single value, but with this use attrset to set global vars. + + Attention: + Regardless of multiline parameter there is no trailing newline. + + Example: + generators.toLua {} + { + cmd = [ "typescript-language-server" "--stdio" ]; + settings.workspace.library = mkLuaInline ''vim.api.nvim_get_runtime_file("", true)''; + } + -> + { + ["cmd"] = { + "typescript-language-server", + "--stdio" + }, + ["settings"] = { + ["workspace"] = { + ["library"] = (vim.api.nvim_get_runtime_file("", true)) + } + } + } + + Type: + toLua :: AttrSet -> Any -> String + */ + toLua = { + /* If this option is true, the output is indented with newlines for attribute sets and lists */ + multiline ? true, + /* Initial indentation level */ + indent ? "", + /* Interpret as variable bindings */ + asBindings ? false, + }@args: v: + with builtins; + let + innerIndent = "${indent} "; + introSpace = if multiline then "\n${innerIndent}" else " "; + outroSpace = if multiline then "\n${indent}" else " "; + innerArgs = args // { + indent = if asBindings then indent else innerIndent; + asBindings = false; + }; + concatItems = concatStringsSep ",${introSpace}"; + isLuaInline = { _type ? null, ... }: _type == "lua-inline"; + + generatedBindings = + assert lib.assertMsg (badVarNames == []) "Bad Lua var names: ${toPretty {} badVarNames}"; + libStr.concatStrings ( + lib.attrsets.mapAttrsToList (key: value: "${indent}${key} = ${toLua innerArgs value}\n") v + ); + + # https://en.wikibooks.org/wiki/Lua_Programming/variable#Variable_names + matchVarName = match "[[:alpha:]_][[:alnum:]_]*(\\.[[:alpha:]_][[:alnum:]_]*)*"; + badVarNames = filter (name: matchVarName name == null) (attrNames v); + in + if asBindings then + generatedBindings + else if v == null then + "nil" + else if isInt v || isFloat v || isString v || isBool v then + builtins.toJSON v + else if isList v then + (if v == [ ] then "{}" else + "{${introSpace}${concatItems (map (value: "${toLua innerArgs value}") v)}${outroSpace}}") + else if isAttrs v then + ( + if isLuaInline v then + "(${v.expr})" + else if v == { } then + "{}" + else + "{${introSpace}${concatItems ( + lib.attrsets.mapAttrsToList (key: value: "[${builtins.toJSON key}] = ${toLua innerArgs value}") v + )}${outroSpace}}" + ) + else + abort "generators.toLua: type ${typeOf v} is unsupported"; + + /* + Mark string as Lua expression to be inlined when processed by toLua. + + Type: + mkLuaInline :: String -> AttrSet + */ + mkLuaInline = expr: { _type = "lua-inline"; inherit expr; }; +} diff --git a/kernel.nix b/kernel.nix new file mode 100644 index 000000000..33da9663a --- /dev/null +++ b/kernel.nix @@ -0,0 +1,27 @@ +{ lib }: + +with lib; +{ + + + # Keeping these around in case we decide to change this horrible implementation :) + option = x: + x // { optional = true; }; + + yes = { tristate = "y"; optional = false; }; + no = { tristate = "n"; optional = false; }; + module = { tristate = "m"; optional = false; }; + unset = { tristate = null; optional = false; }; + freeform = x: { freeform = x; optional = false; }; + + /* + Common patterns/legacy used in common-config/hardened/config.nix + */ + whenHelpers = version: { + whenAtLeast = ver: mkIf (versionAtLeast version ver); + whenOlder = ver: mkIf (versionOlder version ver); + # range is (inclusive, exclusive) + whenBetween = verLow: verHigh: mkIf (versionAtLeast version verLow && versionOlder version verHigh); + }; + +} diff --git a/licenses.nix b/licenses.nix new file mode 100644 index 000000000..1d2565fba --- /dev/null +++ b/licenses.nix @@ -0,0 +1,1104 @@ +{ lib }: + +lib.mapAttrs (lname: lset: let + defaultLicense = rec { + shortName = lname; + free = true; # Most of our licenses are Free, explicitly declare unfree additions as such! + deprecated = false; + }; + + mkLicense = licenseDeclaration: let + applyDefaults = license: defaultLicense // license; + applySpdx = license: + if license ? spdxId + then license // { url = "https://spdx.org/licenses/${license.spdxId}.html"; } + else license; + applyRedistributable = license: { redistributable = license.free; } // license; + in lib.pipe licenseDeclaration [ + applyDefaults + applySpdx + applyRedistributable + ]; +in mkLicense lset) ({ + /* License identifiers from spdx.org where possible. + * If you cannot find your license here, then look for a similar license or + * add it to this list. The URL mentioned above is a good source for inspiration. + */ + + abstyles = { + spdxId = "Abstyles"; + fullName = "Abstyles License"; + }; + + afl20 = { + spdxId = "AFL-2.0"; + fullName = "Academic Free License v2.0"; + }; + + afl21 = { + spdxId = "AFL-2.1"; + fullName = "Academic Free License v2.1"; + }; + + afl3 = { + spdxId = "AFL-3.0"; + fullName = "Academic Free License v3.0"; + }; + + agpl3Only = { + spdxId = "AGPL-3.0-only"; + fullName = "GNU Affero General Public License v3.0 only"; + }; + + agpl3Plus = { + spdxId = "AGPL-3.0-or-later"; + fullName = "GNU Affero General Public License v3.0 or later"; + }; + + aladdin = { + spdxId = "Aladdin"; + fullName = "Aladdin Free Public License"; + free = false; + }; + + amazonsl = { + fullName = "Amazon Software License"; + url = "https://aws.amazon.com/asl/"; + free = false; + }; + + amd = { + fullName = "AMD License Agreement"; + url = "https://developer.amd.com/amd-license-agreement/"; + free = false; + }; + + aom = { + fullName = "Alliance for Open Media Patent License 1.0"; + url = "https://aomedia.org/license/patent-license/"; + }; + + apsl10 = { + spdxId = "APSL-1.0"; + fullName = "Apple Public Source License 1.0"; + }; + + apsl20 = { + spdxId = "APSL-2.0"; + fullName = "Apple Public Source License 2.0"; + }; + + arphicpl = { + fullName = "Arphic Public License"; + url = "https://www.freedesktop.org/wiki/Arphic_Public_License/"; + }; + + artistic1 = { + spdxId = "Artistic-1.0"; + fullName = "Artistic License 1.0"; + }; + + artistic2 = { + spdxId = "Artistic-2.0"; + fullName = "Artistic License 2.0"; + }; + + asl20 = { + spdxId = "Apache-2.0"; + fullName = "Apache License 2.0"; + }; + + asl20-llvm = { + spdxId = "Apache-2.0 WITH LLVM-exception"; + fullName = "Apache License 2.0 with LLVM Exceptions"; + }; + + bitstreamVera = { + spdxId = "Bitstream-Vera"; + fullName = "Bitstream Vera Font License"; + }; + + bitTorrent10 = { + spdxId = "BitTorrent-1.0"; + fullName = " BitTorrent Open Source License v1.0"; + }; + + bitTorrent11 = { + spdxId = "BitTorrent-1.1"; + fullName = " BitTorrent Open Source License v1.1"; + }; + + bola11 = { + url = "https://blitiri.com.ar/p/bola/"; + fullName = "Buena Onda License Agreement 1.1"; + }; + + boost = { + spdxId = "BSL-1.0"; + fullName = "Boost Software License 1.0"; + }; + + beerware = { + spdxId = "Beerware"; + fullName = "Beerware License"; + }; + + blueOak100 = { + spdxId = "BlueOak-1.0.0"; + fullName = "Blue Oak Model License 1.0.0"; + }; + + bsd0 = { + spdxId = "0BSD"; + fullName = "BSD Zero Clause License"; + }; + + bsd1 = { + spdxId = "BSD-1-Clause"; + fullName = "BSD 1-Clause License"; + }; + + bsd2 = { + spdxId = "BSD-2-Clause"; + fullName = ''BSD 2-clause "Simplified" License''; + }; + + bsd2Patent = { + spdxId = "BSD-2-Clause-Patent"; + fullName = "BSD-2-Clause Plus Patent License"; + }; + + bsd2WithViews = { + spdxId = "BSD-2-Clause-Views"; + fullName = "BSD 2-Clause with views sentence"; + }; + + bsd3 = { + spdxId = "BSD-3-Clause"; + fullName = ''BSD 3-clause "New" or "Revised" License''; + }; + + bsdOriginal = { + spdxId = "BSD-4-Clause"; + fullName = ''BSD 4-clause "Original" or "Old" License''; + }; + + bsdOriginalShortened = { + spdxId = "BSD-4-Clause-Shortened"; + fullName = "BSD 4 Clause Shortened"; + }; + + bsdOriginalUC = { + spdxId = "BSD-4-Clause-UC"; + fullName = "BSD 4-Clause University of California-Specific"; + }; + + bsdProtection = { + spdxId = "BSD-Protection"; + fullName = "BSD Protection License"; + }; + + bsl11 = { + fullName = "Business Source License 1.1"; + url = "https://mariadb.com/bsl11"; + free = false; + redistributable = true; + }; + + caossl = { + fullName = "Computer Associates Open Source Licence Version 1.0"; + url = "http://jxplorer.org/licence.html"; + }; + + cal10 = { + fullName = "Cryptographic Autonomy License version 1.0 (CAL-1.0)"; + url = "https://opensource.org/licenses/CAL-1.0"; + }; + + caldera = { + spdxId = "Caldera"; + fullName = "Caldera License"; + url = "http://www.lemis.com/grog/UNIX/ancient-source-all.pdf"; + }; + + capec = { + fullName = "Common Attack Pattern Enumeration and Classification"; + url = "https://capec.mitre.org/about/termsofuse.html"; + }; + + clArtistic = { + spdxId = "ClArtistic"; + fullName = "Clarified Artistic License"; + }; + + cc0 = { + spdxId = "CC0-1.0"; + fullName = "Creative Commons Zero v1.0 Universal"; + }; + + cc-by-nc-nd-30 = { + spdxId = "CC-BY-NC-ND-3.0"; + fullName = "Creative Commons Attribution Non Commercial No Derivative Works 3.0 Unported"; + free = false; + }; + + cc-by-nc-nd-40 = { + spdxId = "CC-BY-NC-ND-4.0"; + fullName = "Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International"; + free = false; + }; + + cc-by-nc-sa-20 = { + spdxId = "CC-BY-NC-SA-2.0"; + fullName = "Creative Commons Attribution Non Commercial Share Alike 2.0"; + free = false; + }; + + cc-by-nc-sa-25 = { + spdxId = "CC-BY-NC-SA-2.5"; + fullName = "Creative Commons Attribution Non Commercial Share Alike 2.5"; + free = false; + }; + + cc-by-nc-sa-30 = { + spdxId = "CC-BY-NC-SA-3.0"; + fullName = "Creative Commons Attribution Non Commercial Share Alike 3.0"; + free = false; + }; + + cc-by-nc-sa-40 = { + spdxId = "CC-BY-NC-SA-4.0"; + fullName = "Creative Commons Attribution Non Commercial Share Alike 4.0"; + free = false; + }; + + cc-by-nc-30 = { + spdxId = "CC-BY-NC-3.0"; + fullName = "Creative Commons Attribution Non Commercial 3.0 Unported"; + free = false; + }; + + cc-by-nc-40 = { + spdxId = "CC-BY-NC-4.0"; + fullName = "Creative Commons Attribution Non Commercial 4.0 International"; + free = false; + }; + + cc-by-nd-30 = { + spdxId = "CC-BY-ND-3.0"; + fullName = "Creative Commons Attribution-No Derivative Works v3.00"; + free = false; + }; + + cc-by-sa-25 = { + spdxId = "CC-BY-SA-2.5"; + fullName = "Creative Commons Attribution Share Alike 2.5"; + }; + + cc-by-30 = { + spdxId = "CC-BY-3.0"; + fullName = "Creative Commons Attribution 3.0"; + }; + + cc-by-sa-30 = { + spdxId = "CC-BY-SA-3.0"; + fullName = "Creative Commons Attribution Share Alike 3.0"; + }; + + cc-by-40 = { + spdxId = "CC-BY-4.0"; + fullName = "Creative Commons Attribution 4.0"; + }; + + cc-by-sa-40 = { + spdxId = "CC-BY-SA-4.0"; + fullName = "Creative Commons Attribution Share Alike 4.0"; + }; + + cddl = { + spdxId = "CDDL-1.0"; + fullName = "Common Development and Distribution License 1.0"; + }; + + cecill20 = { + spdxId = "CECILL-2.0"; + fullName = "CeCILL Free Software License Agreement v2.0"; + }; + + cecill21 = { + spdxId = "CECILL-2.1"; + fullName = "CeCILL Free Software License Agreement v2.1"; + }; + + cecill-b = { + spdxId = "CECILL-B"; + fullName = "CeCILL-B Free Software License Agreement"; + }; + + cecill-c = { + spdxId = "CECILL-C"; + fullName = "CeCILL-C Free Software License Agreement"; + }; + + cpal10 = { + spdxId = "CPAL-1.0"; + fullName = "Common Public Attribution License 1.0"; + }; + + cpl10 = { + spdxId = "CPL-1.0"; + fullName = "Common Public License 1.0"; + }; + + curl = { + spdxId = "curl"; + fullName = "curl License"; + }; + + doc = { + spdxId = "DOC"; + fullName = "DOC License"; + }; + + drl10 = { + spdxId = "DRL-1.0"; + fullName = "Detection Rule License 1.0"; + }; + + eapl = { + fullName = "EPSON AVASYS PUBLIC LICENSE"; + url = "https://avasys.jp/hp/menu000000700/hpg000000603.htm"; + free = false; + }; + + ecl20 = { + fullName = "Educational Community License, Version 2.0"; + url = "https://opensource.org/licenses/ECL-2.0"; + shortName = "ECL 2.0"; + spdxId = "ECL-2.0"; + }; + + efl10 = { + spdxId = "EFL-1.0"; + fullName = "Eiffel Forum License v1.0"; + }; + + efl20 = { + spdxId = "EFL-2.0"; + fullName = "Eiffel Forum License v2.0"; + }; + + elastic = { + fullName = "ELASTIC LICENSE"; + url = "https://github.com/elastic/elasticsearch/blob/master/licenses/ELASTIC-LICENSE.txt"; + free = false; + }; + + epl10 = { + spdxId = "EPL-1.0"; + fullName = "Eclipse Public License 1.0"; + }; + + epl20 = { + spdxId = "EPL-2.0"; + fullName = "Eclipse Public License 2.0"; + }; + + epson = { + fullName = "Seiko Epson Corporation Software License Agreement for Linux"; + url = "https://download.ebz.epson.net/dsc/du/02/eula/global/LINUX_EN.html"; + free = false; + }; + + eupl11 = { + spdxId = "EUPL-1.1"; + fullName = "European Union Public License 1.1"; + }; + + eupl12 = { + spdxId = "EUPL-1.2"; + fullName = "European Union Public License 1.2"; + }; + + fdl11Only = { + spdxId = "GFDL-1.1-only"; + fullName = "GNU Free Documentation License v1.1 only"; + }; + + fdl11Plus = { + spdxId = "GFDL-1.1-or-later"; + fullName = "GNU Free Documentation License v1.1 or later"; + }; + + fdl12Only = { + spdxId = "GFDL-1.2-only"; + fullName = "GNU Free Documentation License v1.2 only"; + }; + + fdl12Plus = { + spdxId = "GFDL-1.2-or-later"; + fullName = "GNU Free Documentation License v1.2 or later"; + }; + + fdl13Only = { + spdxId = "GFDL-1.3-only"; + fullName = "GNU Free Documentation License v1.3 only"; + }; + + fdl13Plus = { + spdxId = "GFDL-1.3-or-later"; + fullName = "GNU Free Documentation License v1.3 or later"; + }; + + ffsl = { + fullName = "Floodgap Free Software License"; + url = "https://www.floodgap.com/software/ffsl/license.html"; + free = false; + }; + + free = { + fullName = "Unspecified free software license"; + }; + + ftl = { + spdxId = "FTL"; + fullName = "Freetype Project License"; + }; + + g4sl = { + fullName = "Geant4 Software License"; + url = "https://geant4.web.cern.ch/geant4/license/LICENSE.html"; + }; + + geogebra = { + fullName = "GeoGebra Non-Commercial License Agreement"; + url = "https://www.geogebra.org/license"; + free = false; + }; + + generaluser = { + fullName = "GeneralUser GS License v2.0"; + url = "http://www.schristiancollins.com/generaluser.php"; # license included in sources + }; + + gpl1Only = { + spdxId = "GPL-1.0-only"; + fullName = "GNU General Public License v1.0 only"; + }; + + gpl1Plus = { + spdxId = "GPL-1.0-or-later"; + fullName = "GNU General Public License v1.0 or later"; + }; + + gpl2Only = { + spdxId = "GPL-2.0-only"; + fullName = "GNU General Public License v2.0 only"; + }; + + gpl2Classpath = { + spdxId = "GPL-2.0-with-classpath-exception"; + fullName = "GNU General Public License v2.0 only (with Classpath exception)"; + }; + + gpl2ClasspathPlus = { + fullName = "GNU General Public License v2.0 or later (with Classpath exception)"; + url = "https://fedoraproject.org/wiki/Licensing/GPL_Classpath_Exception"; + }; + + gpl2Oss = { + fullName = "GNU General Public License version 2 only (with OSI approved licenses linking exception)"; + url = "https://www.mysql.com/about/legal/licensing/foss-exception"; + }; + + gpl2Plus = { + spdxId = "GPL-2.0-or-later"; + fullName = "GNU General Public License v2.0 or later"; + }; + + gpl3Only = { + spdxId = "GPL-3.0-only"; + fullName = "GNU General Public License v3.0 only"; + }; + + gpl3Plus = { + spdxId = "GPL-3.0-or-later"; + fullName = "GNU General Public License v3.0 or later"; + }; + + gpl3ClasspathPlus = { + fullName = "GNU General Public License v3.0 or later (with Classpath exception)"; + url = "https://fedoraproject.org/wiki/Licensing/GPL_Classpath_Exception"; + }; + + hpnd = { + spdxId = "HPND"; + fullName = "Historic Permission Notice and Disclaimer"; + }; + + hpndSellVariant = { + fullName = "Historical Permission Notice and Disclaimer - sell variant"; + spdxId = "HPND-sell-variant"; + }; + + # Intel's license, seems free + iasl = { + fullName = "iASL"; + url = "https://old.calculate-linux.org/packages/licenses/iASL"; + }; + + ijg = { + spdxId = "IJG"; + fullName = "Independent JPEG Group License"; + }; + + imagemagick = { + fullName = "ImageMagick License"; + spdxId = "imagemagick"; + }; + + imlib2 = { + spdxId = "Imlib2"; + fullName = "Imlib2 License"; + }; + + info-zip = { + spdxId = "Info-ZIP"; + fullName = "Info-ZIP License"; + url = "http://www.info-zip.org/pub/infozip/license.html"; + }; + + inria-compcert = { + fullName = "INRIA Non-Commercial License Agreement for the CompCert verified compiler"; + url = "https://compcert.org/doc/LICENSE.txt"; + free = false; + }; + + inria-icesl = { + fullName = "INRIA Non-Commercial License Agreement for IceSL"; + url = "https://icesl.loria.fr/assets/pdf/EULA_IceSL_binary.pdf"; + free = false; + }; + + ipa = { + spdxId = "IPA"; + fullName = "IPA Font License"; + }; + + ipl10 = { + spdxId = "IPL-1.0"; + fullName = "IBM Public License v1.0"; + }; + + isc = { + spdxId = "ISC"; + fullName = "ISC License"; + }; + + # Proprietary binaries; free to redistribute without modification. + databricks = { + fullName = "Databricks Proprietary License"; + url = "https://pypi.org/project/databricks-connect"; + free = false; + }; + + databricks-dbx = { + fullName = "DataBricks eXtensions aka dbx License"; + url = "https://github.com/databrickslabs/dbx/blob/743b579a4ac44531f764c6e522dbe5a81a7dc0e4/LICENSE"; + free = false; + redistributable = false; + }; + + fair = { + fullName = "Fair License"; + spdxId = "Fair"; + free = true; + }; + + issl = { + fullName = "Intel Simplified Software License"; + url = "https://software.intel.com/en-us/license/intel-simplified-software-license"; + free = false; + }; + + lal12 = { + spdxId = "LAL-1.2"; + fullName = "Licence Art Libre 1.2"; + }; + + lal13 = { + spdxId = "LAL-1.3"; + fullName = "Licence Art Libre 1.3"; + }; + + lens = { + fullName = "Lens Terms of Service Agreement"; + url = "https://k8slens.dev/licenses/tos"; + free = false; + }; + + lgpl2Only = { + spdxId = "LGPL-2.0-only"; + fullName = "GNU Library General Public License v2 only"; + }; + + lgpl2Plus = { + spdxId = "LGPL-2.0-or-later"; + fullName = "GNU Library General Public License v2 or later"; + }; + + lgpl21Only = { + spdxId = "LGPL-2.1-only"; + fullName = "GNU Lesser General Public License v2.1 only"; + }; + + lgpl21Plus = { + spdxId = "LGPL-2.1-or-later"; + fullName = "GNU Lesser General Public License v2.1 or later"; + }; + + lgpl3Only = { + spdxId = "LGPL-3.0-only"; + fullName = "GNU Lesser General Public License v3.0 only"; + }; + + lgpl3Plus = { + spdxId = "LGPL-3.0-or-later"; + fullName = "GNU Lesser General Public License v3.0 or later"; + }; + + lgpllr = { + spdxId = "LGPLLR"; + fullName = "Lesser General Public License For Linguistic Resources"; + }; + + libpng = { + spdxId = "Libpng"; + fullName = "libpng License"; + }; + + libpng2 = { + spdxId = "libpng-2.0"; # Used since libpng 1.6.36. + fullName = "PNG Reference Library version 2"; + }; + + libssh2 = { + fullName = "libssh2 License"; + url = "https://www.libssh2.org/license.html"; + }; + + libtiff = { + spdxId = "libtiff"; + fullName = "libtiff License"; + }; + + llgpl21 = { + fullName = "Lisp LGPL; GNU Lesser General Public License version 2.1 with Franz Inc. preamble for clarification of LGPL terms in context of Lisp"; + url = "https://opensource.franz.com/preamble.html"; + }; + + lppl12 = { + spdxId = "LPPL-1.2"; + fullName = "LaTeX Project Public License v1.2"; + }; + + lppl13c = { + spdxId = "LPPL-1.3c"; + fullName = "LaTeX Project Public License v1.3c"; + }; + + lpl-102 = { + spdxId = "LPL-1.02"; + fullName = "Lucent Public License v1.02"; + }; + + miros = { + fullName = "MirOS License"; + url = "https://opensource.org/licenses/MirOS"; + }; + + # spdx.org does not (yet) differentiate between the X11 and Expat versions + # for details see https://en.wikipedia.org/wiki/MIT_License#Various_versions + mit = { + spdxId = "MIT"; + fullName = "MIT License"; + }; + # https://spdx.org/licenses/MIT-feh.html + mit-feh = { + spdxId = "MIT-feh"; + fullName = "feh License"; + }; + + mitAdvertising = { + spdxId = "MIT-advertising"; + fullName = "Enlightenment License (e16)"; + }; + + mit0 = { + spdxId = "MIT-0"; + fullName = "MIT No Attribution"; + }; + + mpl10 = { + spdxId = "MPL-1.0"; + fullName = "Mozilla Public License 1.0"; + }; + + mpl11 = { + spdxId = "MPL-1.1"; + fullName = "Mozilla Public License 1.1"; + }; + + mpl20 = { + spdxId = "MPL-2.0"; + fullName = "Mozilla Public License 2.0"; + }; + + mspl = { + spdxId = "MS-PL"; + fullName = "Microsoft Public License"; + }; + + nasa13 = { + spdxId = "NASA-1.3"; + fullName = "NASA Open Source Agreement 1.3"; + free = false; + }; + + ncsa = { + spdxId = "NCSA"; + fullName = "University of Illinois/NCSA Open Source License"; + }; + + nlpl = { + spdxId = "NLPL"; + fullName = "No Limit Public License"; + }; + + nposl3 = { + spdxId = "NPOSL-3.0"; + fullName = "Non-Profit Open Software License 3.0"; + }; + + obsidian = { + fullName = "Obsidian End User Agreement"; + url = "https://obsidian.md/eula"; + free = false; + }; + + ocamlpro_nc = { + fullName = "OCamlPro Non Commercial license version 1"; + url = "https://alt-ergo.ocamlpro.com/http/alt-ergo-2.2.0/OCamlPro-Non-Commercial-License.pdf"; + free = false; + }; + + odbl = { + spdxId = "ODbL-1.0"; + fullName = "Open Data Commons Open Database License v1.0"; + }; + + ofl = { + spdxId = "OFL-1.1"; + fullName = "SIL Open Font License 1.1"; + }; + + oml = { + spdxId = "OML"; + fullName = "Open Market License"; + }; + + openldap = { + spdxId = "OLDAP-2.8"; + fullName = "Open LDAP Public License v2.8"; + }; + + openssl = { + spdxId = "OpenSSL"; + fullName = "OpenSSL License"; + }; + + osl2 = { + spdxId = "OSL-2.0"; + fullName = "Open Software License 2.0"; + }; + + osl21 = { + spdxId = "OSL-2.1"; + fullName = "Open Software License 2.1"; + }; + + osl3 = { + spdxId = "OSL-3.0"; + fullName = "Open Software License 3.0"; + }; + + parity70 = { + spdxId = "Parity-7.0.0"; + fullName = "Parity Public License 7.0.0"; + url = "https://paritylicense.com/versions/7.0.0.html"; + }; + + php301 = { + spdxId = "PHP-3.01"; + fullName = "PHP License v3.01"; + }; + + postgresql = { + spdxId = "PostgreSQL"; + fullName = "PostgreSQL License"; + }; + + postman = { + fullName = "Postman EULA"; + url = "https://www.getpostman.com/licenses/postman_base_app"; + free = false; + }; + + psfl = { + spdxId = "Python-2.0"; + fullName = "Python Software Foundation License version 2"; + url = "https://docs.python.org/license.html"; + }; + + publicDomain = { + fullName = "Public Domain"; + }; + + purdueBsd = { + fullName = " Purdue BSD-Style License"; # also know as lsof license + url = "https://enterprise.dejacode.com/licenses/public/purdue-bsd"; + }; + + prosperity30 = { + fullName = "Prosperity-3.0.0"; + free = false; + url = "https://prosperitylicense.com/versions/3.0.0.html"; + }; + + qhull = { + spdxId = "Qhull"; + fullName = "Qhull License"; + }; + + qpl = { + spdxId = "QPL-1.0"; + fullName = "Q Public License 1.0"; + }; + + qwt = { + fullName = "Qwt License, Version 1.0"; + url = "https://qwt.sourceforge.io/qwtlicense.html"; + }; + + ruby = { + spdxId = "Ruby"; + fullName = "Ruby License"; + }; + + sendmail = { + spdxId = "Sendmail"; + fullName = "Sendmail License"; + }; + + sgi-b-20 = { + spdxId = "SGI-B-2.0"; + fullName = "SGI Free Software License B v2.0"; + }; + + # Gentoo seems to treat it as a license: + # https://gitweb.gentoo.org/repo/gentoo.git/tree/licenses/SGMLUG?id=7d999af4a47bf55e53e54713d98d145f935935c1 + sgmlug = { + fullName = "SGML UG SGML Parser Materials license"; + }; + + sleepycat = { + spdxId = "Sleepycat"; + fullName = "Sleepycat License"; + }; + + smail = { + shortName = "smail"; + fullName = "SMAIL General Public License"; + url = "https://sources.debian.org/copyright/license/debianutils/4.9.1/"; + }; + + sspl = { + shortName = "SSPL"; + fullName = "Server Side Public License"; + url = "https://www.mongodb.com/licensing/server-side-public-license"; + free = false; + # NOTE Debatable. + # The license a slightly modified AGPL but still considered unfree by the + # OSI for what seem like political reasons + redistributable = true; # Definitely redistributable though, it's an AGPL derivative + }; + + stk = { + shortName = "stk"; + fullName = "Synthesis Tool Kit 4.3"; + url = "https://github.com/thestk/stk/blob/master/LICENSE"; + }; + + tsl = { + shortName = "TSL"; + fullName = "Timescale License Agreegment"; + url = "https://github.com/timescale/timescaledb/blob/main/tsl/LICENSE-TIMESCALE"; + unfree = true; + }; + + tcltk = { + spdxId = "TCL"; + fullName = "TCL/TK License"; + }; + + ucd = { + fullName = "Unicode Character Database License"; + url = "https://fedoraproject.org/wiki/Licensing:UCD"; + }; + + ufl = { + fullName = "Ubuntu Font License 1.0"; + url = "https://ubuntu.com/legal/font-licence"; + }; + + unfree = { + fullName = "Unfree"; + free = false; + }; + + unfreeRedistributable = { + fullName = "Unfree redistributable"; + free = false; + redistributable = true; + }; + + unfreeRedistributableFirmware = { + fullName = "Unfree redistributable firmware"; + redistributable = true; + # Note: we currently consider these "free" for inclusion in the + # channel and NixOS images. + }; + + unicode-dfs-2015 = { + spdxId = "Unicode-DFS-2015"; + fullName = "Unicode License Agreement - Data Files and Software (2015)"; + }; + + unicode-dfs-2016 = { + spdxId = "Unicode-DFS-2016"; + fullName = "Unicode License Agreement - Data Files and Software (2016)"; + }; + + unlicense = { + spdxId = "Unlicense"; + fullName = "The Unlicense"; + }; + + upl = { + fullName = "Universal Permissive License"; + url = "https://oss.oracle.com/licenses/upl/"; + }; + + vim = { + spdxId = "Vim"; + fullName = "Vim License"; + }; + + virtualbox-puel = { + fullName = "Oracle VM VirtualBox Extension Pack Personal Use and Evaluation License (PUEL)"; + url = "https://www.virtualbox.org/wiki/VirtualBox_PUEL"; + free = false; + }; + + vol-sl = { + fullName = "Volatility Software License, Version 1.0"; + url = "https://www.volatilityfoundation.org/license/vsl-v1.0"; + }; + + vsl10 = { + spdxId = "VSL-1.0"; + fullName = "Vovida Software License v1.0"; + }; + + watcom = { + spdxId = "Watcom-1.0"; + fullName = "Sybase Open Watcom Public License 1.0"; + }; + + w3c = { + spdxId = "W3C"; + fullName = "W3C Software Notice and License"; + }; + + wadalab = { + fullName = "Wadalab Font License"; + url = "https://fedoraproject.org/wiki/Licensing:Wadalab?rd=Licensing/Wadalab"; + }; + + wtfpl = { + spdxId = "WTFPL"; + fullName = "Do What The F*ck You Want To Public License"; + }; + + wxWindows = { + spdxId = "wxWindows"; + fullName = "wxWindows Library Licence, Version 3.1"; + }; + + x11 = { + spdxId = "X11"; + fullName = "X11 License"; + }; + + xfig = { + fullName = "xfig"; + url = "http://mcj.sourceforge.net/authors.html#xfig"; # https is broken + }; + + zlib = { + spdxId = "Zlib"; + fullName = "zlib License"; + }; + + zpl20 = { + spdxId = "ZPL-2.0"; + fullName = "Zope Public License 2.0"; + }; + + zpl21 = { + spdxId = "ZPL-2.1"; + fullName = "Zope Public License 2.1"; + }; +} // { + # TODO: remove legacy aliases + agpl3 = { + spdxId = "AGPL-3.0"; + fullName = "GNU Affero General Public License v3.0"; + deprecated = true; + }; + gpl2 = { + spdxId = "GPL-2.0"; + fullName = "GNU General Public License v2.0"; + deprecated = true; + }; + gpl3 = { + spdxId = "GPL-3.0"; + fullName = "GNU General Public License v3.0"; + deprecated = true; + }; + lgpl2 = { + spdxId = "LGPL-2.0"; + fullName = "GNU Library General Public License v2"; + deprecated = true; + }; + lgpl21 = { + spdxId = "LGPL-2.1"; + fullName = "GNU Lesser General Public License v2.1"; + deprecated = true; + }; + lgpl3 = { + spdxId = "LGPL-3.0"; + fullName = "GNU Lesser General Public License v3.0"; + deprecated = true; + }; +}) diff --git a/lists.nix b/lists.nix new file mode 100644 index 000000000..bea9bf43c --- /dev/null +++ b/lists.nix @@ -0,0 +1,724 @@ +# General list operations. + +{ lib }: +let + inherit (lib.strings) toInt; + inherit (lib.trivial) compare min; + inherit (lib.attrsets) mapAttrs; +in +rec { + + inherit (builtins) head tail length isList elemAt concatLists filter elem genList map; + + /* Create a list consisting of a single element. `singleton x` is + sometimes more convenient with respect to indentation than `[x]` + when x spans multiple lines. + + Type: singleton :: a -> [a] + + Example: + singleton "foo" + => [ "foo" ] + */ + singleton = x: [x]; + + /* Apply the function to each element in the list. Same as `map`, but arguments + flipped. + + Type: forEach :: [a] -> (a -> b) -> [b] + + Example: + forEach [ 1 2 ] (x: + toString x + ) + => [ "1" "2" ] + */ + forEach = xs: f: map f xs; + + /* “right fold” a binary function `op` between successive elements of + `list` with `nul` as the starting value, i.e., + `foldr op nul [x_1 x_2 ... x_n] == op x_1 (op x_2 ... (op x_n nul))`. + + Type: foldr :: (a -> b -> b) -> b -> [a] -> b + + Example: + concat = foldr (a: b: a + b) "z" + concat [ "a" "b" "c" ] + => "abcz" + # different types + strange = foldr (int: str: toString (int + 1) + str) "a" + strange [ 1 2 3 4 ] + => "2345a" + */ + foldr = op: nul: list: + let + len = length list; + fold' = n: + if n == len + then nul + else op (elemAt list n) (fold' (n + 1)); + in fold' 0; + + /* `fold` is an alias of `foldr` for historic reasons */ + # FIXME(Profpatsch): deprecate? + fold = foldr; + + + /* “left fold”, like `foldr`, but from the left: + `foldl op nul [x_1 x_2 ... x_n] == op (... (op (op nul x_1) x_2) ... x_n)`. + + Type: foldl :: (b -> a -> b) -> b -> [a] -> b + + Example: + lconcat = foldl (a: b: a + b) "z" + lconcat [ "a" "b" "c" ] + => "zabc" + # different types + lstrange = foldl (str: int: str + toString (int + 1)) "a" + lstrange [ 1 2 3 4 ] + => "a2345" + */ + foldl = op: nul: list: + let + foldl' = n: + if n == -1 + then nul + else op (foldl' (n - 1)) (elemAt list n); + in foldl' (length list - 1); + + /* Strict version of `foldl`. + + The difference is that evaluation is forced upon access. Usually used + with small whole results (in contrast with lazily-generated list or large + lists where only a part is consumed.) + + Type: foldl' :: (b -> a -> b) -> b -> [a] -> b + */ + foldl' = builtins.foldl' or foldl; + + /* Map with index starting from 0 + + Type: imap0 :: (int -> a -> b) -> [a] -> [b] + + Example: + imap0 (i: v: "${v}-${toString i}") ["a" "b"] + => [ "a-0" "b-1" ] + */ + imap0 = f: list: genList (n: f n (elemAt list n)) (length list); + + /* Map with index starting from 1 + + Type: imap1 :: (int -> a -> b) -> [a] -> [b] + + Example: + imap1 (i: v: "${v}-${toString i}") ["a" "b"] + => [ "a-1" "b-2" ] + */ + imap1 = f: list: genList (n: f (n + 1) (elemAt list n)) (length list); + + /* Map and concatenate the result. + + Type: concatMap :: (a -> [b]) -> [a] -> [b] + + Example: + concatMap (x: [x] ++ ["z"]) ["a" "b"] + => [ "a" "z" "b" "z" ] + */ + concatMap = builtins.concatMap or (f: list: concatLists (map f list)); + + /* Flatten the argument into a single list; that is, nested lists are + spliced into the top-level lists. + + Example: + flatten [1 [2 [3] 4] 5] + => [1 2 3 4 5] + flatten 1 + => [1] + */ + flatten = x: + if isList x + then concatMap (y: flatten y) x + else [x]; + + /* Remove elements equal to 'e' from a list. Useful for buildInputs. + + Type: remove :: a -> [a] -> [a] + + Example: + remove 3 [ 1 3 4 3 ] + => [ 1 4 ] + */ + remove = + # Element to remove from the list + e: filter (x: x != e); + + /* Find the sole element in the list matching the specified + predicate, returns `default` if no such element exists, or + `multiple` if there are multiple matching elements. + + Type: findSingle :: (a -> bool) -> a -> a -> [a] -> a + + Example: + findSingle (x: x == 3) "none" "multiple" [ 1 3 3 ] + => "multiple" + findSingle (x: x == 3) "none" "multiple" [ 1 3 ] + => 3 + findSingle (x: x == 3) "none" "multiple" [ 1 9 ] + => "none" + */ + findSingle = + # Predicate + pred: + # Default value to return if element was not found. + default: + # Default value to return if more than one element was found + multiple: + # Input list + list: + let found = filter pred list; len = length found; + in if len == 0 then default + else if len != 1 then multiple + else head found; + + /* Find the first element in the list matching the specified + predicate or return `default` if no such element exists. + + Type: findFirst :: (a -> bool) -> a -> [a] -> a + + Example: + findFirst (x: x > 3) 7 [ 1 6 4 ] + => 6 + findFirst (x: x > 9) 7 [ 1 6 4 ] + => 7 + */ + findFirst = + # Predicate + pred: + # Default value to return + default: + # Input list + list: + let + # A naive recursive implementation would be much simpler, but + # would also overflow the evaluator stack. We use `foldl'` as a workaround + # because it reuses the same stack space, evaluating the function for one + # element after another. We can't return early, so this means that we + # sacrifice early cutoff, but that appears to be an acceptable cost. A + # clever scheme with "exponential search" is possible, but appears over- + # engineered for now. See https://github.com/NixOS/nixpkgs/pull/235267 + + # Invariant: + # - if index < 0 then el == elemAt list (- index - 1) and all elements before el didn't satisfy pred + # - if index >= 0 then pred (elemAt list index) and all elements before (elemAt list index) didn't satisfy pred + # + # We start with index -1 and the 0'th element of the list, which satisfies the invariant + resultIndex = foldl' (index: el: + if index < 0 then + # No match yet before the current index, we need to check the element + if pred el then + # We have a match! Turn it into the actual index to prevent future iterations from modifying it + - index - 1 + else + # Still no match, update the index to the next element (we're counting down, so minus one) + index - 1 + else + # There's already a match, propagate the index without evaluating anything + index + ) (-1) list; + in + if resultIndex < 0 then + default + else + elemAt list resultIndex; + + /* Return true if function `pred` returns true for at least one + element of `list`. + + Type: any :: (a -> bool) -> [a] -> bool + + Example: + any isString [ 1 "a" { } ] + => true + any isString [ 1 { } ] + => false + */ + any = builtins.any or (pred: foldr (x: y: if pred x then true else y) false); + + /* Return true if function `pred` returns true for all elements of + `list`. + + Type: all :: (a -> bool) -> [a] -> bool + + Example: + all (x: x < 3) [ 1 2 ] + => true + all (x: x < 3) [ 1 2 3 ] + => false + */ + all = builtins.all or (pred: foldr (x: y: if pred x then y else false) true); + + /* Count how many elements of `list` match the supplied predicate + function. + + Type: count :: (a -> bool) -> [a] -> int + + Example: + count (x: x == 3) [ 3 2 3 4 6 ] + => 2 + */ + count = + # Predicate + pred: foldl' (c: x: if pred x then c + 1 else c) 0; + + /* Return a singleton list or an empty list, depending on a boolean + value. Useful when building lists with optional elements + (e.g. `++ optional (system == "i686-linux") firefox`). + + Type: optional :: bool -> a -> [a] + + Example: + optional true "foo" + => [ "foo" ] + optional false "foo" + => [ ] + */ + optional = cond: elem: if cond then [elem] else []; + + /* Return a list or an empty list, depending on a boolean value. + + Type: optionals :: bool -> [a] -> [a] + + Example: + optionals true [ 2 3 ] + => [ 2 3 ] + optionals false [ 2 3 ] + => [ ] + */ + optionals = + # Condition + cond: + # List to return if condition is true + elems: if cond then elems else []; + + + /* If argument is a list, return it; else, wrap it in a singleton + list. If you're using this, you should almost certainly + reconsider if there isn't a more "well-typed" approach. + + Example: + toList [ 1 2 ] + => [ 1 2 ] + toList "hi" + => [ "hi "] + */ + toList = x: if isList x then x else [x]; + + /* Return a list of integers from `first` up to and including `last`. + + Type: range :: int -> int -> [int] + + Example: + range 2 4 + => [ 2 3 4 ] + range 3 2 + => [ ] + */ + range = + # First integer in the range + first: + # Last integer in the range + last: + if first > last then + [] + else + genList (n: first + n) (last - first + 1); + + /* Return a list with `n` copies of an element. + + Type: replicate :: int -> a -> [a] + + Example: + replicate 3 "a" + => [ "a" "a" "a" ] + replicate 2 true + => [ true true ] + */ + replicate = n: elem: genList (_: elem) n; + + /* Splits the elements of a list in two lists, `right` and + `wrong`, depending on the evaluation of a predicate. + + Type: (a -> bool) -> [a] -> { right :: [a]; wrong :: [a]; } + + Example: + partition (x: x > 2) [ 5 1 2 3 4 ] + => { right = [ 5 3 4 ]; wrong = [ 1 2 ]; } + */ + partition = builtins.partition or (pred: + foldr (h: t: + if pred h + then { right = [h] ++ t.right; wrong = t.wrong; } + else { right = t.right; wrong = [h] ++ t.wrong; } + ) { right = []; wrong = []; }); + + /* Splits the elements of a list into many lists, using the return value of a predicate. + Predicate should return a string which becomes keys of attrset `groupBy` returns. + + `groupBy'` allows to customise the combining function and initial value + + Example: + groupBy (x: boolToString (x > 2)) [ 5 1 2 3 4 ] + => { true = [ 5 3 4 ]; false = [ 1 2 ]; } + groupBy (x: x.name) [ {name = "icewm"; script = "icewm &";} + {name = "xfce"; script = "xfce4-session &";} + {name = "icewm"; script = "icewmbg &";} + {name = "mate"; script = "gnome-session &";} + ] + => { icewm = [ { name = "icewm"; script = "icewm &"; } + { name = "icewm"; script = "icewmbg &"; } ]; + mate = [ { name = "mate"; script = "gnome-session &"; } ]; + xfce = [ { name = "xfce"; script = "xfce4-session &"; } ]; + } + + groupBy' builtins.add 0 (x: boolToString (x > 2)) [ 5 1 2 3 4 ] + => { true = 12; false = 3; } + */ + groupBy' = op: nul: pred: lst: mapAttrs (name: foldl op nul) (groupBy pred lst); + + groupBy = builtins.groupBy or ( + pred: foldl' (r: e: + let + key = pred e; + in + r // { ${key} = (r.${key} or []) ++ [e]; } + ) {}); + + /* Merges two lists of the same size together. If the sizes aren't the same + the merging stops at the shortest. How both lists are merged is defined + by the first argument. + + Type: zipListsWith :: (a -> b -> c) -> [a] -> [b] -> [c] + + Example: + zipListsWith (a: b: a + b) ["h" "l"] ["e" "o"] + => ["he" "lo"] + */ + zipListsWith = + # Function to zip elements of both lists + f: + # First list + fst: + # Second list + snd: + genList + (n: f (elemAt fst n) (elemAt snd n)) (min (length fst) (length snd)); + + /* Merges two lists of the same size together. If the sizes aren't the same + the merging stops at the shortest. + + Type: zipLists :: [a] -> [b] -> [{ fst :: a; snd :: b; }] + + Example: + zipLists [ 1 2 ] [ "a" "b" ] + => [ { fst = 1; snd = "a"; } { fst = 2; snd = "b"; } ] + */ + zipLists = zipListsWith (fst: snd: { inherit fst snd; }); + + /* Reverse the order of the elements of a list. + + Type: reverseList :: [a] -> [a] + + Example: + + reverseList [ "b" "o" "j" ] + => [ "j" "o" "b" ] + */ + reverseList = xs: + let l = length xs; in genList (n: elemAt xs (l - n - 1)) l; + + /* Depth-First Search (DFS) for lists `list != []`. + + `before a b == true` means that `b` depends on `a` (there's an + edge from `b` to `a`). + + Example: + listDfs true hasPrefix [ "/home/user" "other" "/" "/home" ] + == { minimal = "/"; # minimal element + visited = [ "/home/user" ]; # seen elements (in reverse order) + rest = [ "/home" "other" ]; # everything else + } + + listDfs true hasPrefix [ "/home/user" "other" "/" "/home" "/" ] + == { cycle = "/"; # cycle encountered at this element + loops = [ "/" ]; # and continues to these elements + visited = [ "/" "/home/user" ]; # elements leading to the cycle (in reverse order) + rest = [ "/home" "other" ]; # everything else + + */ + listDfs = stopOnCycles: before: list: + let + dfs' = us: visited: rest: + let + c = filter (x: before x us) visited; + b = partition (x: before x us) rest; + in if stopOnCycles && (length c > 0) + then { cycle = us; loops = c; inherit visited rest; } + else if length b.right == 0 + then # nothing is before us + { minimal = us; inherit visited rest; } + else # grab the first one before us and continue + dfs' (head b.right) + ([ us ] ++ visited) + (tail b.right ++ b.wrong); + in dfs' (head list) [] (tail list); + + /* Sort a list based on a partial ordering using DFS. This + implementation is O(N^2), if your ordering is linear, use `sort` + instead. + + `before a b == true` means that `b` should be after `a` + in the result. + + Example: + + toposort hasPrefix [ "/home/user" "other" "/" "/home" ] + == { result = [ "/" "/home" "/home/user" "other" ]; } + + toposort hasPrefix [ "/home/user" "other" "/" "/home" "/" ] + == { cycle = [ "/home/user" "/" "/" ]; # path leading to a cycle + loops = [ "/" ]; } # loops back to these elements + + toposort hasPrefix [ "other" "/home/user" "/home" "/" ] + == { result = [ "other" "/" "/home" "/home/user" ]; } + + toposort (a: b: a < b) [ 3 2 1 ] == { result = [ 1 2 3 ]; } + + */ + toposort = before: list: + let + dfsthis = listDfs true before list; + toporest = toposort before (dfsthis.visited ++ dfsthis.rest); + in + if length list < 2 + then # finish + { result = list; } + else if dfsthis ? cycle + then # there's a cycle, starting from the current vertex, return it + { cycle = reverseList ([ dfsthis.cycle ] ++ dfsthis.visited); + inherit (dfsthis) loops; } + else if toporest ? cycle + then # there's a cycle somewhere else in the graph, return it + toporest + # Slow, but short. Can be made a bit faster with an explicit stack. + else # there are no cycles + { result = [ dfsthis.minimal ] ++ toporest.result; }; + + /* Sort a list based on a comparator function which compares two + elements and returns true if the first argument is strictly below + the second argument. The returned list is sorted in an increasing + order. The implementation does a quick-sort. + + Example: + sort (a: b: a < b) [ 5 3 7 ] + => [ 3 5 7 ] + */ + sort = builtins.sort or ( + strictLess: list: + let + len = length list; + first = head list; + pivot' = n: acc@{ left, right }: let el = elemAt list n; next = pivot' (n + 1); in + if n == len + then acc + else if strictLess first el + then next { inherit left; right = [ el ] ++ right; } + else + next { left = [ el ] ++ left; inherit right; }; + pivot = pivot' 1 { left = []; right = []; }; + in + if len < 2 then list + else (sort strictLess pivot.left) ++ [ first ] ++ (sort strictLess pivot.right)); + + /* Compare two lists element-by-element. + + Example: + compareLists compare [] [] + => 0 + compareLists compare [] [ "a" ] + => -1 + compareLists compare [ "a" ] [] + => 1 + compareLists compare [ "a" "b" ] [ "a" "c" ] + => -1 + */ + compareLists = cmp: a: b: + if a == [] + then if b == [] + then 0 + else -1 + else if b == [] + then 1 + else let rel = cmp (head a) (head b); in + if rel == 0 + then compareLists cmp (tail a) (tail b) + else rel; + + /* Sort list using "Natural sorting". + Numeric portions of strings are sorted in numeric order. + + Example: + naturalSort ["disk11" "disk8" "disk100" "disk9"] + => ["disk8" "disk9" "disk11" "disk100"] + naturalSort ["10.46.133.149" "10.5.16.62" "10.54.16.25"] + => ["10.5.16.62" "10.46.133.149" "10.54.16.25"] + naturalSort ["v0.2" "v0.15" "v0.0.9"] + => [ "v0.0.9" "v0.2" "v0.15" ] + */ + naturalSort = lst: + let + vectorise = s: map (x: if isList x then toInt (head x) else x) (builtins.split "(0|[1-9][0-9]*)" s); + prepared = map (x: [ (vectorise x) x ]) lst; # remember vectorised version for O(n) regex splits + less = a: b: (compareLists compare (head a) (head b)) < 0; + in + map (x: elemAt x 1) (sort less prepared); + + /* Return the first (at most) N elements of a list. + + Type: take :: int -> [a] -> [a] + + Example: + take 2 [ "a" "b" "c" "d" ] + => [ "a" "b" ] + take 2 [ ] + => [ ] + */ + take = + # Number of elements to take + count: sublist 0 count; + + /* Remove the first (at most) N elements of a list. + + Type: drop :: int -> [a] -> [a] + + Example: + drop 2 [ "a" "b" "c" "d" ] + => [ "c" "d" ] + drop 2 [ ] + => [ ] + */ + drop = + # Number of elements to drop + count: + # Input list + list: sublist count (length list) list; + + commonPrefixLength = lhs: rhs: + let + minLength = min (length lhs) (length rhs); + recurse = index: + if index >= minLength || elemAt lhs index != elemAt rhs index then + index + else + recurse (index + 1); + in recurse 0; + + commonPrefix = lhs: rhs: + take (commonPrefixLength lhs rhs) lhs; + + /* Return a list consisting of at most `count` elements of `list`, + starting at index `start`. + + Type: sublist :: int -> int -> [a] -> [a] + + Example: + sublist 1 3 [ "a" "b" "c" "d" "e" ] + => [ "b" "c" "d" ] + sublist 1 3 [ ] + => [ ] + */ + sublist = + # Index at which to start the sublist + start: + # Number of elements to take + count: + # Input list + list: + let len = length list; in + genList + (n: elemAt list (n + start)) + (if start >= len then 0 + else if start + count > len then len - start + else count); + + /* Return the last element of a list. + + This function throws an error if the list is empty. + + Type: last :: [a] -> a + + Example: + last [ 1 2 3 ] + => 3 + */ + last = list: + assert lib.assertMsg (list != []) "lists.last: list must not be empty!"; + elemAt list (length list - 1); + + /* Return all elements but the last. + + This function throws an error if the list is empty. + + Type: init :: [a] -> [a] + + Example: + init [ 1 2 3 ] + => [ 1 2 ] + */ + init = list: + assert lib.assertMsg (list != []) "lists.init: list must not be empty!"; + take (length list - 1) list; + + + /* Return the image of the cross product of some lists by a function. + + Example: + crossLists (x:y: "${toString x}${toString y}") [[1 2] [3 4]] + => [ "13" "14" "23" "24" ] + */ + crossLists = builtins.trace + "lib.crossLists is deprecated, use lib.cartesianProductOfSets instead" + (f: foldl (fs: args: concatMap (f: map f args) fs) [f]); + + + /* Remove duplicate elements from the list. O(n^2) complexity. + + Type: unique :: [a] -> [a] + + Example: + unique [ 3 2 3 4 ] + => [ 3 2 4 ] + */ + unique = foldl' (acc: e: if elem e acc then acc else acc ++ [ e ]) []; + + /* Intersects list 'e' and another list. O(nm) complexity. + + Example: + intersectLists [ 1 2 3 ] [ 6 3 2 ] + => [ 3 2 ] + */ + intersectLists = e: filter (x: elem x e); + + /* Subtracts list 'e' from another list. O(nm) complexity. + + Example: + subtractLists [ 3 2 ] [ 1 2 3 4 5 3 ] + => [ 1 4 5 ] + */ + subtractLists = e: filter (x: !(elem x e)); + + /* Test if two lists have no common element. + It should be slightly more efficient than (intersectLists a b == []) + */ + mutuallyExclusive = a: b: length a == 0 || !(any (x: elem x a) b); + +} diff --git a/meta.nix b/meta.nix new file mode 100644 index 000000000..5fd55c4e9 --- /dev/null +++ b/meta.nix @@ -0,0 +1,148 @@ +/* Some functions for manipulating meta attributes, as well as the + name attribute. */ + +{ lib }: + +rec { + + + /* Add to or override the meta attributes of the given + derivation. + + Example: + addMetaAttrs {description = "Bla blah";} somePkg + */ + addMetaAttrs = newAttrs: drv: + drv // { meta = (drv.meta or {}) // newAttrs; }; + + + /* Disable Hydra builds of given derivation. + */ + dontDistribute = drv: addMetaAttrs { hydraPlatforms = []; } drv; + + + /* Change the symbolic name of a package for presentation purposes + (i.e., so that nix-env users can tell them apart). + */ + setName = name: drv: drv // {inherit name;}; + + + /* Like `setName`, but takes the previous name as an argument. + + Example: + updateName (oldName: oldName + "-experimental") somePkg + */ + updateName = updater: drv: drv // {name = updater (drv.name);}; + + + /* Append a suffix to the name of a package (before the version + part). */ + appendToName = suffix: updateName (name: + let x = builtins.parseDrvName name; in "${x.name}-${suffix}-${x.version}"); + + + /* Apply a function to each derivation and only to derivations in an attrset. + */ + mapDerivationAttrset = f: set: lib.mapAttrs (name: pkg: if lib.isDerivation pkg then (f pkg) else pkg) set; + + /* Set the nix-env priority of the package. + */ + setPrio = priority: addMetaAttrs { inherit priority; }; + + /* Decrease the nix-env priority of the package, i.e., other + versions/variants of the package will be preferred. + */ + lowPrio = setPrio 10; + + /* Apply lowPrio to an attrset with derivations + */ + lowPrioSet = set: mapDerivationAttrset lowPrio set; + + + /* Increase the nix-env priority of the package, i.e., this + version/variant of the package will be preferred. + */ + hiPrio = setPrio (-10); + + /* Apply hiPrio to an attrset with derivations + */ + hiPrioSet = set: mapDerivationAttrset hiPrio set; + + + /* Check to see if a platform is matched by the given `meta.platforms` + element. + + A `meta.platform` pattern is either + + 1. (legacy) a system string. + + 2. (modern) a pattern for the entire platform structure (see `lib.systems.inspect.platformPatterns`). + + 3. (modern) a pattern for the platform `parsed` field (see `lib.systems.inspect.patterns`). + + We can inject these into a pattern for the whole of a structured platform, + and then match that. + */ + platformMatch = platform: elem: let + pattern = + if builtins.isString elem + then { system = elem; } + else if elem?parsed + then elem + else { parsed = elem; }; + in lib.matchAttrs pattern platform; + + /* Check if a package is available on a given platform. + + A package is available on a platform if both + + 1. One of `meta.platforms` pattern matches the given + platform, or `meta.platforms` is not present. + + 2. None of `meta.badPlatforms` pattern matches the given platform. + */ + availableOn = platform: pkg: + ((!pkg?meta.platforms) || lib.any (platformMatch platform) pkg.meta.platforms) && + lib.all (elem: !platformMatch platform elem) (pkg.meta.badPlatforms or []); + + /* Get the corresponding attribute in lib.licenses + from the SPDX ID. + For SPDX IDs, see + https://spdx.org/licenses + + Type: + getLicenseFromSpdxId :: str -> AttrSet + + Example: + lib.getLicenseFromSpdxId "MIT" == lib.licenses.mit + => true + lib.getLicenseFromSpdxId "mIt" == lib.licenses.mit + => true + lib.getLicenseFromSpdxId "MY LICENSE" + => trace: warning: getLicenseFromSpdxId: No license matches the given SPDX ID: MY LICENSE + => { shortName = "MY LICENSE"; } + */ + getLicenseFromSpdxId = + let + spdxLicenses = lib.mapAttrs (id: ls: assert lib.length ls == 1; builtins.head ls) + (lib.groupBy (l: lib.toLower l.spdxId) (lib.filter (l: l ? spdxId) (lib.attrValues lib.licenses))); + in licstr: + spdxLicenses.${ lib.toLower licstr } or ( + lib.warn "getLicenseFromSpdxId: No license matches the given SPDX ID: ${licstr}" + { shortName = licstr; } + ); + + /* Get the path to the main program of a derivation with either + meta.mainProgram or pname or name + + Type: getExe :: derivation -> string + + Example: + getExe pkgs.hello + => "/nix/store/g124820p9hlv4lj8qplzxw1c44dxaw1k-hello-2.12/bin/hello" + getExe pkgs.mustache-go + => "/nix/store/am9ml4f4ywvivxnkiaqwr0hyxka1xjsf-mustache-go-1.3.0/bin/mustache" + */ + getExe = x: + "${lib.getBin x}/bin/${x.meta.mainProgram or (lib.getName x)}"; +} diff --git a/minver.nix b/minver.nix new file mode 100644 index 000000000..507d45bba --- /dev/null +++ b/minver.nix @@ -0,0 +1,2 @@ +# Expose the minimum required version for evaluating Nixpkgs +"2.3" diff --git a/modules.nix b/modules.nix new file mode 100644 index 000000000..4dc8c663b --- /dev/null +++ b/modules.nix @@ -0,0 +1,1287 @@ +{ lib }: + +let + inherit (lib) + all + any + attrByPath + attrNames + catAttrs + concatLists + concatMap + concatStringsSep + elem + filter + foldl' + getAttrFromPath + head + id + imap1 + isAttrs + isBool + isFunction + isList + isPath + isString + length + mapAttrs + mapAttrsToList + mapAttrsRecursiveCond + min + optional + optionalAttrs + optionalString + recursiveUpdate + reverseList sort + setAttrByPath + types + warnIf + zipAttrsWith + ; + inherit (lib.options) + isOption + mkOption + showDefs + showFiles + showOption + unknownModule + ; + inherit (lib.strings) + isConvertibleWithToString + ; + + showDeclPrefix = loc: decl: prefix: + " - option(s) with prefix `${showOption (loc ++ [prefix])}' in module `${decl._file}'"; + showRawDecls = loc: decls: + concatStringsSep "\n" + (sort (a: b: a < b) + (concatMap + (decl: map + (showDeclPrefix loc decl) + (attrNames decl.options) + ) + decls + )); + + /* See https://nixos.org/manual/nixpkgs/unstable/#module-system-lib-evalModules + or file://./../doc/module-system/module-system.chapter.md + + !!! Please think twice before adding to this argument list! The more + that is specified here instead of in the modules themselves the harder + it is to transparently move a set of modules to be a submodule of another + config (as the proper arguments need to be replicated at each call to + evalModules) and the less declarative the module set is. */ + evalModules = evalModulesArgs@ + { modules + , prefix ? [] + , # This should only be used for special arguments that need to be evaluated + # when resolving module structure (like in imports). For everything else, + # there's _module.args. If specialArgs.modulesPath is defined it will be + # used as the base path for disabledModules. + specialArgs ? {} + , # `class`: + # A nominal type for modules. When set and non-null, this adds a check to + # make sure that only compatible modules are imported. + # This would be remove in the future, Prefer _module.args option instead. + class ? null + , args ? {} + , # This would be remove in the future, Prefer _module.check option instead. + check ? true + }: + let + withWarnings = x: + lib.warnIf (evalModulesArgs?args) "The args argument to evalModules is deprecated. Please set config._module.args instead." + lib.warnIf (evalModulesArgs?check) "The check argument to evalModules is deprecated. Please set config._module.check instead." + x; + + legacyModules = + optional (evalModulesArgs?args) { + config = { + _module.args = args; + }; + } + ++ optional (evalModulesArgs?check) { + config = { + _module.check = mkDefault check; + }; + }; + regularModules = modules ++ legacyModules; + + # This internal module declare internal options under the `_module' + # attribute. These options are fragile, as they are used by the + # module system to change the interpretation of modules. + # + # When extended with extendModules or moduleType, a fresh instance of + # this module is used, to avoid conflicts and allow chaining of + # extendModules. + internalModule = rec { + _file = "lib/modules.nix"; + + key = _file; + + options = { + _module.args = mkOption { + # Because things like `mkIf` are entirely useless for + # `_module.args` (because there's no way modules can check which + # arguments were passed), we'll use `lazyAttrsOf` which drops + # support for that, in turn it's lazy in its values. This means e.g. + # a `_module.args.pkgs = import (fetchTarball { ... }) {}` won't + # start a download when `pkgs` wasn't evaluated. + type = types.lazyAttrsOf types.raw; + # Only render documentation once at the root of the option tree, + # not for all individual submodules. + # Allow merging option decls to make this internal regardless. + ${if prefix == [] + then null # unset => visible + else "internal"} = true; + # TODO: hidden during the markdown transition to not expose downstream + # users of the docs infra to markdown if they're not ready for it. + # we don't make this visible conditionally because it can impact + # performance (https://github.com/NixOS/nixpkgs/pull/208407#issuecomment-1368246192) + visible = false; + # TODO: Change the type of this option to a submodule with a + # freeformType, so that individual arguments can be documented + # separately + description = lib.mdDoc '' + Additional arguments passed to each module in addition to ones + like `lib`, `config`, + and `pkgs`, `modulesPath`. + + This option is also available to all submodules. Submodules do not + inherit args from their parent module, nor do they provide args to + their parent module or sibling submodules. The sole exception to + this is the argument `name` which is provided by + parent modules to a submodule and contains the attribute name + the submodule is bound to, or a unique generated name if it is + not bound to an attribute. + + Some arguments are already passed by default, of which the + following *cannot* be changed with this option: + - {var}`lib`: The nixpkgs library. + - {var}`config`: The results of all options after merging the values from all modules together. + - {var}`options`: The options declared in all modules. + - {var}`specialArgs`: The `specialArgs` argument passed to `evalModules`. + - All attributes of {var}`specialArgs` + + Whereas option values can generally depend on other option values + thanks to laziness, this does not apply to `imports`, which + must be computed statically before anything else. + + For this reason, callers of the module system can provide `specialArgs` + which are available during import resolution. + + For NixOS, `specialArgs` includes + {var}`modulesPath`, which allows you to import + extra modules from the nixpkgs package tree without having to + somehow make the module aware of the location of the + `nixpkgs` or NixOS directories. + ``` + { modulesPath, ... }: { + imports = [ + (modulesPath + "/profiles/minimal.nix") + ]; + } + ``` + + For NixOS, the default value for this option includes at least this argument: + - {var}`pkgs`: The nixpkgs package set according to + the {option}`nixpkgs.pkgs` option. + ''; + }; + + _module.check = mkOption { + type = types.bool; + internal = true; + default = true; + description = lib.mdDoc "Whether to check whether all option definitions have matching declarations."; + }; + + _module.freeformType = mkOption { + type = types.nullOr types.optionType; + internal = true; + default = null; + description = lib.mdDoc '' + If set, merge all definitions that don't have an associated option + together using this type. The result then gets combined with the + values of all declared options to produce the final ` + config` value. + + If this is `null`, definitions without an option + will throw an error unless {option}`_module.check` is + turned off. + ''; + }; + + _module.specialArgs = mkOption { + readOnly = true; + internal = true; + description = lib.mdDoc '' + Externally provided module arguments that can't be modified from + within a configuration, but can be used in module imports. + ''; + }; + }; + + config = { + _module.args = { + inherit extendModules; + moduleType = type; + }; + _module.specialArgs = specialArgs; + }; + }; + + merged = + let collected = collectModules + class + (specialArgs.modulesPath or "") + (regularModules ++ [ internalModule ]) + ({ inherit lib options config specialArgs; } // specialArgs); + in mergeModules prefix (reverseList collected); + + options = merged.matchedOptions; + + config = + let + + # For definitions that have an associated option + declaredConfig = mapAttrsRecursiveCond (v: ! isOption v) (_: v: v.value) options; + + # If freeformType is set, this is for definitions that don't have an associated option + freeformConfig = + let + defs = map (def: { + file = def.file; + value = setAttrByPath def.prefix def.value; + }) merged.unmatchedDefns; + in if defs == [] then {} + else declaredConfig._module.freeformType.merge prefix defs; + + in if declaredConfig._module.freeformType == null then declaredConfig + # Because all definitions that had an associated option ended in + # declaredConfig, freeformConfig can only contain the non-option + # paths, meaning recursiveUpdate will never override any value + else recursiveUpdate freeformConfig declaredConfig; + + checkUnmatched = + if config._module.check && config._module.freeformType == null && merged.unmatchedDefns != [] then + let + firstDef = head merged.unmatchedDefns; + baseMsg = + let + optText = showOption (prefix ++ firstDef.prefix); + defText = + builtins.addErrorContext + "while evaluating the error message for definitions for `${optText}', which is an option that does not exist" + (builtins.addErrorContext + "while evaluating a definition from `${firstDef.file}'" + ( showDefs [ firstDef ]) + ); + in + "The option `${optText}' does not exist. Definition values:${defText}"; + in + if attrNames options == [ "_module" ] + then + let + optionName = showOption prefix; + in + if optionName == "" + then throw '' + ${baseMsg} + + It seems as if you're trying to declare an option by placing it into `config' rather than `options'! + '' + else + throw '' + ${baseMsg} + + However there are no options defined in `${showOption prefix}'. Are you sure you've + declared your options properly? This can happen if you e.g. declared your options in `types.submodule' + under `config' rather than `options'. + '' + else throw baseMsg + else null; + + checked = builtins.seq checkUnmatched; + + extendModules = extendArgs@{ + modules ? [], + specialArgs ? {}, + prefix ? [], + }: + evalModules (evalModulesArgs // { + inherit class; + modules = regularModules ++ modules; + specialArgs = evalModulesArgs.specialArgs or {} // specialArgs; + prefix = extendArgs.prefix or evalModulesArgs.prefix or []; + }); + + type = lib.types.submoduleWith { + inherit modules specialArgs class; + }; + + result = withWarnings { + _type = "configuration"; + options = checked options; + config = checked (removeAttrs config [ "_module" ]); + _module = checked (config._module); + inherit extendModules type; + class = class; + }; + in result; + + # collectModules :: (class: String) -> (modulesPath: String) -> (modules: [ Module ]) -> (args: Attrs) -> [ Module ] + # + # Collects all modules recursively through `import` statements, filtering out + # all modules in disabledModules. + collectModules = class: let + + # Like unifyModuleSyntax, but also imports paths and calls functions if necessary + loadModule = args: fallbackFile: fallbackKey: m: + if isFunction m then + unifyModuleSyntax fallbackFile fallbackKey (applyModuleArgs fallbackKey m args) + else if isAttrs m then + if m._type or "module" == "module" then + unifyModuleSyntax fallbackFile fallbackKey m + else if m._type == "if" || m._type == "override" then + loadModule args fallbackFile fallbackKey { config = m; } + else + throw ( + "Could not load a value as a module, because it is of type ${lib.strings.escapeNixString m._type}" + + lib.optionalString (fallbackFile != unknownModule) ", in file ${toString fallbackFile}." + + lib.optionalString (m._type == "configuration") " If you do intend to import this configuration, please only import the modules that make up the configuration. You may have to create a `let` binding, file or attribute to give yourself access to the relevant modules.\nWhile loading a configuration into the module system is a very sensible idea, it can not be done cleanly in practice." + # Extended explanation: That's because a finalized configuration is more than just a set of modules. For instance, it has its own `specialArgs` that, by the nature of `specialArgs` can't be loaded through `imports` or the the `modules` argument. So instead, we have to ask you to extract the relevant modules and use those instead. This way, we keep the module system comparatively simple, and hopefully avoid a bad surprise down the line. + ) + else if isList m then + let defs = [{ file = fallbackFile; value = m; }]; in + throw "Module imports can't be nested lists. Perhaps you meant to remove one level of lists? Definitions: ${showDefs defs}" + else unifyModuleSyntax (toString m) (toString m) (applyModuleArgsIfFunction (toString m) (import m) args); + + checkModule = + if class != null + then + m: + if m._class != null -> m._class == class + then m + else + throw "The module ${m._file or m.key} was imported into ${class} instead of ${m._class}." + else + m: m; + + /* + Collects all modules recursively into the form + + { + disabled = [ ]; + # All modules of the main module list + modules = [ + { + key = ; + module = ; + # All modules imported by the module for key1 + modules = [ + { + key = ; + module = ; + # All modules imported by the module for key1-1 + modules = [ ... ]; + } + ... + ]; + } + ... + ]; + } + */ + collectStructuredModules = + let + collectResults = modules: { + disabled = concatLists (catAttrs "disabled" modules); + inherit modules; + }; + in parentFile: parentKey: initialModules: args: collectResults (imap1 (n: x: + let + module = checkModule (loadModule args parentFile "${parentKey}:anon-${toString n}" x); + collectedImports = collectStructuredModules module._file module.key module.imports args; + in { + key = module.key; + module = module; + modules = collectedImports.modules; + disabled = (if module.disabledModules != [] then [{ file = module._file; disabled = module.disabledModules; }] else []) ++ collectedImports.disabled; + }) initialModules); + + # filterModules :: String -> { disabled, modules } -> [ Module ] + # + # Filters a structure as emitted by collectStructuredModules by removing all disabled + # modules recursively. It returns the final list of unique-by-key modules + filterModules = modulesPath: { disabled, modules }: + let + moduleKey = file: m: + if isString m + then + if builtins.substring 0 1 m == "/" + then m + else toString modulesPath + "/" + m + + else if isConvertibleWithToString m + then + if m?key && m.key != toString m + then + throw "Module `${file}` contains a disabledModules item that is an attribute set that can be converted to a string (${toString m}) but also has a `.key` attribute (${m.key}) with a different value. This makes it ambiguous which module should be disabled." + else + toString m + + else if m?key + then + m.key + + else if isAttrs m + then throw "Module `${file}` contains a disabledModules item that is an attribute set, presumably a module, that does not have a `key` attribute. This means that the module system doesn't have any means to identify the module that should be disabled. Make sure that you've put the correct value in disabledModules: a string path relative to modulesPath, a path value, or an attribute set with a `key` attribute." + else throw "Each disabledModules item must be a path, string, or a attribute set with a key attribute, or a value supported by toString. However, one of the disabledModules items in `${toString file}` is none of that, but is of type ${builtins.typeOf m}."; + + disabledKeys = concatMap ({ file, disabled }: map (moduleKey file) disabled) disabled; + keyFilter = filter (attrs: ! elem attrs.key disabledKeys); + in map (attrs: attrs.module) (builtins.genericClosure { + startSet = keyFilter modules; + operator = attrs: keyFilter attrs.modules; + }); + + in modulesPath: initialModules: args: + filterModules modulesPath (collectStructuredModules unknownModule "" initialModules args); + + /* Wrap a module with a default location for reporting errors. */ + setDefaultModuleLocation = file: m: + { _file = file; imports = [ m ]; }; + + /* Massage a module into canonical form, that is, a set consisting + of ‘options’, ‘config’ and ‘imports’ attributes. */ + unifyModuleSyntax = file: key: m: + let + addMeta = config: if m ? meta + then mkMerge [ config { meta = m.meta; } ] + else config; + addFreeformType = config: if m ? freeformType + then mkMerge [ config { _module.freeformType = m.freeformType; } ] + else config; + in + if m ? config || m ? options then + let badAttrs = removeAttrs m ["_class" "_file" "key" "disabledModules" "imports" "options" "config" "meta" "freeformType"]; in + if badAttrs != {} then + throw "Module `${key}' has an unsupported attribute `${head (attrNames badAttrs)}'. This is caused by introducing a top-level `config' or `options' attribute. Add configuration attributes immediately on the top level instead, or move all of them (namely: ${toString (attrNames badAttrs)}) into the explicit `config' attribute." + else + { _file = toString m._file or file; + _class = m._class or null; + key = toString m.key or key; + disabledModules = m.disabledModules or []; + imports = m.imports or []; + options = m.options or {}; + config = addFreeformType (addMeta (m.config or {})); + } + else + # shorthand syntax + lib.throwIfNot (isAttrs m) "module ${file} (${key}) does not look like a module." + { _file = toString m._file or file; + _class = m._class or null; + key = toString m.key or key; + disabledModules = m.disabledModules or []; + imports = m.require or [] ++ m.imports or []; + options = {}; + config = addFreeformType (removeAttrs m ["_class" "_file" "key" "disabledModules" "require" "imports" "freeformType"]); + }; + + applyModuleArgsIfFunction = key: f: args@{ config, options, lib, ... }: + if isFunction f then applyModuleArgs key f args else f; + + applyModuleArgs = key: f: args@{ config, options, lib, ... }: + let + # Module arguments are resolved in a strict manner when attribute set + # deconstruction is used. As the arguments are now defined with the + # config._module.args option, the strictness used on the attribute + # set argument would cause an infinite loop, if the result of the + # option is given as argument. + # + # To work-around the strictness issue on the deconstruction of the + # attributes set argument, we create a new attribute set which is + # constructed to satisfy the expected set of attributes. Thus calling + # a module will resolve strictly the attributes used as argument but + # not their values. The values are forwarding the result of the + # evaluation of the option. + context = name: ''while evaluating the module argument `${name}' in "${key}":''; + extraArgs = builtins.mapAttrs (name: _: + builtins.addErrorContext (context name) + (args.${name} or config._module.args.${name}) + ) (lib.functionArgs f); + + # Note: we append in the opposite order such that we can add an error + # context on the explicit arguments of "args" too. This update + # operator is used to make the "args@{ ... }: with args.lib;" notation + # works. + in f (args // extraArgs); + + /* Merge a list of modules. This will recurse over the option + declarations in all modules, combining them into a single set. + At the same time, for each option declaration, it will merge the + corresponding option definitions in all machines, returning them + in the ‘value’ attribute of each option. + + This returns a set like + { + # A recursive set of options along with their final values + matchedOptions = { + foo = { _type = "option"; value = "option value of foo"; ... }; + bar.baz = { _type = "option"; value = "option value of bar.baz"; ... }; + ... + }; + # A list of definitions that weren't matched by any option + unmatchedDefns = [ + { file = "file.nix"; prefix = [ "qux" ]; value = "qux"; } + ... + ]; + } + */ + mergeModules = prefix: modules: + mergeModules' prefix modules + (concatMap (m: map (config: { file = m._file; inherit config; }) (pushDownProperties m.config)) modules); + + mergeModules' = prefix: options: configs: + let + /* byName is like foldAttrs, but will look for attributes to merge in the + specified attribute name. + + byName "foo" (module: value: ["module.hidden=${module.hidden},value=${value}"]) + [ + { + hidden="baz"; + foo={qux="bar"; gla="flop";}; + } + { + hidden="fli"; + foo={qux="gne"; gli="flip";}; + } + ] + ===> + { + gla = [ "module.hidden=baz,value=flop" ]; + gli = [ "module.hidden=fli,value=flip" ]; + qux = [ "module.hidden=baz,value=bar" "module.hidden=fli,value=gne" ]; + } + */ + byName = attr: f: modules: + zipAttrsWith (n: concatLists) + (map (module: let subtree = module.${attr}; in + if !(builtins.isAttrs subtree) then + throw (if attr == "config" then '' + You're trying to define a value of type `${builtins.typeOf subtree}' + rather than an attribute set for the option + `${builtins.concatStringsSep "." prefix}'! + + This usually happens if `${builtins.concatStringsSep "." prefix}' has option + definitions inside that are not matched. Please check how to properly define + this option by e.g. referring to `man 5 configuration.nix'! + '' else '' + An option declaration for `${builtins.concatStringsSep "." prefix}' has type + `${builtins.typeOf subtree}' rather than an attribute set. + Did you mean to define this outside of `options'? + '') + else + mapAttrs (n: f module) subtree + ) modules); + # an attrset 'name' => list of submodules that declare ‘name’. + declsByName = byName "options" (module: option: + [{ inherit (module) _file; options = option; }] + ) options; + # an attrset 'name' => list of submodules that define ‘name’. + defnsByName = byName "config" (module: value: + map (config: { inherit (module) file; inherit config; }) (pushDownProperties value) + ) configs; + # extract the definitions for each loc + defnsByName' = byName "config" (module: value: + [{ inherit (module) file; inherit value; }] + ) configs; + + # Convert an option tree decl to a submodule option decl + optionTreeToOption = decl: + if isOption decl.options + then decl + else decl // { + options = mkOption { + type = types.submoduleWith { + modules = [ { options = decl.options; } ]; + # `null` is not intended for use by modules. It is an internal + # value that means "whatever the user has declared elsewhere". + # This might become obsolete with https://github.com/NixOS/nixpkgs/issues/162398 + shorthandOnlyDefinesConfig = null; + }; + }; + }; + + resultsByName = mapAttrs (name: decls: + # We're descending into attribute ‘name’. + let + loc = prefix ++ [name]; + defns = defnsByName.${name} or []; + defns' = defnsByName'.${name} or []; + optionDecls = filter (m: isOption m.options) decls; + in + if length optionDecls == length decls then + let opt = fixupOptionType loc (mergeOptionDecls loc decls); + in { + matchedOptions = evalOptionValue loc opt defns'; + unmatchedDefns = []; + } + else if optionDecls != [] then + if all (x: x.options.type.name == "submodule") optionDecls + # Raw options can only be merged into submodules. Merging into + # attrsets might be nice, but ambiguous. Suppose we have + # attrset as a `attrsOf submodule`. User declares option + # attrset.foo.bar, this could mean: + # a. option `bar` is only available in `attrset.foo` + # b. option `foo.bar` is available in all `attrset.*` + # c. reject and require "" as a reminder that it behaves like (b). + # d. magically combine (a) and (c). + # All of the above are merely syntax sugar though. + then + let opt = fixupOptionType loc (mergeOptionDecls loc (map optionTreeToOption decls)); + in { + matchedOptions = evalOptionValue loc opt defns'; + unmatchedDefns = []; + } + else + let + nonOptions = filter (m: !isOption m.options) decls; + in + throw "The option `${showOption loc}' in module `${(lib.head optionDecls)._file}' would be a parent of the following options, but its type `${(lib.head optionDecls).options.type.description or ""}' does not support nested options.\n${ + showRawDecls loc nonOptions + }" + else + mergeModules' loc decls defns) declsByName; + + matchedOptions = mapAttrs (n: v: v.matchedOptions) resultsByName; + + # an attrset 'name' => list of unmatched definitions for 'name' + unmatchedDefnsByName = + # Propagate all unmatched definitions from nested option sets + mapAttrs (n: v: v.unmatchedDefns) resultsByName + # Plus the definitions for the current prefix that don't have a matching option + // removeAttrs defnsByName' (attrNames matchedOptions); + in { + inherit matchedOptions; + + # Transforms unmatchedDefnsByName into a list of definitions + unmatchedDefns = + if configs == [] + then + # When no config values exist, there can be no unmatched config, so + # we short circuit and avoid evaluating more _options_ than necessary. + [] + else + concatLists (mapAttrsToList (name: defs: + map (def: def // { + # Set this so we know when the definition first left unmatched territory + prefix = [name] ++ (def.prefix or []); + }) defs + ) unmatchedDefnsByName); + }; + + /* Merge multiple option declarations into a single declaration. In + general, there should be only one declaration of each option. + The exception is the ‘options’ attribute, which specifies + sub-options. These can be specified multiple times to allow one + module to add sub-options to an option declared somewhere else + (e.g. multiple modules define sub-options for ‘fileSystems’). + + 'loc' is the list of attribute names where the option is located. + + 'opts' is a list of modules. Each module has an options attribute which + correspond to the definition of 'loc' in 'opt.file'. */ + mergeOptionDecls = + loc: opts: + foldl' (res: opt: + let t = res.type; + t' = opt.options.type; + mergedType = t.typeMerge t'.functor; + typesMergeable = mergedType != null; + typeSet = if (bothHave "type") && typesMergeable + then { type = mergedType; } + else {}; + bothHave = k: opt.options ? ${k} && res ? ${k}; + in + if bothHave "default" || + bothHave "example" || + bothHave "description" || + bothHave "apply" || + (bothHave "type" && (! typesMergeable)) + then + throw "The option `${showOption loc}' in `${opt._file}' is already declared in ${showFiles res.declarations}." + else + let + getSubModules = opt.options.type.getSubModules or null; + submodules = + if getSubModules != null then map (setDefaultModuleLocation opt._file) getSubModules ++ res.options + else res.options; + in opt.options // res // + { declarations = res.declarations ++ [opt._file]; + options = submodules; + } // typeSet + ) { inherit loc; declarations = []; options = []; } opts; + + /* Merge all the definitions of an option to produce the final + config value. */ + evalOptionValue = loc: opt: defs: + let + # Add in the default value for this option, if any. + defs' = + (optional (opt ? default) + { file = head opt.declarations; value = mkOptionDefault opt.default; }) ++ defs; + + # Handle properties, check types, and merge everything together. + res = + if opt.readOnly or false && length defs' > 1 then + let + # For a better error message, evaluate all readOnly definitions as + # if they were the only definition. + separateDefs = map (def: def // { + value = (mergeDefinitions loc opt.type [ def ]).mergedValue; + }) defs'; + in throw "The option `${showOption loc}' is read-only, but it's set multiple times. Definition values:${showDefs separateDefs}" + else + mergeDefinitions loc opt.type defs'; + + # Apply the 'apply' function to the merged value. This allows options to + # yield a value computed from the definitions + value = if opt ? apply then opt.apply res.mergedValue else res.mergedValue; + + warnDeprecation = + warnIf (opt.type.deprecationMessage != null) + "The type `types.${opt.type.name}' of option `${showOption loc}' defined in ${showFiles opt.declarations} is deprecated. ${opt.type.deprecationMessage}"; + + in warnDeprecation opt // + { value = builtins.addErrorContext "while evaluating the option `${showOption loc}':" value; + inherit (res.defsFinal') highestPrio; + definitions = map (def: def.value) res.defsFinal; + files = map (def: def.file) res.defsFinal; + definitionsWithLocations = res.defsFinal; + inherit (res) isDefined; + # This allows options to be correctly displayed using `${options.path.to.it}` + __toString = _: showOption loc; + }; + + # Merge definitions of a value of a given type. + mergeDefinitions = loc: type: defs: rec { + defsFinal' = + let + # Process mkMerge and mkIf properties. + defs' = concatMap (m: + map (value: { inherit (m) file; inherit value; }) (builtins.addErrorContext "while evaluating definitions from `${m.file}':" (dischargeProperties m.value)) + ) defs; + + # Process mkOverride properties. + defs'' = filterOverrides' defs'; + + # Sort mkOrder properties. + defs''' = + # Avoid sorting if we don't have to. + if any (def: def.value._type or "" == "order") defs''.values + then sortProperties defs''.values + else defs''.values; + in { + values = defs'''; + inherit (defs'') highestPrio; + }; + defsFinal = defsFinal'.values; + + # Type-check the remaining definitions, and merge them. Or throw if no definitions. + mergedValue = + if isDefined then + if all (def: type.check def.value) defsFinal then type.merge loc defsFinal + else let allInvalid = filter (def: ! type.check def.value) defsFinal; + in throw "A definition for option `${showOption loc}' is not of type `${type.description}'. Definition values:${showDefs allInvalid}" + else + # (nixos-option detects this specific error message and gives it special + # handling. If changed here, please change it there too.) + throw "The option `${showOption loc}' is used but not defined."; + + isDefined = defsFinal != []; + + optionalValue = + if isDefined then { value = mergedValue; } + else {}; + }; + + /* Given a config set, expand mkMerge properties, and push down the + other properties into the children. The result is a list of + config sets that do not have properties at top-level. For + example, + + mkMerge [ { boot = set1; } (mkIf cond { boot = set2; services = set3; }) ] + + is transformed into + + [ { boot = set1; } { boot = mkIf cond set2; services = mkIf cond set3; } ]. + + This transform is the critical step that allows mkIf conditions + to refer to the full configuration without creating an infinite + recursion. + */ + pushDownProperties = cfg: + if cfg._type or "" == "merge" then + concatMap pushDownProperties cfg.contents + else if cfg._type or "" == "if" then + map (mapAttrs (n: v: mkIf cfg.condition v)) (pushDownProperties cfg.content) + else if cfg._type or "" == "override" then + map (mapAttrs (n: v: mkOverride cfg.priority v)) (pushDownProperties cfg.content) + else # FIXME: handle mkOrder? + [ cfg ]; + + /* Given a config value, expand mkMerge properties, and discharge + any mkIf conditions. That is, this is the place where mkIf + conditions are actually evaluated. The result is a list of + config values. For example, ‘mkIf false x’ yields ‘[]’, + ‘mkIf true x’ yields ‘[x]’, and + + mkMerge [ 1 (mkIf true 2) (mkIf true (mkIf false 3)) ] + + yields ‘[ 1 2 ]’. + */ + dischargeProperties = def: + if def._type or "" == "merge" then + concatMap dischargeProperties def.contents + else if def._type or "" == "if" then + if isBool def.condition then + if def.condition then + dischargeProperties def.content + else + [ ] + else + throw "‘mkIf’ called with a non-Boolean condition" + else + [ def ]; + + /* Given a list of config values, process the mkOverride properties, + that is, return the values that have the highest (that is, + numerically lowest) priority, and strip the mkOverride + properties. For example, + + [ { file = "/1"; value = mkOverride 10 "a"; } + { file = "/2"; value = mkOverride 20 "b"; } + { file = "/3"; value = "z"; } + { file = "/4"; value = mkOverride 10 "d"; } + ] + + yields + + [ { file = "/1"; value = "a"; } + { file = "/4"; value = "d"; } + ] + + Note that "z" has the default priority 100. + */ + filterOverrides = defs: (filterOverrides' defs).values; + + filterOverrides' = defs: + let + getPrio = def: if def.value._type or "" == "override" then def.value.priority else defaultOverridePriority; + highestPrio = foldl' (prio: def: min (getPrio def) prio) 9999 defs; + strip = def: if def.value._type or "" == "override" then def // { value = def.value.content; } else def; + in { + values = concatMap (def: if getPrio def == highestPrio then [(strip def)] else []) defs; + inherit highestPrio; + }; + + /* Sort a list of properties. The sort priority of a property is + defaultOrderPriority by default, but can be overridden by wrapping the property + using mkOrder. */ + sortProperties = defs: + let + strip = def: + if def.value._type or "" == "order" + then def // { value = def.value.content; inherit (def.value) priority; } + else def; + defs' = map strip defs; + compare = a: b: (a.priority or defaultOrderPriority) < (b.priority or defaultOrderPriority); + in sort compare defs'; + + # This calls substSubModules, whose entire purpose is only to ensure that + # option declarations in submodules have accurate position information. + # TODO: Merge this into mergeOptionDecls + fixupOptionType = loc: opt: + if opt.type.getSubModules or null == null + then opt // { type = opt.type or types.unspecified; } + else opt // { type = opt.type.substSubModules opt.options; options = []; }; + + + /* Properties. */ + + mkIf = condition: content: + { _type = "if"; + inherit condition content; + }; + + mkAssert = assertion: message: content: + mkIf + (if assertion then true else throw "\nFailed assertion: ${message}") + content; + + mkMerge = contents: + { _type = "merge"; + inherit contents; + }; + + mkOverride = priority: content: + { _type = "override"; + inherit priority content; + }; + + mkOptionDefault = mkOverride 1500; # priority of option defaults + mkDefault = mkOverride 1000; # used in config sections of non-user modules to set a default + defaultOverridePriority = 100; + mkImageMediaOverride = mkOverride 60; # image media profiles can be derived by inclusion into host config, hence needing to override host config, but do allow user to mkForce + mkForce = mkOverride 50; + mkVMOverride = mkOverride 10; # used by ‘nixos-rebuild build-vm’ + + defaultPriority = lib.warnIf (lib.isInOldestRelease 2305) "lib.modules.defaultPriority is deprecated, please use lib.modules.defaultOverridePriority instead." defaultOverridePriority; + + mkFixStrictness = lib.warn "lib.mkFixStrictness has no effect and will be removed. It returns its argument unmodified, so you can just remove any calls." id; + + mkOrder = priority: content: + { _type = "order"; + inherit priority content; + }; + + mkBefore = mkOrder 500; + defaultOrderPriority = 1000; + mkAfter = mkOrder 1500; + + # Convenient property used to transfer all definitions and their + # properties from one option to another. This property is useful for + # renaming options, and also for including properties from another module + # system, including sub-modules. + # + # { config, options, ... }: + # + # { + # # 'bar' might not always be defined in the current module-set. + # config.foo.enable = mkAliasDefinitions (options.bar.enable or {}); + # + # # 'barbaz' has to be defined in the current module-set. + # config.foobar.paths = mkAliasDefinitions options.barbaz.paths; + # } + # + # Note, this is different than taking the value of the option and using it + # as a definition, as the new definition will not keep the mkOverride / + # mkDefault properties of the previous option. + # + mkAliasDefinitions = mkAliasAndWrapDefinitions id; + mkAliasAndWrapDefinitions = wrap: option: + mkAliasIfDef option (wrap (mkMerge option.definitions)); + + # Similar to mkAliasAndWrapDefinitions but copies over the priority from the + # option as well. + # + # If a priority is not set, it assumes a priority of defaultOverridePriority. + mkAliasAndWrapDefsWithPriority = wrap: option: + let + prio = option.highestPrio or defaultOverridePriority; + defsWithPrio = map (mkOverride prio) option.definitions; + in mkAliasIfDef option (wrap (mkMerge defsWithPrio)); + + mkAliasIfDef = option: + mkIf (isOption option && option.isDefined); + + /* Compatibility. */ + fixMergeModules = modules: args: evalModules { inherit modules args; check = false; }; + + + /* Return a module that causes a warning to be shown if the + specified option is defined. For example, + + mkRemovedOptionModule [ "boot" "loader" "grub" "bootDevice" ] "" + + causes a assertion if the user defines boot.loader.grub.bootDevice. + + replacementInstructions is a string that provides instructions on + how to achieve the same functionality without the removed option, + or alternatively a reasoning why the functionality is not needed. + replacementInstructions SHOULD be provided! + */ + mkRemovedOptionModule = optionName: replacementInstructions: + { options, ... }: + { options = setAttrByPath optionName (mkOption { + visible = false; + apply = x: throw "The option `${showOption optionName}' can no longer be used since it's been removed. ${replacementInstructions}"; + }); + config.assertions = + let opt = getAttrFromPath optionName options; in [{ + assertion = !opt.isDefined; + message = '' + The option definition `${showOption optionName}' in ${showFiles opt.files} no longer has any effect; please remove it. + ${replacementInstructions} + ''; + }]; + }; + + /* Return a module that causes a warning to be shown if the + specified "from" option is defined; the defined value is however + forwarded to the "to" option. This can be used to rename options + while providing backward compatibility. For example, + + mkRenamedOptionModule [ "boot" "copyKernels" ] [ "boot" "loader" "grub" "copyKernels" ] + + forwards any definitions of boot.copyKernels to + boot.loader.grub.copyKernels while printing a warning. + + This also copies over the priority from the aliased option to the + non-aliased option. + */ + mkRenamedOptionModule = from: to: doRename { + inherit from to; + visible = false; + warn = true; + use = builtins.trace "Obsolete option `${showOption from}' is used. It was renamed to `${showOption to}'."; + }; + + mkRenamedOptionModuleWith = { + /* Old option path as list of strings. */ + from, + /* New option path as list of strings. */ + to, + + /* + Release number of the first release that contains the rename, ignoring backports. + Set it to the upcoming release, matching the nixpkgs/.version file. + */ + sinceRelease, + + }: doRename { + inherit from to; + visible = false; + warn = lib.isInOldestRelease sinceRelease; + use = lib.warnIf (lib.isInOldestRelease sinceRelease) + "Obsolete option `${showOption from}' is used. It was renamed to `${showOption to}'."; + }; + + /* Return a module that causes a warning to be shown if any of the "from" + option is defined; the defined values can be used in the "mergeFn" to set + the "to" value. + This function can be used to merge multiple options into one that has a + different type. + + "mergeFn" takes the module "config" as a parameter and must return a value + of "to" option type. + + mkMergedOptionModule + [ [ "a" "b" "c" ] + [ "d" "e" "f" ] ] + [ "x" "y" "z" ] + (config: + let value = p: getAttrFromPath p config; + in + if (value [ "a" "b" "c" ]) == true then "foo" + else if (value [ "d" "e" "f" ]) == true then "bar" + else "baz") + + - options.a.b.c is a removed boolean option + - options.d.e.f is a removed boolean option + - options.x.y.z is a new str option that combines a.b.c and d.e.f + functionality + + This show a warning if any a.b.c or d.e.f is set, and set the value of + x.y.z to the result of the merge function + */ + mkMergedOptionModule = from: to: mergeFn: + { config, options, ... }: + { + options = foldl' recursiveUpdate {} (map (path: setAttrByPath path (mkOption { + visible = false; + # To use the value in mergeFn without triggering errors + default = "_mkMergedOptionModule"; + })) from); + + config = { + warnings = filter (x: x != "") (map (f: + let val = getAttrFromPath f config; + opt = getAttrFromPath f options; + in + optionalString + (val != "_mkMergedOptionModule") + "The option `${showOption f}' defined in ${showFiles opt.files} has been changed to `${showOption to}' that has a different type. Please read `${showOption to}' documentation and update your configuration accordingly." + ) from); + } // setAttrByPath to (mkMerge + (optional + (any (f: (getAttrFromPath f config) != "_mkMergedOptionModule") from) + (mergeFn config))); + }; + + /* Single "from" version of mkMergedOptionModule. + Return a module that causes a warning to be shown if the "from" option is + defined; the defined value can be used in the "mergeFn" to set the "to" + value. + This function can be used to change an option into another that has a + different type. + + "mergeFn" takes the module "config" as a parameter and must return a value of + "to" option type. + + mkChangedOptionModule [ "a" "b" "c" ] [ "x" "y" "z" ] + (config: + let value = getAttrFromPath [ "a" "b" "c" ] config; + in + if value > 100 then "high" + else "normal") + + - options.a.b.c is a removed int option + - options.x.y.z is a new str option that supersedes a.b.c + + This show a warning if a.b.c is set, and set the value of x.y.z to the + result of the change function + */ + mkChangedOptionModule = from: to: changeFn: + mkMergedOptionModule [ from ] to changeFn; + + /* Like ‘mkRenamedOptionModule’, but doesn't show a warning. */ + mkAliasOptionModule = from: to: doRename { + inherit from to; + visible = true; + warn = false; + use = id; + }; + + /* Transitional version of mkAliasOptionModule that uses MD docs. */ + mkAliasOptionModuleMD = from: to: doRename { + inherit from to; + visible = true; + warn = false; + use = id; + markdown = true; + }; + + /* mkDerivedConfig : Option a -> (a -> Definition b) -> Definition b + + Create config definitions with the same priority as the definition of another option. + This should be used for option definitions where one option sets the value of another as a convenience. + For instance a config file could be set with a `text` or `source` option, where text translates to a `source` + value using `mkDerivedConfig options.text (pkgs.writeText "filename.conf")`. + + It takes care of setting the right priority using `mkOverride`. + */ + # TODO: make the module system error message include information about `opt` in + # error messages about conflicts. E.g. introduce a variation of `mkOverride` which + # adds extra location context to the definition object. This will allow context to be added + # to all messages that report option locations "this value was derived from + # which was defined in ". It can provide a trace of options that contributed + # to definitions. + mkDerivedConfig = opt: f: + mkOverride + (opt.highestPrio or defaultOverridePriority) + (f opt.value); + + doRename = { from, to, visible, warn, use, withPriority ? true, markdown ? false }: + { config, options, ... }: + let + fromOpt = getAttrFromPath from options; + toOf = attrByPath to + (abort "Renaming error: option `${showOption to}' does not exist."); + toType = let opt = attrByPath to {} options; in opt.type or (types.submodule {}); + in + { + options = setAttrByPath from (mkOption { + inherit visible; + description = if markdown + then lib.mdDoc "Alias of {option}`${showOption to}`." + else "Alias of ."; + apply = x: use (toOf config); + } // optionalAttrs (toType != null) { + type = toType; + }); + config = mkMerge [ + (optionalAttrs (options ? warnings) { + warnings = optional (warn && fromOpt.isDefined) + "The option `${showOption from}' defined in ${showFiles fromOpt.files} has been renamed to `${showOption to}'."; + }) + (if withPriority + then mkAliasAndWrapDefsWithPriority (setAttrByPath to) fromOpt + else mkAliasAndWrapDefinitions (setAttrByPath to) fromOpt) + ]; + }; + + /* Use this function to import a JSON file as NixOS configuration. + + modules.importJSON :: path -> attrs + */ + importJSON = file: { + _file = file; + config = lib.importJSON file; + }; + + /* Use this function to import a TOML file as NixOS configuration. + + modules.importTOML :: path -> attrs + */ + importTOML = file: { + _file = file; + config = lib.importTOML file; + }; + + private = lib.mapAttrs + (k: lib.warn "External use of `lib.modules.${k}` is deprecated. If your use case isn't covered by non-deprecated functions, we'd like to know more and perhaps support your use case well, instead of providing access to these low level functions. In this case please open an issue in https://github.com/nixos/nixpkgs/issues/.") + { + inherit + applyModuleArgsIfFunction + dischargeProperties + evalOptionValue + mergeModules + mergeModules' + pushDownProperties + unifyModuleSyntax + ; + collectModules = collectModules null; + }; + +in +private // +{ + # NOTE: not all of these functions are necessarily public interfaces; some + # are just needed by types.nix, but are not meant to be consumed + # externally. + inherit + defaultOrderPriority + defaultOverridePriority + defaultPriority + doRename + evalModules + filterOverrides + filterOverrides' + fixMergeModules + fixupOptionType # should be private? + importJSON + importTOML + mergeDefinitions + mergeOptionDecls # should be private? + mkAfter + mkAliasAndWrapDefinitions + mkAliasAndWrapDefsWithPriority + mkAliasDefinitions + mkAliasIfDef + mkAliasOptionModule + mkAliasOptionModuleMD + mkAssert + mkBefore + mkChangedOptionModule + mkDefault + mkDerivedConfig + mkFixStrictness + mkForce + mkIf + mkImageMediaOverride + mkMerge + mkMergedOptionModule + mkOptionDefault + mkOrder + mkOverride + mkRemovedOptionModule + mkRenamedOptionModule + mkRenamedOptionModuleWith + mkVMOverride + setDefaultModuleLocation + sortProperties; +} diff --git a/options.nix b/options.nix new file mode 100644 index 000000000..af7914bb5 --- /dev/null +++ b/options.nix @@ -0,0 +1,432 @@ +# Nixpkgs/NixOS option handling. +{ lib }: + +let + inherit (lib) + all + collect + concatLists + concatMap + concatMapStringsSep + filter + foldl' + head + tail + isAttrs + isBool + isDerivation + isFunction + isInt + isList + isString + length + mapAttrs + optional + optionals + take + ; + inherit (lib.attrsets) + attrByPath + optionalAttrs + ; + inherit (lib.strings) + concatMapStrings + concatStringsSep + ; + inherit (lib.types) + mkOptionType + ; + inherit (lib.lists) + last + ; + prioritySuggestion = '' + Use `lib.mkForce value` or `lib.mkDefault value` to change the priority on any of these definitions. + ''; +in +rec { + + /* Returns true when the given argument is an option + + Type: isOption :: a -> bool + + Example: + isOption 1 // => false + isOption (mkOption {}) // => true + */ + isOption = lib.isType "option"; + + /* Creates an Option attribute set. mkOption accepts an attribute set with the following keys: + + All keys default to `null` when not given. + + Example: + mkOption { } // => { _type = "option"; } + mkOption { default = "foo"; } // => { _type = "option"; default = "foo"; } + */ + mkOption = + { + # Default value used when no definition is given in the configuration. + default ? null, + # Textual representation of the default, for the manual. + defaultText ? null, + # Example value used in the manual. + example ? null, + # String describing the option. + description ? null, + # Related packages used in the manual (see `genRelatedPackages` in ../nixos/lib/make-options-doc/default.nix). + relatedPackages ? null, + # Option type, providing type-checking and value merging. + type ? null, + # Function that converts the option value to something else. + apply ? null, + # Whether the option is for NixOS developers only. + internal ? null, + # Whether the option shows up in the manual. Default: true. Use false to hide the option and any sub-options from submodules. Use "shallow" to hide only sub-options. + visible ? null, + # Whether the option can be set only once + readOnly ? null, + } @ attrs: + attrs // { _type = "option"; }; + + /* Creates an Option attribute set for a boolean value option i.e an + option to be toggled on or off: + + Example: + mkEnableOption "foo" + => { _type = "option"; default = false; description = "Whether to enable foo."; example = true; type = { ... }; } + */ + mkEnableOption = + # Name for the created option + name: mkOption { + default = false; + example = true; + description = + if name ? _type && name._type == "mdDoc" + then lib.mdDoc "Whether to enable ${name.text}." + else "Whether to enable ${name}."; + type = lib.types.bool; + }; + + /* Creates an Option attribute set for an option that specifies the + package a module should use for some purpose. + + The package is specified in the third argument under `default` as a list of strings + representing its attribute path in nixpkgs (or another package set). + Because of this, you need to pass nixpkgs itself (or a subset) as the first argument. + + The second argument may be either a string or a list of strings. + It provides the display name of the package in the description of the generated option + (using only the last element if the passed value is a list) + and serves as the fallback value for the `default` argument. + + To include extra information in the description, pass `extraDescription` to + append arbitrary text to the generated description. + You can also pass an `example` value, either a literal string or an attribute path. + + The default argument can be omitted if the provided name is + an attribute of pkgs (if name is a string) or a + valid attribute path in pkgs (if name is a list). + + If you wish to explicitly provide no default, pass `null` as `default`. + + Type: mkPackageOption :: pkgs -> (string|[string]) -> { default? :: [string], example? :: null|string|[string], extraDescription? :: string } -> option + + Example: + mkPackageOption pkgs "hello" { } + => { _type = "option"; default = «derivation /nix/store/3r2vg51hlxj3cx5vscp0vkv60bqxkaq0-hello-2.10.drv»; defaultText = { ... }; description = "The hello package to use."; type = { ... }; } + + Example: + mkPackageOption pkgs "GHC" { + default = [ "ghc" ]; + example = "pkgs.haskell.packages.ghc92.ghc.withPackages (hkgs: [ hkgs.primes ])"; + } + => { _type = "option"; default = «derivation /nix/store/jxx55cxsjrf8kyh3fp2ya17q99w7541r-ghc-8.10.7.drv»; defaultText = { ... }; description = "The GHC package to use."; example = { ... }; type = { ... }; } + + Example: + mkPackageOption pkgs [ "python39Packages" "pytorch" ] { + extraDescription = "This is an example and doesn't actually do anything."; + } + => { _type = "option"; default = «derivation /nix/store/gvqgsnc4fif9whvwd9ppa568yxbkmvk8-python3.9-pytorch-1.10.2.drv»; defaultText = { ... }; description = "The pytorch package to use. This is an example and doesn't actually do anything."; type = { ... }; } + + */ + mkPackageOption = + # Package set (a specific version of nixpkgs or a subset) + pkgs: + # Name for the package, shown in option description + name: + { + # Whether the package can be null, for example to disable installing a package altogether. + nullable ? false, + # The attribute path where the default package is located (may be omitted) + default ? name, + # A string or an attribute path to use as an example (may be omitted) + example ? null, + # Additional text to include in the option description (may be omitted) + extraDescription ? "", + }: + let + name' = if isList name then last name else name; + in mkOption ({ + type = with lib.types; (if nullable then nullOr else lib.id) package; + description = "The ${name'} package to use." + + (if extraDescription == "" then "" else " ") + extraDescription; + } // (if default != null then let + default' = if isList default then default else [ default ]; + defaultPath = concatStringsSep "." default'; + defaultValue = attrByPath default' + (throw "${defaultPath} cannot be found in pkgs") pkgs; + in { + default = defaultValue; + defaultText = literalExpression ("pkgs." + defaultPath); + } else if nullable then { + default = null; + } else { }) // lib.optionalAttrs (example != null) { + example = literalExpression + (if isList example then "pkgs." + concatStringsSep "." example else example); + }); + + /* Like mkPackageOption, but emit an mdDoc description instead of DocBook. */ + mkPackageOptionMD = pkgs: name: extra: + let option = mkPackageOption pkgs name extra; + in option // { description = lib.mdDoc option.description; }; + + /* This option accepts anything, but it does not produce any result. + + This is useful for sharing a module across different module sets + without having to implement similar features as long as the + values of the options are not accessed. */ + mkSinkUndeclaredOptions = attrs: mkOption ({ + internal = true; + visible = false; + default = false; + description = "Sink for option definitions."; + type = mkOptionType { + name = "sink"; + check = x: true; + merge = loc: defs: false; + }; + apply = x: throw "Option value is not readable because the option is not declared."; + } // attrs); + + mergeDefaultOption = loc: defs: + let list = getValues defs; in + if length list == 1 then head list + else if all isFunction list then x: mergeDefaultOption loc (map (f: f x) list) + else if all isList list then concatLists list + else if all isAttrs list then foldl' lib.mergeAttrs {} list + else if all isBool list then foldl' lib.or false list + else if all isString list then lib.concatStrings list + else if all isInt list && all (x: x == head list) list then head list + else throw "Cannot merge definitions of `${showOption loc}'. Definition values:${showDefs defs}"; + + mergeOneOption = mergeUniqueOption { message = ""; }; + + mergeUniqueOption = { message }: loc: defs: + if length defs == 1 + then (head defs).value + else assert length defs > 1; + throw "The option `${showOption loc}' is defined multiple times while it's expected to be unique.\n${message}\nDefinition values:${showDefs defs}\n${prioritySuggestion}"; + + /* "Merge" option definitions by checking that they all have the same value. */ + mergeEqualOption = loc: defs: + if defs == [] then abort "This case should never happen." + # Return early if we only have one element + # This also makes it work for functions, because the foldl' below would try + # to compare the first element with itself, which is false for functions + else if length defs == 1 then (head defs).value + else (foldl' (first: def: + if def.value != first.value then + throw "The option `${showOption loc}' has conflicting definition values:${showDefs [ first def ]}\n${prioritySuggestion}" + else + first) (head defs) (tail defs)).value; + + /* Extracts values of all "value" keys of the given list. + + Type: getValues :: [ { value :: a; } ] -> [a] + + Example: + getValues [ { value = 1; } { value = 2; } ] // => [ 1 2 ] + getValues [ ] // => [ ] + */ + getValues = map (x: x.value); + + /* Extracts values of all "file" keys of the given list + + Type: getFiles :: [ { file :: a; } ] -> [a] + + Example: + getFiles [ { file = "file1"; } { file = "file2"; } ] // => [ "file1" "file2" ] + getFiles [ ] // => [ ] + */ + getFiles = map (x: x.file); + + # Generate documentation template from the list of option declaration like + # the set generated with filterOptionSets. + optionAttrSetToDocList = optionAttrSetToDocList' []; + + optionAttrSetToDocList' = _: options: + concatMap (opt: + let + name = showOption opt.loc; + docOption = { + loc = opt.loc; + inherit name; + description = opt.description or null; + declarations = filter (x: x != unknownModule) opt.declarations; + internal = opt.internal or false; + visible = + if (opt?visible && opt.visible == "shallow") + then true + else opt.visible or true; + readOnly = opt.readOnly or false; + type = opt.type.description or "unspecified"; + } + // optionalAttrs (opt ? example) { + example = + builtins.addErrorContext "while evaluating the example of option `${name}`" ( + renderOptionValue opt.example + ); + } + // optionalAttrs (opt ? defaultText || opt ? default) { + default = + builtins.addErrorContext "while evaluating the ${if opt?defaultText then "defaultText" else "default value"} of option `${name}`" ( + renderOptionValue (opt.defaultText or opt.default) + ); + } + // optionalAttrs (opt ? relatedPackages && opt.relatedPackages != null) { inherit (opt) relatedPackages; }; + + subOptions = + let ss = opt.type.getSubOptions opt.loc; + in if ss != {} then optionAttrSetToDocList' opt.loc ss else []; + subOptionsVisible = docOption.visible && opt.visible or null != "shallow"; + in + # To find infinite recursion in NixOS option docs: + # builtins.trace opt.loc + [ docOption ] ++ optionals subOptionsVisible subOptions) (collect isOption options); + + + /* This function recursively removes all derivation attributes from + `x` except for the `name` attribute. + + This is to make the generation of `options.xml` much more + efficient: the XML representation of derivations is very large + (on the order of megabytes) and is not actually used by the + manual generator. + + This function was made obsolete by renderOptionValue and is kept for + compatibility with out-of-tree code. + */ + scrubOptionValue = x: + if isDerivation x then + { type = "derivation"; drvPath = x.name; outPath = x.name; name = x.name; } + else if isList x then map scrubOptionValue x + else if isAttrs x then mapAttrs (n: v: scrubOptionValue v) (removeAttrs x ["_args"]) + else x; + + + /* Ensures that the given option value (default or example) is a `_type`d string + by rendering Nix values to `literalExpression`s. + */ + renderOptionValue = v: + if v ? _type && v ? text then v + else literalExpression (lib.generators.toPretty { + multiline = true; + allowPrettyValues = true; + } v); + + + /* For use in the `defaultText` and `example` option attributes. Causes the + given string to be rendered verbatim in the documentation as Nix code. This + is necessary for complex values, e.g. functions, or values that depend on + other values or packages. + */ + literalExpression = text: + if ! isString text then throw "literalExpression expects a string." + else { _type = "literalExpression"; inherit text; }; + + literalExample = lib.warn "literalExample is deprecated, use literalExpression instead, or use literalDocBook for a non-Nix description." literalExpression; + + + /* For use in the `defaultText` and `example` option attributes. Causes the + given DocBook text to be inserted verbatim in the documentation, for when + a `literalExpression` would be too hard to read. + */ + literalDocBook = text: + if ! isString text then throw "literalDocBook expects a string." + else + lib.warnIf (lib.isInOldestRelease 2211) + "literalDocBook is deprecated, use literalMD instead" + { _type = "literalDocBook"; inherit text; }; + + /* Transition marker for documentation that's already migrated to markdown + syntax. + */ + mdDoc = text: + if ! isString text then throw "mdDoc expects a string." + else { _type = "mdDoc"; inherit text; }; + + /* For use in the `defaultText` and `example` option attributes. Causes the + given MD text to be inserted verbatim in the documentation, for when + a `literalExpression` would be too hard to read. + */ + literalMD = text: + if ! isString text then throw "literalMD expects a string." + else { _type = "literalMD"; inherit text; }; + + # Helper functions. + + /* Convert an option, described as a list of the option parts to a + human-readable version. + + Example: + (showOption ["foo" "bar" "baz"]) == "foo.bar.baz" + (showOption ["foo" "bar.baz" "tux"]) == "foo.\"bar.baz\".tux" + (showOption ["windowManager" "2bwm" "enable"]) == "windowManager.\"2bwm\".enable" + + Placeholders will not be quoted as they are not actual values: + (showOption ["foo" "*" "bar"]) == "foo.*.bar" + (showOption ["foo" "" "bar"]) == "foo..bar" + */ + showOption = parts: let + escapeOptionPart = part: + let + # We assume that these are "special values" and not real configuration data. + # If it is real configuration data, it is rendered incorrectly. + specialIdentifiers = [ + "" # attrsOf (submodule {}) + "*" # listOf (submodule {}) + "" # functionTo + ]; + in if builtins.elem part specialIdentifiers + then part + else lib.strings.escapeNixIdentifier part; + in (concatStringsSep ".") (map escapeOptionPart parts); + showFiles = files: concatStringsSep " and " (map (f: "`${f}'") files); + + showDefs = defs: concatMapStrings (def: + let + # Pretty print the value for display, if successful + prettyEval = builtins.tryEval + (lib.generators.toPretty { } + (lib.generators.withRecursion { depthLimit = 10; throwOnDepthLimit = false; } def.value)); + # Split it into its lines + lines = filter (v: ! isList v) (builtins.split "\n" prettyEval.value); + # Only display the first 5 lines, and indent them for better visibility + value = concatStringsSep "\n " (take 5 lines ++ optional (length lines > 5) "..."); + result = + # Don't print any value if evaluating the value strictly fails + if ! prettyEval.success then "" + # Put it on a new line if it consists of multiple + else if length lines > 1 then ":\n " + value + else ": " + value; + in "\n- In `${def.file}'${result}" + ) defs; + + showOptionWithDefLocs = opt: '' + ${showOption opt.loc}, with values defined in: + ${concatMapStringsSep "\n" (defFile: " - ${defFile}") opt.files} + ''; + + unknownModule = ""; + +} diff --git a/path/README.md b/path/README.md new file mode 100644 index 000000000..87e552d12 --- /dev/null +++ b/path/README.md @@ -0,0 +1,196 @@ +# Path library + +This document explains why the `lib.path` library is designed the way it is. + +The purpose of this library is to process [filesystem paths]. It does not read files from the filesystem. +It exists to support the native Nix [path value type] with extra functionality. + +[filesystem paths]: https://en.m.wikipedia.org/wiki/Path_(computing) +[path value type]: https://nixos.org/manual/nix/stable/language/values.html#type-path + +As an extension of the path value type, it inherits the same intended use cases and limitations: +- Only use paths to access files at evaluation time, such as the local project source. +- Paths cannot point to derivations, so they are unfit to represent dependencies. +- A path implicitly imports the referenced files into the Nix store when interpolated to a string. Therefore paths are not suitable to access files at build- or run-time, as you risk importing the path from the evaluation system instead. + +Overall, this library works with two types of paths: +- Absolute paths are represented with the Nix [path value type]. Nix automatically normalises these paths. +- Subpaths are represented with the [string value type] since path value types don't support relative paths. This library normalises these paths as safely as possible. Absolute paths in strings are not supported. + + A subpath refers to a specific file or directory within an absolute base directory. + It is a stricter form of a relative path, notably [without support for `..` components][parents] since those could escape the base directory. + +[string value type]: https://nixos.org/manual/nix/stable/language/values.html#type-string + +This library is designed to be as safe and intuitive as possible, throwing errors when operations are attempted that would produce surprising results, and giving the expected result otherwise. + +This library is designed to work well as a dependency for the `lib.filesystem` and `lib.sources` library components. Contrary to these library components, `lib.path` does not read any paths from the filesystem. + +This library makes only these assumptions about paths and no others: +- `dirOf path` returns the path to the parent directory of `path`, unless `path` is the filesystem root, in which case `path` is returned. + - There can be multiple filesystem roots: `p == dirOf p` and `q == dirOf q` does not imply `p == q`. + - While there's only a single filesystem root in stable Nix, the [lazy trees feature](https://github.com/NixOS/nix/pull/6530) introduces [additional filesystem roots](https://github.com/NixOS/nix/pull/6530#discussion_r1041442173). +- `path + ("/" + string)` returns the path to the `string` subdirectory in `path`. + - If `string` contains no `/` characters, then `dirOf (path + ("/" + string)) == path`. + - If `string` contains no `/` characters, then `baseNameOf (path + ("/" + string)) == string`. +- `path1 == path2` returns `true` only if `path1` points to the same filesystem path as `path2`. + +Notably we do not make the assumption that we can turn paths into strings using `toString path`. + +## Design decisions + +Each subsection here contains a decision along with arguments and counter-arguments for (+) and against (-) that decision. + +### Leading dots for relative paths +[leading-dots]: #leading-dots-for-relative-paths + +Observing: Since subpaths are a form of relative paths, they can have a leading `./` to indicate it being a relative path, this is generally not necessary for tools though. + +Considering: Paths should be as explicit, consistent and unambiguous as possible. + +Decision: Returned subpaths should always have a leading `./`. + +
+Arguments + +- (+) In shells, just running `foo` as a command wouldn't execute the file `foo`, whereas `./foo` would execute the file. In contrast, `foo/bar` does execute that file without the need for `./`. This can lead to confusion about when a `./` needs to be prefixed. If a `./` is always included, this becomes a non-issue. This effectively then means that paths don't overlap with command names. +- (+) Prepending with `./` makes the subpaths always valid as relative Nix path expressions. +- (+) Using paths in command line arguments could give problems if not escaped properly, e.g. if a path was `--version`. This is not a problem with `./--version`. This effectively then means that paths don't overlap with GNU-style command line options. +- (-) `./` is not required to resolve relative paths, resolution always has an implicit `./` as prefix. +- (-) It's less noisy without the `./`, e.g. in error messages. + - (+) But similarly, it could be confusing whether something was even a path. + e.g. `foo` could be anything, but `./foo` is more clearly a path. +- (+) Makes it more uniform with absolute paths (those always start with `/`). + - (-) That is not relevant for practical purposes. +- (+) `find` also outputs results with `./`. + - (-) But only if you give it an argument of `.`. If you give it the argument `some-directory`, it won't prefix that. +- (-) `realpath --relative-to` doesn't prefix relative paths with `./`. + - (+) There is no need to return the same result as `realpath`. + +
+ +### Representation of the current directory +[curdir]: #representation-of-the-current-directory + +Observing: The subpath that produces the base directory can be represented with `.` or `./` or `./.`. + +Considering: Paths should be as consistent and unambiguous as possible. + +Decision: It should be `./.`. + +
+Arguments + +- (+) `./` would be inconsistent with [the decision to not persist trailing slashes][trailing-slashes]. +- (-) `.` is how `realpath` normalises paths. +- (+) `.` can be interpreted as a shell command (it's a builtin for sourcing files in `bash` and `zsh`). +- (+) `.` would be the only path without a `/`. It could not be used as a Nix path expression, since those require at least one `/` to be parsed as such. +- (-) `./.` is rather long. + - (-) We don't require users to type this though, as it's only output by the library. + As inputs all three variants are supported for subpaths (and we can't do anything about absolute paths) +- (-) `builtins.dirOf "foo" == "."`, so `.` would be consistent with that. +- (+) `./.` is consistent with the [decision to have leading `./`][leading-dots]. +- (+) `./.` is a valid Nix path expression, although this property does not hold for every relative path or subpath. + +
+ +### Subpath representation +[relrepr]: #subpath-representation + +Observing: Subpaths such as `foo/bar` can be represented in various ways: +- string: `"foo/bar"` +- list with all the components: `[ "foo" "bar" ]` +- attribute set: `{ type = "relative-path"; components = [ "foo" "bar" ]; }` + +Considering: Paths should be as safe to use as possible. We should generate string outputs in the library and not encourage users to do that themselves. + +Decision: Paths are represented as strings. + +
+Arguments + +- (+) It's simpler for the users of the library. One doesn't have to convert a path a string before it can be used. + - (+) Naively converting the list representation to a string with `concatStringsSep "/"` would break for `[]`, requiring library users to be more careful. +- (+) It doesn't encourage people to do their own path processing and instead use the library. + With a list representation it would seem easy to just use `lib.lists.init` to get the parent directory, but then it breaks for `.`, which would be represented as `[ ]`. +- (+) `+` is convenient and doesn't work on lists and attribute sets. + - (-) Shouldn't use `+` anyways, we export safer functions for path manipulation. + +
+ +### Parent directory +[parents]: #parent-directory + +Observing: Relative paths can have `..` components, which refer to the parent directory. + +Considering: Paths should be as safe and unambiguous as possible. + +Decision: `..` path components in string paths are not supported, neither as inputs nor as outputs. Hence, string paths are called subpaths, rather than relative paths. + +
+Arguments + +- (+) If we wanted relative paths to behave according to the "physical" interpretation (as a directory tree with relations between nodes), it would require resolving symlinks, since e.g. `foo/..` would not be the same as `.` if `foo` is a symlink. + - (-) The "logical" interpretation is also valid (treating paths as a sequence of names), and is used by some software. It is simpler, and not using symlinks at all is safer. + - (+) Mixing both models can lead to surprises. + - (+) We can't resolve symlinks without filesystem access. + - (+) Nix also doesn't support reading symlinks at evaluation time. + - (-) We could just not handle such cases, e.g. `equals "foo" "foo/bar/.. == false`. The paths are different, we don't need to check whether the paths point to the same thing. + - (+) Assume we said `relativeTo /foo /bar == "../bar"`. If this is used like `/bar/../foo` in the end, and `bar` turns out to be a symlink to somewhere else, this won't be accurate. + - (-) We could decide to not support such ambiguous operations, or mark them as such, e.g. the normal `relativeTo` will error on such a case, but there could be `extendedRelativeTo` supporting that. +- (-) `..` are a part of paths, a path library should therefore support it. + - (+) If we can convincingly argue that all such use cases are better done e.g. with runtime tools, the library not supporting it can nudge people towards using those. +- (-) We could allow "..", but only in the prefix. + - (+) Then we'd have to throw an error for doing `append /some/path "../foo"`, making it non-composable. + - (+) The same is for returning paths with `..`: `relativeTo /foo /bar => "../bar"` would produce a non-composable path. +- (+) We argue that `..` is not needed at the Nix evaluation level, since we'd always start evaluation from the project root and don't go up from there. + - (+) `..` is supported in Nix paths, turning them into absolute paths. + - (-) This is ambiguous in the presence of symlinks. +- (+) If you need `..` for building or runtime, you can use build-/run-time tooling to create those (e.g. `realpath` with `--relative-to`), or use absolute paths instead. + This also gives you the ability to correctly handle symlinks. + +
+ +### Trailing slashes +[trailing-slashes]: #trailing-slashes + +Observing: Subpaths can contain trailing slashes, like `foo/`, indicating that the path points to a directory and not a file. + +Considering: Paths should be as consistent as possible, there should only be a single normalisation for the same path. + +Decision: All functions remove trailing slashes in their results. + +
+Arguments + +- (+) It allows normalisations to be unique, in that there's only a single normalisation for the same path. If trailing slashes were preserved, both `foo/bar` and `foo/bar/` would be valid but different normalisations for the same path. +- Comparison to other frameworks to figure out the least surprising behavior: + - (+) Nix itself doesn't support trailing slashes when parsing and doesn't preserve them when appending paths. + - (-) [Rust's std::path](https://doc.rust-lang.org/std/path/index.html) does preserve them during [construction](https://doc.rust-lang.org/std/path/struct.Path.html#method.new). + - (+) Doesn't preserve them when returning individual [components](https://doc.rust-lang.org/std/path/struct.Path.html#method.components). + - (+) Doesn't preserve them when [canonicalizing](https://doc.rust-lang.org/std/path/struct.Path.html#method.canonicalize). + - (+) [Python 3's pathlib](https://docs.python.org/3/library/pathlib.html#module-pathlib) doesn't preserve them during [construction](https://docs.python.org/3/library/pathlib.html#pathlib.PurePath). + - Notably it represents the individual components as a list internally. + - (-) [Haskell's filepath](https://hackage.haskell.org/package/filepath-1.4.100.0) has [explicit support](https://hackage.haskell.org/package/filepath-1.4.100.0/docs/System-FilePath.html#g:6) for handling trailing slashes. + - (-) Does preserve them for [normalisation](https://hackage.haskell.org/package/filepath-1.4.100.0/docs/System-FilePath.html#v:normalise). + - (-) [NodeJS's Path library](https://nodejs.org/api/path.html) preserves trailing slashes for [normalisation](https://nodejs.org/api/path.html#pathnormalizepath). + - (+) For [parsing a path](https://nodejs.org/api/path.html#pathparsepath) into its significant elements, trailing slashes are not preserved. +- (+) Nix's builtin function `dirOf` gives an unexpected result for paths with trailing slashes: `dirOf "foo/bar/" == "foo/bar"`. + Inconsistently, `baseNameOf` works correctly though: `baseNameOf "foo/bar/" == "bar"`. + - (-) We are writing a path library to improve handling of paths though, so we shouldn't use these functions and discourage their use. +- (-) Unexpected result when normalising intermediate paths, like `relative.normalise ("foo" + "/") + "bar" == "foobar"`. + - (+) This is not a practical use case though. + - (+) Don't use `+` to append paths, this library has a `join` function for that. + - (-) Users might use `+` out of habit though. +- (+) The `realpath` command also removes trailing slashes. +- (+) Even with a trailing slash, the path is the same, it's only an indication that it's a directory. + +
+ +## Other implementations and references + +- [Rust](https://doc.rust-lang.org/std/path/struct.Path.html) +- [Python](https://docs.python.org/3/library/pathlib.html) +- [Haskell](https://hackage.haskell.org/package/filepath-1.4.100.0/docs/System-FilePath.html) +- [Nodejs](https://nodejs.org/api/path.html) +- [POSIX.1-2017](https://pubs.opengroup.org/onlinepubs/9699919799/nframe.html) diff --git a/path/default.nix b/path/default.nix new file mode 100644 index 000000000..ce5d77599 --- /dev/null +++ b/path/default.nix @@ -0,0 +1,433 @@ +# Functions for working with paths, see ./path.md +{ lib }: +let + + inherit (builtins) + isString + isPath + split + match + typeOf + ; + + inherit (lib.lists) + length + head + last + genList + elemAt + all + concatMap + foldl' + take + drop + ; + + inherit (lib.strings) + concatStringsSep + substring + ; + + inherit (lib.asserts) + assertMsg + ; + + inherit (lib.path.subpath) + isValid + ; + + # Return the reason why a subpath is invalid, or `null` if it's valid + subpathInvalidReason = value: + if ! isString value then + "The given value is of type ${builtins.typeOf value}, but a string was expected" + else if value == "" then + "The given string is empty" + else if substring 0 1 value == "/" then + "The given string \"${value}\" starts with a `/`, representing an absolute path" + # We don't support ".." components, see ./path.md#parent-directory + else if match "(.*/)?\\.\\.(/.*)?" value != null then + "The given string \"${value}\" contains a `..` component, which is not allowed in subpaths" + else null; + + # Split and normalise a relative path string into its components. + # Error for ".." components and doesn't include "." components + splitRelPath = path: + let + # Split the string into its parts using regex for efficiency. This regex + # matches patterns like "/", "/./", "/././", with arbitrarily many "/"s + # together. These are the main special cases: + # - Leading "./" gets split into a leading "." part + # - Trailing "/." or "/" get split into a trailing "." or "" + # part respectively + # + # These are the only cases where "." and "" parts can occur + parts = split "/+(\\./+)*" path; + + # `split` creates a list of 2 * k + 1 elements, containing the k + + # 1 parts, interleaved with k matches where k is the number of + # (non-overlapping) matches. This calculation here gets the number of parts + # back from the list length + # floor( (2 * k + 1) / 2 ) + 1 == floor( k + 1/2 ) + 1 == k + 1 + partCount = length parts / 2 + 1; + + # To assemble the final list of components we want to: + # - Skip a potential leading ".", normalising "./foo" to "foo" + # - Skip a potential trailing "." or "", normalising "foo/" and "foo/." to + # "foo". See ./path.md#trailing-slashes + skipStart = if head parts == "." then 1 else 0; + skipEnd = if last parts == "." || last parts == "" then 1 else 0; + + # We can now know the length of the result by removing the number of + # skipped parts from the total number + componentCount = partCount - skipEnd - skipStart; + + in + # Special case of a single "." path component. Such a case leaves a + # componentCount of -1 due to the skipStart/skipEnd not verifying that + # they don't refer to the same character + if path == "." then [] + + # Generate the result list directly. This is more efficient than a + # combination of `filter`, `init` and `tail`, because here we don't + # allocate any intermediate lists + else genList (index: + # To get to the element we need to add the number of parts we skip and + # multiply by two due to the interleaved layout of `parts` + elemAt parts ((skipStart + index) * 2) + ) componentCount; + + # Join relative path components together + joinRelPath = components: + # Always return relative paths with `./` as a prefix (./path.md#leading-dots-for-relative-paths) + "./" + + # An empty string is not a valid relative path, so we need to return a `.` when we have no components + (if components == [] then "." else concatStringsSep "/" components); + + # Deconstruct a path value type into: + # - root: The filesystem root of the path, generally `/` + # - components: All the path's components + # + # This is similar to `splitString "/" (toString path)` but safer + # because it can distinguish different filesystem roots + deconstructPath = + let + recurse = components: path: + # If the parent of a path is the path itself, then it's a filesystem root + if path == dirOf path then { root = path; inherit components; } + else recurse ([ (baseNameOf path) ] ++ components) (dirOf path); + in recurse []; + + # Used as an abstraction between `hasPrefix`, `hasProperPrefix` and `removePrefix` + # + # Takes four arguments: + # - context: A string describing the callee, for error messages + # - continue: A function with four arguments + # - path1: A path + # - path2: Another path + # + # The function checks whether `path1` is a path and computes its + # decomposition (`deconPath1`) before taking `path2` as an argument, which + # allows the computation to be cached in a thunk between multiple calls. + # With `path2` also provided, it checks whether it's also a path, computes + # the decomposition (`deconPath2`), checks whether both paths have the same + # filesystem root, and then calls `continue path1 deconPath1 path2 deconPath2`, + # allowing the caller to decide what to do with these values. + withTwoDeconstructedPaths = context: continue: + path1: + assert assertMsg + (isPath path1) + "${context}: First argument is of type ${typeOf path1}, but a path was expected"; + let + deconPath1 = deconstructPath path1; + in + path2: + assert assertMsg + (isPath path2) + "${context}: Second argument is of type ${typeOf path2}, but a path was expected"; + let + deconPath2 = deconstructPath path2; + in + assert assertMsg + (deconPath1.root == deconPath2.root) '' + ${context}: Filesystem roots must be the same for both paths, but paths with different roots were given: + first argument: "${toString path1}" (root "${toString deconPath1.root}") + second argument: "${toString path2}" (root "${toString deconPath2.root}")''; + continue path1 deconPath1 path2 deconPath2; + +in /* No rec! Add dependencies on this file at the top. */ { + + /* Append a subpath string to a path. + + Like `path + ("/" + string)` but safer, because it errors instead of returning potentially surprising results. + More specifically, it checks that the first argument is a [path value type](https://nixos.org/manual/nix/stable/language/values.html#type-path"), + and that the second argument is a valid subpath string (see `lib.path.subpath.isValid`). + + Type: + append :: Path -> String -> Path + + Example: + append /foo "bar/baz" + => /foo/bar/baz + + # subpaths don't need to be normalised + append /foo "./bar//baz/./" + => /foo/bar/baz + + # can append to root directory + append /. "foo/bar" + => /foo/bar + + # first argument needs to be a path value type + append "/foo" "bar" + => + + # second argument needs to be a valid subpath string + append /foo /bar + => + append /foo "" + => + append /foo "/bar" + => + append /foo "../bar" + => + */ + append = + # The absolute path to append to + path: + # The subpath string to append + subpath: + assert assertMsg (isPath path) '' + lib.path.append: The first argument is of type ${builtins.typeOf path}, but a path was expected''; + assert assertMsg (isValid subpath) '' + lib.path.append: Second argument is not a valid subpath string: + ${subpathInvalidReason subpath}''; + path + ("/" + subpath); + + hasPrefix = withTwoDeconstructedPaths "lib.path.hasPrefix" (prefix: deconPrefix: path: deconPath: + take (length deconPrefix.components) deconPath.components == deconPrefix.components + ); + + removePrefix = withTwoDeconstructedPaths "lib.path.removePrefix" (prefix: deconPrefix: path: deconPath: + if take (length deconPrefix.components) deconPath.components == deconPrefix.components + then drop (length deconPrefix.components) deconPath.components + else throw '' + lib.path.removePrefix: The first prefix path argument (${toString prefix}) is not a prefix of the second path argument (${toString path})'' + ); + + /* Whether a value is a valid subpath string. + + - The value is a string + + - The string is not empty + + - The string doesn't start with a `/` + + - The string doesn't contain any `..` path components + + Type: + subpath.isValid :: String -> Bool + + Example: + # Not a string + subpath.isValid null + => false + + # Empty string + subpath.isValid "" + => false + + # Absolute path + subpath.isValid "/foo" + => false + + # Contains a `..` path component + subpath.isValid "../foo" + => false + + # Valid subpath + subpath.isValid "foo/bar" + => true + + # Doesn't need to be normalised + subpath.isValid "./foo//bar/" + => true + */ + subpath.isValid = + # The value to check + value: + subpathInvalidReason value == null; + + + /* Join subpath strings together using `/`, returning a normalised subpath string. + + Like `concatStringsSep "/"` but safer, specifically: + + - All elements must be valid subpath strings, see `lib.path.subpath.isValid` + + - The result gets normalised, see `lib.path.subpath.normalise` + + - The edge case of an empty list gets properly handled by returning the neutral subpath `"./."` + + Laws: + + - Associativity: + + subpath.join [ x (subpath.join [ y z ]) ] == subpath.join [ (subpath.join [ x y ]) z ] + + - Identity - `"./."` is the neutral element for normalised paths: + + subpath.join [ ] == "./." + subpath.join [ (subpath.normalise p) "./." ] == subpath.normalise p + subpath.join [ "./." (subpath.normalise p) ] == subpath.normalise p + + - Normalisation - the result is normalised according to `lib.path.subpath.normalise`: + + subpath.join ps == subpath.normalise (subpath.join ps) + + - For non-empty lists, the implementation is equivalent to normalising the result of `concatStringsSep "/"`. + Note that the above laws can be derived from this one. + + ps != [] -> subpath.join ps == subpath.normalise (concatStringsSep "/" ps) + + Type: + subpath.join :: [ String ] -> String + + Example: + subpath.join [ "foo" "bar/baz" ] + => "./foo/bar/baz" + + # normalise the result + subpath.join [ "./foo" "." "bar//./baz/" ] + => "./foo/bar/baz" + + # passing an empty list results in the current directory + subpath.join [ ] + => "./." + + # elements must be valid subpath strings + subpath.join [ /foo ] + => + subpath.join [ "" ] + => + subpath.join [ "/foo" ] + => + subpath.join [ "../foo" ] + => + */ + subpath.join = + # The list of subpaths to join together + subpaths: + # Fast in case all paths are valid + if all isValid subpaths + then joinRelPath (concatMap splitRelPath subpaths) + else + # Otherwise we take our time to gather more info for a better error message + # Strictly go through each path, throwing on the first invalid one + # Tracks the list index in the fold accumulator + foldl' (i: path: + if isValid path + then i + 1 + else throw '' + lib.path.subpath.join: Element at index ${toString i} is not a valid subpath string: + ${subpathInvalidReason path}'' + ) 0 subpaths; + + /* Normalise a subpath. Throw an error if the subpath isn't valid, see + `lib.path.subpath.isValid` + + - Limit repeating `/` to a single one + + - Remove redundant `.` components + + - Remove trailing `/` and `/.` + + - Add leading `./` + + Laws: + + - Idempotency - normalising multiple times gives the same result: + + subpath.normalise (subpath.normalise p) == subpath.normalise p + + - Uniqueness - there's only a single normalisation for the paths that lead to the same file system node: + + subpath.normalise p != subpath.normalise q -> $(realpath ${p}) != $(realpath ${q}) + + - Don't change the result when appended to a Nix path value: + + base + ("/" + p) == base + ("/" + subpath.normalise p) + + - Don't change the path according to `realpath`: + + $(realpath ${p}) == $(realpath ${subpath.normalise p}) + + - Only error on invalid subpaths: + + builtins.tryEval (subpath.normalise p)).success == subpath.isValid p + + Type: + subpath.normalise :: String -> String + + Example: + # limit repeating `/` to a single one + subpath.normalise "foo//bar" + => "./foo/bar" + + # remove redundant `.` components + subpath.normalise "foo/./bar" + => "./foo/bar" + + # add leading `./` + subpath.normalise "foo/bar" + => "./foo/bar" + + # remove trailing `/` + subpath.normalise "foo/bar/" + => "./foo/bar" + + # remove trailing `/.` + subpath.normalise "foo/bar/." + => "./foo/bar" + + # Return the current directory as `./.` + subpath.normalise "." + => "./." + + # error on `..` path components + subpath.normalise "foo/../bar" + => + + # error on empty string + subpath.normalise "" + => + + # error on absolute path + subpath.normalise "/foo" + => + */ + subpath.normalise = + # The subpath string to normalise + subpath: + assert assertMsg (isValid subpath) '' + lib.path.subpath.normalise: Argument is not a valid subpath string: + ${subpathInvalidReason subpath}''; + joinRelPath (splitRelPath subpath); + + commonAncestor = a: b: + let + a' = deconstructPath a; + b' = deconstructPath b; + in + if a'.root != b'.root then + throw "lib.path.commonAncestor: Given paths don't have the same filesystem root" + else + a'.root + ("/" + concatStringsSep "/" (lib.lists.commonPrefix a'.components b'.components)); + + deconstruct = path: deconstructPath path; + construct = { root, components }: root + ("/" + concatStringsSep "/" components); + + components = { + toSubpath = joinRelPath; + fromSubpath = splitRelPath; + }; +} diff --git a/path/tests/default.nix b/path/tests/default.nix new file mode 100644 index 000000000..9a31e4282 --- /dev/null +++ b/path/tests/default.nix @@ -0,0 +1,34 @@ +{ + nixpkgs ? ../../.., + system ? builtins.currentSystem, + pkgs ? import nixpkgs { + config = {}; + overlays = []; + inherit system; + }, + libpath ? ../.., + # Random seed + seed ? null, +}: +pkgs.runCommand "lib-path-tests" { + nativeBuildInputs = with pkgs; [ + nix + jq + bc + ]; +} '' + # Needed to make Nix evaluation work + export NIX_STATE_DIR=$(mktemp -d) + + cp -r ${libpath} lib + export TEST_LIB=$PWD/lib + + echo "Running unit tests lib/path/tests/unit.nix" + nix-instantiate --eval lib/path/tests/unit.nix \ + --argstr libpath "$TEST_LIB" + + echo "Running property tests lib/path/tests/prop.sh" + bash lib/path/tests/prop.sh ${toString seed} + + touch $out +'' diff --git a/path/tests/generate.awk b/path/tests/generate.awk new file mode 100644 index 000000000..811dd0c46 --- /dev/null +++ b/path/tests/generate.awk @@ -0,0 +1,64 @@ +# Generate random path-like strings, separated by null characters. +# +# Invocation: +# +# awk -f ./generate.awk -v = | tr '\0' '\n' +# +# Customizable variables (all default to 0): +# - seed: Deterministic random seed to use for generation +# - count: Number of paths to generate +# - extradotweight: Give extra weight to dots being generated +# - extraslashweight: Give extra weight to slashes being generated +# - extranullweight: Give extra weight to null being generated, making paths shorter +BEGIN { + # Random seed, passed explicitly for reproducibility + srand(seed) + + # Don't include special characters below 32 + minascii = 32 + # Don't include DEL at 128 + maxascii = 127 + upperascii = maxascii - minascii + + # add extra weight for ., in addition to the one weight from the ascii range + upperdot = upperascii + extradotweight + + # add extra weight for /, in addition to the one weight from the ascii range + upperslash = upperdot + extraslashweight + + # add extra weight for null, indicating the end of the string + # Must be at least 1 to have strings end at all + total = upperslash + 1 + extranullweight + + # new=1 indicates that it's a new string + new=1 + while (count > 0) { + + # Random integer between [0, total) + value = int(rand() * total) + + if (value < upperascii) { + # Ascii range + printf("%c", value + minascii) + new=0 + + } else if (value < upperdot) { + # Dot range + printf "." + new=0 + + } else if (value < upperslash) { + # If it's the start of a new path, only generate a / in 10% of cases + # This is always an invalid subpath, which is not a very interesting case + if (new && rand() > 0.1) continue + printf "/" + + } else { + # Do not generate empty strings + if (new) continue + printf "\x00" + count-- + new=1 + } + } +} diff --git a/path/tests/prop.nix b/path/tests/prop.nix new file mode 100644 index 000000000..67e5c1e9d --- /dev/null +++ b/path/tests/prop.nix @@ -0,0 +1,60 @@ +# Given a list of path-like strings, check some properties of the path library +# using those paths and return a list of attribute sets of the following form: +# +# { = ; } +# +# If `normalise` fails to evaluate, the attribute value is set to `""`. +# If not, the resulting value is normalised again and an appropriate attribute set added to the output list. +{ + # The path to the nixpkgs lib to use + libpath, + # A flat directory containing files with randomly-generated + # path-like values + dir, +}: +let + lib = import libpath; + + # read each file into a string + strings = map (name: + builtins.readFile (dir + "/${name}") + ) (builtins.attrNames (builtins.readDir dir)); + + inherit (lib.path.subpath) normalise isValid; + inherit (lib.asserts) assertMsg; + + normaliseAndCheck = str: + let + originalValid = isValid str; + + tryOnce = builtins.tryEval (normalise str); + tryTwice = builtins.tryEval (normalise tryOnce.value); + + absConcatOrig = /. + ("/" + str); + absConcatNormalised = /. + ("/" + tryOnce.value); + in + # Check the lib.path.subpath.normalise property to only error on invalid subpaths + assert assertMsg + (originalValid -> tryOnce.success) + "Even though string \"${str}\" is valid as a subpath, the normalisation for it failed"; + assert assertMsg + (! originalValid -> ! tryOnce.success) + "Even though string \"${str}\" is invalid as a subpath, the normalisation for it succeeded"; + + # Check normalisation idempotency + assert assertMsg + (originalValid -> tryTwice.success) + "For valid subpath \"${str}\", the normalisation \"${tryOnce.value}\" was not a valid subpath"; + assert assertMsg + (originalValid -> tryOnce.value == tryTwice.value) + "For valid subpath \"${str}\", normalising it once gives \"${tryOnce.value}\" but normalising it twice gives a different result: \"${tryTwice.value}\""; + + # Check that normalisation doesn't change a string when appended to an absolute Nix path value + assert assertMsg + (originalValid -> absConcatOrig == absConcatNormalised) + "For valid subpath \"${str}\", appending to an absolute Nix path value gives \"${absConcatOrig}\", but appending the normalised result \"${tryOnce.value}\" gives a different value \"${absConcatNormalised}\""; + + # Return an empty string when failed + if tryOnce.success then tryOnce.value else ""; + +in lib.genAttrs strings normaliseAndCheck diff --git a/path/tests/prop.sh b/path/tests/prop.sh new file mode 100755 index 000000000..e48c6667f --- /dev/null +++ b/path/tests/prop.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash + +# Property tests for the `lib.path` library +# +# It generates random path-like strings and runs the functions on +# them, checking that the expected laws of the functions hold + +set -euo pipefail +shopt -s inherit_errexit + +# https://stackoverflow.com/a/246128 +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +if test -z "${TEST_LIB:-}"; then + TEST_LIB=$SCRIPT_DIR/../.. +fi + +tmp="$(mktemp -d)" +clean_up() { + rm -rf "$tmp" +} +trap clean_up EXIT +mkdir -p "$tmp/work" +cd "$tmp/work" + +# Defaulting to a random seed but the first argument can override this +seed=${1:-$RANDOM} +echo >&2 "Using seed $seed, use \`lib/path/tests/prop.sh $seed\` to reproduce this result" + +# The number of random paths to generate. This specific number was chosen to +# be fast enough while still generating enough variety to detect bugs. +count=500 + +debug=0 +# debug=1 # print some extra info +# debug=2 # print generated values + +# Fine tuning parameters to balance the number of generated invalid paths +# to the variance in generated paths. +extradotweight=64 # Larger value: more dots +extraslashweight=64 # Larger value: more slashes +extranullweight=16 # Larger value: shorter strings + +die() { + echo >&2 "test case failed: " "$@" + exit 1 +} + +if [[ "$debug" -ge 1 ]]; then + echo >&2 "Generating $count random path-like strings" +fi + +# Read stream of null-terminated strings entry-by-entry into bash, +# write it to a file and the `strings` array. +declare -a strings=() +mkdir -p "$tmp/strings" +while IFS= read -r -d $'\0' str; do + printf "%s" "$str" > "$tmp/strings/${#strings[@]}" + strings+=("$str") +done < <(awk \ + -f "$SCRIPT_DIR"/generate.awk \ + -v seed="$seed" \ + -v count="$count" \ + -v extradotweight="$extradotweight" \ + -v extraslashweight="$extraslashweight" \ + -v extranullweight="$extranullweight") + +if [[ "$debug" -ge 1 ]]; then + echo >&2 "Trying to normalise the generated path-like strings with Nix" +fi + +# Precalculate all normalisations with a single Nix call. Calling Nix for each +# string individually would take way too long +nix-instantiate --eval --strict --json \ + --argstr libpath "$TEST_LIB" \ + --argstr dir "$tmp/strings" \ + "$SCRIPT_DIR"/prop.nix \ + >"$tmp/result.json" + +# Uses some jq magic to turn the resulting attribute set into an associative +# bash array assignment +declare -A normalised_result="($(jq ' + to_entries + | map("[\(.key | @sh)]=\(.value | @sh)") + | join(" \n")' -r < "$tmp/result.json"))" + +# Looks up a normalisation result for a string +# Checks that the normalisation is only failing iff it's an invalid subpath +# For valid subpaths, returns 0 and prints the normalisation result +# For invalid subpaths, returns 1 +normalise() { + local str=$1 + # Uses the same check for validity as in the library implementation + if [[ "$str" == "" || "$str" == /* || "$str" =~ ^(.*/)?\.\.(/.*)?$ ]]; then + valid= + else + valid=1 + fi + + normalised=${normalised_result[$str]} + # An empty string indicates failure, this is encoded in ./prop.nix + if [[ -n "$normalised" ]]; then + if [[ -n "$valid" ]]; then + echo "$normalised" + else + die "For invalid subpath \"$str\", lib.path.subpath.normalise returned this result: \"$normalised\"" + fi + else + if [[ -n "$valid" ]]; then + die "For valid subpath \"$str\", lib.path.subpath.normalise failed" + else + if [[ "$debug" -ge 2 ]]; then + echo >&2 "String \"$str\" is not a valid subpath" + fi + # Invalid and it correctly failed, we let the caller continue if they catch the exit code + return 1 + fi + fi +} + +# Intermediate result populated by test_idempotency_realpath +# and used in test_normalise_uniqueness +# +# Contains a mapping from a normalised subpath to the realpath result it represents +declare -A norm_to_real + +test_idempotency_realpath() { + if [[ "$debug" -ge 1 ]]; then + echo >&2 "Checking idempotency of each result and making sure the realpath result isn't changed" + fi + + # Count invalid subpaths to display stats + invalid=0 + for str in "${strings[@]}"; do + if ! result=$(normalise "$str"); then + ((invalid++)) || true + continue + fi + + # Check the law that it doesn't change the result of a realpath + mkdir -p -- "$str" "$result" + real_orig=$(realpath -- "$str") + real_norm=$(realpath -- "$result") + + if [[ "$real_orig" != "$real_norm" ]]; then + die "realpath of the original string \"$str\" (\"$real_orig\") is not the same as realpath of the normalisation \"$result\" (\"$real_norm\")" + fi + + if [[ "$debug" -ge 2 ]]; then + echo >&2 "String \"$str\" gets normalised to \"$result\" and file path \"$real_orig\"" + fi + norm_to_real["$result"]="$real_orig" + done + if [[ "$debug" -ge 1 ]]; then + echo >&2 "$(bc <<< "scale=1; 100 / $count * $invalid")% of the total $count generated strings were invalid subpath strings, and were therefore ignored" + fi +} + +test_normalise_uniqueness() { + if [[ "$debug" -ge 1 ]]; then + echo >&2 "Checking for the uniqueness law" + fi + + for norm_p in "${!norm_to_real[@]}"; do + real_p=${norm_to_real["$norm_p"]} + for norm_q in "${!norm_to_real[@]}"; do + real_q=${norm_to_real["$norm_q"]} + # Checks normalisation uniqueness law for each pair of values + if [[ "$norm_p" != "$norm_q" && "$real_p" == "$real_q" ]]; then + die "Normalisations \"$norm_p\" and \"$norm_q\" are different, but the realpath of them is the same: \"$real_p\"" + fi + done + done +} + +test_idempotency_realpath +test_normalise_uniqueness + +echo >&2 tests ok diff --git a/path/tests/unit.nix b/path/tests/unit.nix new file mode 100644 index 000000000..61c4ab4d6 --- /dev/null +++ b/path/tests/unit.nix @@ -0,0 +1,193 @@ +# Unit tests for lib.path functions. Use `nix-build` in this directory to +# run these +{ libpath }: +let + lib = import libpath; + inherit (lib.path) append subpath; + + cases = lib.runTests { + # Test examples from the lib.path.append documentation + testAppendExample1 = { + expr = append /foo "bar/baz"; + expected = /foo/bar/baz; + }; + testAppendExample2 = { + expr = append /foo "./bar//baz/./"; + expected = /foo/bar/baz; + }; + testAppendExample3 = { + expr = append /. "foo/bar"; + expected = /foo/bar; + }; + testAppendExample4 = { + expr = (builtins.tryEval (append "/foo" "bar")).success; + expected = false; + }; + testAppendExample5 = { + expr = (builtins.tryEval (append /foo /bar)).success; + expected = false; + }; + testAppendExample6 = { + expr = (builtins.tryEval (append /foo "")).success; + expected = false; + }; + testAppendExample7 = { + expr = (builtins.tryEval (append /foo "/bar")).success; + expected = false; + }; + testAppendExample8 = { + expr = (builtins.tryEval (append /foo "../bar")).success; + expected = false; + }; + + # Test examples from the lib.path.subpath.isValid documentation + testSubpathIsValidExample1 = { + expr = subpath.isValid null; + expected = false; + }; + testSubpathIsValidExample2 = { + expr = subpath.isValid ""; + expected = false; + }; + testSubpathIsValidExample3 = { + expr = subpath.isValid "/foo"; + expected = false; + }; + testSubpathIsValidExample4 = { + expr = subpath.isValid "../foo"; + expected = false; + }; + testSubpathIsValidExample5 = { + expr = subpath.isValid "foo/bar"; + expected = true; + }; + testSubpathIsValidExample6 = { + expr = subpath.isValid "./foo//bar/"; + expected = true; + }; + # Some extra tests + testSubpathIsValidTwoDotsEnd = { + expr = subpath.isValid "foo/.."; + expected = false; + }; + testSubpathIsValidTwoDotsMiddle = { + expr = subpath.isValid "foo/../bar"; + expected = false; + }; + testSubpathIsValidTwoDotsPrefix = { + expr = subpath.isValid "..foo"; + expected = true; + }; + testSubpathIsValidTwoDotsSuffix = { + expr = subpath.isValid "foo.."; + expected = true; + }; + testSubpathIsValidTwoDotsPrefixComponent = { + expr = subpath.isValid "foo/..bar/baz"; + expected = true; + }; + testSubpathIsValidTwoDotsSuffixComponent = { + expr = subpath.isValid "foo/bar../baz"; + expected = true; + }; + testSubpathIsValidThreeDots = { + expr = subpath.isValid "..."; + expected = true; + }; + testSubpathIsValidFourDots = { + expr = subpath.isValid "...."; + expected = true; + }; + testSubpathIsValidThreeDotsComponent = { + expr = subpath.isValid "foo/.../bar"; + expected = true; + }; + testSubpathIsValidFourDotsComponent = { + expr = subpath.isValid "foo/..../bar"; + expected = true; + }; + + # Test examples from the lib.path.subpath.join documentation + testSubpathJoinExample1 = { + expr = subpath.join [ "foo" "bar/baz" ]; + expected = "./foo/bar/baz"; + }; + testSubpathJoinExample2 = { + expr = subpath.join [ "./foo" "." "bar//./baz/" ]; + expected = "./foo/bar/baz"; + }; + testSubpathJoinExample3 = { + expr = subpath.join [ ]; + expected = "./."; + }; + testSubpathJoinExample4 = { + expr = (builtins.tryEval (subpath.join [ /foo ])).success; + expected = false; + }; + testSubpathJoinExample5 = { + expr = (builtins.tryEval (subpath.join [ "" ])).success; + expected = false; + }; + testSubpathJoinExample6 = { + expr = (builtins.tryEval (subpath.join [ "/foo" ])).success; + expected = false; + }; + testSubpathJoinExample7 = { + expr = (builtins.tryEval (subpath.join [ "../foo" ])).success; + expected = false; + }; + + # Test examples from the lib.path.subpath.normalise documentation + testSubpathNormaliseExample1 = { + expr = subpath.normalise "foo//bar"; + expected = "./foo/bar"; + }; + testSubpathNormaliseExample2 = { + expr = subpath.normalise "foo/./bar"; + expected = "./foo/bar"; + }; + testSubpathNormaliseExample3 = { + expr = subpath.normalise "foo/bar"; + expected = "./foo/bar"; + }; + testSubpathNormaliseExample4 = { + expr = subpath.normalise "foo/bar/"; + expected = "./foo/bar"; + }; + testSubpathNormaliseExample5 = { + expr = subpath.normalise "foo/bar/."; + expected = "./foo/bar"; + }; + testSubpathNormaliseExample6 = { + expr = subpath.normalise "."; + expected = "./."; + }; + testSubpathNormaliseExample7 = { + expr = (builtins.tryEval (subpath.normalise "foo/../bar")).success; + expected = false; + }; + testSubpathNormaliseExample8 = { + expr = (builtins.tryEval (subpath.normalise "")).success; + expected = false; + }; + testSubpathNormaliseExample9 = { + expr = (builtins.tryEval (subpath.normalise "/foo")).success; + expected = false; + }; + # Some extra tests + testSubpathNormaliseIsValidDots = { + expr = subpath.normalise "./foo/.bar/.../baz...qux"; + expected = "./foo/.bar/.../baz...qux"; + }; + testSubpathNormaliseWrongType = { + expr = (builtins.tryEval (subpath.normalise null)).success; + expected = false; + }; + testSubpathNormaliseTwoDots = { + expr = (builtins.tryEval (subpath.normalise "..")).success; + expected = false; + }; + }; +in + if cases == [] then "Unit tests successful" + else throw "Path unit tests failed: ${lib.generators.toPretty {} cases}" diff --git a/source-types.nix b/source-types.nix new file mode 100644 index 000000000..c4f263dcf --- /dev/null +++ b/source-types.nix @@ -0,0 +1,19 @@ +{ lib }: + +let + defaultSourceType = tname: { + shortName = tname; + isSource = false; + }; +in lib.mapAttrs (tname: tset: defaultSourceType tname // tset) { + + fromSource = { + isSource = true; + }; + + binaryNativeCode = {}; + + binaryBytecode = {}; + + binaryFirmware = {}; +} diff --git a/sources.nix b/sources.nix new file mode 100644 index 000000000..d990777c6 --- /dev/null +++ b/sources.nix @@ -0,0 +1,292 @@ +# Functions for copying sources to the Nix store. +{ lib }: + +# Tested in lib/tests/sources.sh +let + inherit (builtins) + match + readDir + split + storeDir + tryEval + ; + inherit (lib) + boolToString + filter + getAttr + isString + pathExists + readFile + ; + inherit (lib.filesystem) + pathType + pathIsDirectory + pathIsRegularFile + ; + + /* + A basic filter for `cleanSourceWith` that removes + directories of version control system, backup files (*~) + and some generated files. + */ + cleanSourceFilter = name: type: let baseName = baseNameOf (toString name); in ! ( + # Filter out version control software files/directories + (baseName == ".git" || type == "directory" && (baseName == ".svn" || baseName == "CVS" || baseName == ".hg")) || + # Filter out editor backup / swap files. + lib.hasSuffix "~" baseName || + match "^\\.sw[a-z]$" baseName != null || + match "^\\..*\\.sw[a-z]$" baseName != null || + + # Filter out generates files. + lib.hasSuffix ".o" baseName || + lib.hasSuffix ".so" baseName || + # Filter out nix-build result symlinks + (type == "symlink" && lib.hasPrefix "result" baseName) || + # Filter out sockets and other types of files we can't have in the store. + (type == "unknown") + ); + + /* + Filters a source tree removing version control files and directories using cleanSourceFilter. + + Example: + cleanSource ./. + */ + cleanSource = src: cleanSourceWith { filter = cleanSourceFilter; inherit src; }; + + /* + Like `builtins.filterSource`, except it will compose with itself, + allowing you to chain multiple calls together without any + intermediate copies being put in the nix store. + + Example: + lib.cleanSourceWith { + filter = f; + src = lib.cleanSourceWith { + filter = g; + src = ./.; + }; + } + # Succeeds! + + builtins.filterSource f (builtins.filterSource g ./.) + # Fails! + + */ + cleanSourceWith = + { + # A path or cleanSourceWith result to filter and/or rename. + src, + # Optional with default value: constant true (include everything) + # The function will be combined with the && operator such + # that src.filter is called lazily. + # For implementing a filter, see + # https://nixos.org/nix/manual/#builtin-filterSource + # Type: A function (path -> type -> bool) + filter ? _path: _type: true, + # Optional name to use as part of the store path. + # This defaults to `src.name` or otherwise `"source"`. + name ? null + }: + let + orig = toSourceAttributes src; + in fromSourceAttributes { + inherit (orig) origSrc; + filter = path: type: filter path type && orig.filter path type; + name = if name != null then name else orig.name; + }; + + /* + Add logging to a source, for troubleshooting the filtering behavior. + Type: + sources.trace :: sourceLike -> Source + */ + trace = + # Source to debug. The returned source will behave like this source, but also log its filter invocations. + src: + let + attrs = toSourceAttributes src; + in + fromSourceAttributes ( + attrs // { + filter = path: type: + let + r = attrs.filter path type; + in + builtins.trace "${attrs.name}.filter ${path} = ${boolToString r}" r; + } + ) // { + satisfiesSubpathInvariant = src ? satisfiesSubpathInvariant && src.satisfiesSubpathInvariant; + }; + + /* + Filter sources by a list of regular expressions. + + Example: src = sourceByRegex ./my-subproject [".*\.py$" "^database.sql$"] + */ + sourceByRegex = src: regexes: + let + isFiltered = src ? _isLibCleanSourceWith; + origSrc = if isFiltered then src.origSrc else src; + in lib.cleanSourceWith { + filter = (path: type: + let relPath = lib.removePrefix (toString origSrc + "/") (toString path); + in lib.any (re: match re relPath != null) regexes); + inherit src; + }; + + /* + Get all files ending with the specified suffices from the given + source directory or its descendants, omitting files that do not match + any suffix. The result of the example below will include files like + `./dir/module.c` and `./dir/subdir/doc.xml` if present. + + Type: sourceLike -> [String] -> Source + + Example: + sourceFilesBySuffices ./. [ ".xml" ".c" ] + */ + sourceFilesBySuffices = + # Path or source containing the files to be returned + src: + # A list of file suffix strings + exts: + let filter = name: type: + let base = baseNameOf (toString name); + in type == "directory" || lib.any (ext: lib.hasSuffix ext base) exts; + in cleanSourceWith { inherit filter src; }; + + pathIsGitRepo = path: (_commitIdFromGitRepoOrError path)?value; + + /* + Get the commit id of a git repo. + + Example: commitIdFromGitRepo + */ + commitIdFromGitRepo = path: + let commitIdOrError = _commitIdFromGitRepoOrError path; + in commitIdOrError.value or (throw commitIdOrError.error); + + # Get the commit id of a git repo. + + # Returns `{ value = commitHash }` or `{ error = "... message ..." }`. + + # Example: commitIdFromGitRepo + # not exported, used for commitIdFromGitRepo + _commitIdFromGitRepoOrError = + let readCommitFromFile = file: path: + let fileName = path + "/${file}"; + packedRefsName = path + "/packed-refs"; + absolutePath = base: path: + if lib.hasPrefix "/" path + then path + else toString (/. + "${base}/${path}"); + in if pathIsRegularFile path + # Resolve git worktrees. See gitrepository-layout(5) + then + let m = match "^gitdir: (.*)$" (lib.fileContents path); + in if m == null + then { error = "File contains no gitdir reference: " + path; } + else + let gitDir = absolutePath (dirOf path) (lib.head m); + commonDir'' = if pathIsRegularFile "${gitDir}/commondir" + then lib.fileContents "${gitDir}/commondir" + else gitDir; + commonDir' = lib.removeSuffix "/" commonDir''; + commonDir = absolutePath gitDir commonDir'; + refFile = lib.removePrefix "${commonDir}/" "${gitDir}/${file}"; + in readCommitFromFile refFile commonDir + + else if pathIsRegularFile fileName + # Sometimes git stores the commitId directly in the file but + # sometimes it stores something like: «ref: refs/heads/branch-name» + then + let fileContent = lib.fileContents fileName; + matchRef = match "^ref: (.*)$" fileContent; + in if matchRef == null + then { value = fileContent; } + else readCommitFromFile (lib.head matchRef) path + + else if pathIsRegularFile packedRefsName + # Sometimes, the file isn't there at all and has been packed away in the + # packed-refs file, so we have to grep through it: + then + let fileContent = readFile packedRefsName; + matchRef = match "([a-z0-9]+) ${file}"; + isRef = s: isString s && (matchRef s) != null; + # there is a bug in libstdc++ leading to stackoverflow for long strings: + # https://github.com/NixOS/nix/issues/2147#issuecomment-659868795 + refs = filter isRef (split "\n" fileContent); + in if refs == [] + then { error = "Could not find " + file + " in " + packedRefsName; } + else { value = lib.head (matchRef (lib.head refs)); } + + else { error = "Not a .git directory: " + toString path; }; + in readCommitFromFile "HEAD"; + + pathHasContext = builtins.hasContext or (lib.hasPrefix storeDir); + + canCleanSource = src: src ? _isLibCleanSourceWith || !(pathHasContext (toString src)); + + # -------------------------------------------------------------------------- # + # Internal functions + # + + # toSourceAttributes : sourceLike -> SourceAttrs + # + # Convert any source-like object into a simple, singular representation. + # We don't expose this representation in order to avoid having a fifth path- + # like class of objects in the wild. + # (Existing ones being: paths, strings, sources and x//{outPath}) + # So instead of exposing internals, we build a library of combinator functions. + toSourceAttributes = src: + let + isFiltered = src ? _isLibCleanSourceWith; + in + { + # The original path + origSrc = if isFiltered then src.origSrc else src; + filter = if isFiltered then src.filter else _: _: true; + name = if isFiltered then src.name else "source"; + }; + + # fromSourceAttributes : SourceAttrs -> Source + # + # Inverse of toSourceAttributes for Source objects. + fromSourceAttributes = { origSrc, filter, name }: + { + _isLibCleanSourceWith = true; + inherit origSrc filter name; + outPath = builtins.path { inherit filter name; path = origSrc; }; + }; + +in { + + pathType = lib.warnIf (lib.isInOldestRelease 2305) + "lib.sources.pathType has been moved to lib.filesystem.pathType." + lib.filesystem.pathType; + + pathIsDirectory = lib.warnIf (lib.isInOldestRelease 2305) + "lib.sources.pathIsDirectory has been moved to lib.filesystem.pathIsDirectory." + lib.filesystem.pathIsDirectory; + + pathIsRegularFile = lib.warnIf (lib.isInOldestRelease 2305) + "lib.sources.pathIsRegularFile has been moved to lib.filesystem.pathIsRegularFile." + lib.filesystem.pathIsRegularFile; + + inherit + pathIsGitRepo + commitIdFromGitRepo + + cleanSource + cleanSourceWith + cleanSourceFilter + pathHasContext + canCleanSource + + sourceByRegex + sourceFilesBySuffices + + trace + ; +} diff --git a/strings-with-deps.nix b/strings-with-deps.nix new file mode 100644 index 000000000..7b88b018d --- /dev/null +++ b/strings-with-deps.nix @@ -0,0 +1,84 @@ +{ lib }: +/* +Usage: + + You define you custom builder script by adding all build steps to a list. + for example: + builder = writeScript "fsg-4.4-builder" + (textClosure [doUnpack addInputs preBuild doMake installPhase doForceShare]); + + a step is defined by noDepEntry, fullDepEntry or packEntry. + To ensure that prerequisite are met those are added before the task itself by + textClosureDupList. Duplicated items are removed again. + + See trace/nixpkgs/trunk/pkgs/top-level/builder-defs.nix for some predefined build steps + + Attention: + + let + pkgs = (import ) {}; + in let + inherit (pkgs.stringsWithDeps) fullDepEntry packEntry noDepEntry textClosureMap; + inherit (pkgs.lib) id; + + nameA = noDepEntry "Text a"; + nameB = fullDepEntry "Text b" ["nameA"]; + nameC = fullDepEntry "Text c" ["nameA"]; + + stages = { + nameHeader = noDepEntry "#! /bin/sh \n"; + inherit nameA nameB nameC; + }; + in + textClosureMap id stages + [ "nameHeader" "nameA" "nameB" "nameC" + nameC # <- added twice. add a dep entry if you know that it will be added once only [1] + "nameB" # <- this will not be added again because the attr name (reference) is used + ] + + # result: Str("#! /bin/sh \n\nText a\nText b\nText c\nText c",[]) + + [1] maybe this behaviour should be removed to keep things simple (?) +*/ + +let + inherit (lib) + concatStringsSep + head + isAttrs + listToAttrs + tail + ; +in +rec { + + /* !!! The interface of this function is kind of messed up, since + it's way too overloaded and almost but not quite computes a + topological sort of the depstrings. */ + + textClosureList = predefined: arg: + let + f = done: todo: + if todo == [] then {result = []; inherit done;} + else + let entry = head todo; in + if isAttrs entry then + let x = f done entry.deps; + y = f x.done (tail todo); + in { result = x.result ++ [entry.text] ++ y.result; + done = y.done; + } + else if done ? ${entry} then f done (tail todo) + else f (done // listToAttrs [{name = entry; value = 1;}]) ([predefined.${entry}] ++ tail todo); + in (f {} arg).result; + + textClosureMap = f: predefined: names: + concatStringsSep "\n" (map f (textClosureList predefined names)); + + noDepEntry = text: {inherit text; deps = [];}; + fullDepEntry = text: deps: {inherit text deps;}; + packEntry = deps: {inherit deps; text="";}; + + stringAfter = deps: text: { inherit text deps; }; + +} diff --git a/strings.nix b/strings.nix new file mode 100644 index 000000000..e875520c6 --- /dev/null +++ b/strings.nix @@ -0,0 +1,1243 @@ +/* String manipulation functions. */ +{ lib }: +let + + inherit (builtins) length; + + inherit (lib.trivial) warnIf; + +asciiTable = import ./ascii-table.nix; + +in + +rec { + + inherit (builtins) + compareVersions + elem + elemAt + filter + fromJSON + head + isInt + isList + isAttrs + isPath + isString + match + parseDrvName + readFile + replaceStrings + split + storeDir + stringLength + substring + tail + toJSON + typeOf + unsafeDiscardStringContext + ; + + /* Concatenate a list of strings. + + Type: concatStrings :: [string] -> string + + Example: + concatStrings ["foo" "bar"] + => "foobar" + */ + concatStrings = builtins.concatStringsSep ""; + + /* Map a function over a list and concatenate the resulting strings. + + Type: concatMapStrings :: (a -> string) -> [a] -> string + + Example: + concatMapStrings (x: "a" + x) ["foo" "bar"] + => "afooabar" + */ + concatMapStrings = f: list: concatStrings (map f list); + + /* Like `concatMapStrings` except that the f functions also gets the + position as a parameter. + + Type: concatImapStrings :: (int -> a -> string) -> [a] -> string + + Example: + concatImapStrings (pos: x: "${toString pos}-${x}") ["foo" "bar"] + => "1-foo2-bar" + */ + concatImapStrings = f: list: concatStrings (lib.imap1 f list); + + /* Place an element between each element of a list + + Type: intersperse :: a -> [a] -> [a] + + Example: + intersperse "/" ["usr" "local" "bin"] + => ["usr" "/" "local" "/" "bin"]. + */ + intersperse = + # Separator to add between elements + separator: + # Input list + list: + if list == [] || length list == 1 + then list + else tail (lib.concatMap (x: [separator x]) list); + + /* Concatenate a list of strings with a separator between each element + + Type: concatStringsSep :: string -> [string] -> string + + Example: + concatStringsSep "/" ["usr" "local" "bin"] + => "usr/local/bin" + */ + concatStringsSep = builtins.concatStringsSep or (separator: list: + lib.foldl' (x: y: x + y) "" (intersperse separator list)); + + /* Maps a function over a list of strings and then concatenates the + result with the specified separator interspersed between + elements. + + Type: concatMapStringsSep :: string -> (a -> string) -> [a] -> string + + Example: + concatMapStringsSep "-" (x: toUpper x) ["foo" "bar" "baz"] + => "FOO-BAR-BAZ" + */ + concatMapStringsSep = + # Separator to add between elements + sep: + # Function to map over the list + f: + # List of input strings + list: concatStringsSep sep (map f list); + + /* Same as `concatMapStringsSep`, but the mapping function + additionally receives the position of its argument. + + Type: concatIMapStringsSep :: string -> (int -> a -> string) -> [a] -> string + + Example: + concatImapStringsSep "-" (pos: x: toString (x / pos)) [ 6 6 6 ] + => "6-3-2" + */ + concatImapStringsSep = + # Separator to add between elements + sep: + # Function that receives elements and their positions + f: + # List of input strings + list: concatStringsSep sep (lib.imap1 f list); + + /* Concatenate a list of strings, adding a newline at the end of each one. + Defined as `concatMapStrings (s: s + "\n")`. + + Type: concatLines :: [string] -> string + + Example: + concatLines [ "foo" "bar" ] + => "foo\nbar\n" + */ + concatLines = concatMapStrings (s: s + "\n"); + + /* Construct a Unix-style, colon-separated search path consisting of + the given `subDir` appended to each of the given paths. + + Type: makeSearchPath :: string -> [string] -> string + + Example: + makeSearchPath "bin" ["/root" "/usr" "/usr/local"] + => "/root/bin:/usr/bin:/usr/local/bin" + makeSearchPath "bin" [""] + => "/bin" + */ + makeSearchPath = + # Directory name to append + subDir: + # List of base paths + paths: + concatStringsSep ":" (map (path: path + "/" + subDir) (filter (x: x != null) paths)); + + /* Construct a Unix-style search path by appending the given + `subDir` to the specified `output` of each of the packages. If no + output by the given name is found, fallback to `.out` and then to + the default. + + Type: string -> string -> [package] -> string + + Example: + makeSearchPathOutput "dev" "bin" [ pkgs.openssl pkgs.zlib ] + => "/nix/store/9rz8gxhzf8sw4kf2j2f1grr49w8zx5vj-openssl-1.0.1r-dev/bin:/nix/store/wwh7mhwh269sfjkm6k5665b5kgp7jrk2-zlib-1.2.8/bin" + */ + makeSearchPathOutput = + # Package output to use + output: + # Directory name to append + subDir: + # List of packages + pkgs: makeSearchPath subDir (map (lib.getOutput output) pkgs); + + /* Construct a library search path (such as RPATH) containing the + libraries for a set of packages + + Example: + makeLibraryPath [ "/usr" "/usr/local" ] + => "/usr/lib:/usr/local/lib" + pkgs = import { } + makeLibraryPath [ pkgs.openssl pkgs.zlib ] + => "/nix/store/9rz8gxhzf8sw4kf2j2f1grr49w8zx5vj-openssl-1.0.1r/lib:/nix/store/wwh7mhwh269sfjkm6k5665b5kgp7jrk2-zlib-1.2.8/lib" + */ + makeLibraryPath = makeSearchPathOutput "lib" "lib"; + + /* Construct a binary search path (such as $PATH) containing the + binaries for a set of packages. + + Example: + makeBinPath ["/root" "/usr" "/usr/local"] + => "/root/bin:/usr/bin:/usr/local/bin" + */ + makeBinPath = makeSearchPathOutput "bin" "bin"; + + /* Normalize path, removing extraneous /s + + Type: normalizePath :: string -> string + + Example: + normalizePath "/a//b///c/" + => "/a/b/c/" + */ + normalizePath = s: + warnIf + (isPath s) + '' + lib.strings.normalizePath: The argument (${toString s}) is a path value, but only strings are supported. + Path values are always normalised in Nix, so there's no need to call this function on them. + This function also copies the path to the Nix store and returns the store path, the same as "''${path}" will, which may not be what you want. + This behavior is deprecated and will throw an error in the future.'' + ( + builtins.foldl' + (x: y: if y == "/" && hasSuffix "/" x then x else x+y) + "" + (stringToCharacters s) + ); + + /* Depending on the boolean `cond', return either the given string + or the empty string. Useful to concatenate against a bigger string. + + Type: optionalString :: bool -> string -> string + + Example: + optionalString true "some-string" + => "some-string" + optionalString false "some-string" + => "" + */ + optionalString = + # Condition + cond: + # String to return if condition is true + string: if cond then string else ""; + + /* Determine whether a string has given prefix. + + Type: hasPrefix :: string -> string -> bool + + Example: + hasPrefix "foo" "foobar" + => true + hasPrefix "foo" "barfoo" + => false + */ + hasPrefix = + # Prefix to check for + pref: + # Input string + str: + # Before 23.05, paths would be copied to the store before converting them + # to strings and comparing. This was surprising and confusing. + warnIf + (isPath pref) + '' + lib.strings.hasPrefix: The first argument (${toString pref}) is a path value, but only strings are supported. + There is almost certainly a bug in the calling code, since this function always returns `false` in such a case. + This function also copies the path to the Nix store, which may not be what you want. + This behavior is deprecated and will throw an error in the future.'' + (substring 0 (stringLength pref) str == pref); + + /* Determine whether a string has given suffix. + + Type: hasSuffix :: string -> string -> bool + + Example: + hasSuffix "foo" "foobar" + => false + hasSuffix "foo" "barfoo" + => true + */ + hasSuffix = + # Suffix to check for + suffix: + # Input string + content: + let + lenContent = stringLength content; + lenSuffix = stringLength suffix; + in + # Before 23.05, paths would be copied to the store before converting them + # to strings and comparing. This was surprising and confusing. + warnIf + (isPath suffix) + '' + lib.strings.hasSuffix: The first argument (${toString suffix}) is a path value, but only strings are supported. + There is almost certainly a bug in the calling code, since this function always returns `false` in such a case. + This function also copies the path to the Nix store, which may not be what you want. + This behavior is deprecated and will throw an error in the future.'' + ( + lenContent >= lenSuffix + && substring (lenContent - lenSuffix) lenContent content == suffix + ); + + /* Determine whether a string contains the given infix + + Type: hasInfix :: string -> string -> bool + + Example: + hasInfix "bc" "abcd" + => true + hasInfix "ab" "abcd" + => true + hasInfix "cd" "abcd" + => true + hasInfix "foo" "abcd" + => false + */ + hasInfix = infix: content: + # Before 23.05, paths would be copied to the store before converting them + # to strings and comparing. This was surprising and confusing. + warnIf + (isPath infix) + '' + lib.strings.hasInfix: The first argument (${toString infix}) is a path value, but only strings are supported. + There is almost certainly a bug in the calling code, since this function always returns `false` in such a case. + This function also copies the path to the Nix store, which may not be what you want. + This behavior is deprecated and will throw an error in the future.'' + (builtins.match ".*${escapeRegex infix}.*" "${content}" != null); + + /* Convert a string to a list of characters (i.e. singleton strings). + This allows you to, e.g., map a function over each character. However, + note that this will likely be horribly inefficient; Nix is not a + general purpose programming language. Complex string manipulations + should, if appropriate, be done in a derivation. + Also note that Nix treats strings as a list of bytes and thus doesn't + handle unicode. + + Type: stringToCharacters :: string -> [string] + + Example: + stringToCharacters "" + => [ ] + stringToCharacters "abc" + => [ "a" "b" "c" ] + stringToCharacters "🦄" + => [ "�" "�" "�" "�" ] + */ + stringToCharacters = s: + map (p: substring p 1 s) (lib.range 0 (stringLength s - 1)); + + /* Manipulate a string character by character and replace them by + strings before concatenating the results. + + Type: stringAsChars :: (string -> string) -> string -> string + + Example: + stringAsChars (x: if x == "a" then "i" else x) "nax" + => "nix" + */ + stringAsChars = + # Function to map over each individual character + f: + # Input string + s: concatStrings ( + map f (stringToCharacters s) + ); + + /* Convert char to ascii value, must be in printable range + + Type: charToInt :: string -> int + + Example: + charToInt "A" + => 65 + charToInt "(" + => 40 + + */ + charToInt = c: builtins.getAttr c asciiTable; + + /* Escape occurrence of the elements of `list` in `string` by + prefixing it with a backslash. + + Type: escape :: [string] -> string -> string + + Example: + escape ["(" ")"] "(foo)" + => "\\(foo\\)" + */ + escape = list: replaceStrings list (map (c: "\\${c}") list); + + /* Escape occurrence of the element of `list` in `string` by + converting to its ASCII value and prefixing it with \\x. + Only works for printable ascii characters. + + Type: escapeC = [string] -> string -> string + + Example: + escapeC [" "] "foo bar" + => "foo\\x20bar" + + */ + escapeC = list: replaceStrings list (map (c: "\\x${ toLower (lib.toHexString (charToInt c))}") list); + + /* Escape the string so it can be safely placed inside a URL + query. + + Type: escapeURL :: string -> string + + Example: + escapeURL "foo/bar baz" + => "foo%2Fbar%20baz" + */ + escapeURL = let + unreserved = [ "A" "B" "C" "D" "E" "F" "G" "H" "I" "J" "K" "L" "M" "N" "O" "P" "Q" "R" "S" "T" "U" "V" "W" "X" "Y" "Z" "a" "b" "c" "d" "e" "f" "g" "h" "i" "j" "k" "l" "m" "n" "o" "p" "q" "r" "s" "t" "u" "v" "w" "x" "y" "z" "0" "1" "2" "3" "4" "5" "6" "7" "8" "9" "-" "_" "." "~" ]; + toEscape = builtins.removeAttrs asciiTable unreserved; + in + replaceStrings (builtins.attrNames toEscape) (lib.mapAttrsToList (_: c: "%${fixedWidthString 2 "0" (lib.toHexString c)}") toEscape); + + /* Quote string to be used safely within the Bourne shell. + + Type: escapeShellArg :: string -> string + + Example: + escapeShellArg "esc'ape\nme" + => "'esc'\\''ape\nme'" + */ + escapeShellArg = arg: "'${replaceStrings ["'"] ["'\\''"] (toString arg)}'"; + + /* Quote all arguments to be safely passed to the Bourne shell. + + Type: escapeShellArgs :: [string] -> string + + Example: + escapeShellArgs ["one" "two three" "four'five"] + => "'one' 'two three' 'four'\\''five'" + */ + escapeShellArgs = concatMapStringsSep " " escapeShellArg; + + /* Test whether the given name is a valid POSIX shell variable name. + + Type: string -> bool + + Example: + isValidPosixName "foo_bar000" + => true + isValidPosixName "0-bad.jpg" + => false + */ + isValidPosixName = name: match "[a-zA-Z_][a-zA-Z0-9_]*" name != null; + + /* Translate a Nix value into a shell variable declaration, with proper escaping. + + The value can be a string (mapped to a regular variable), a list of strings + (mapped to a Bash-style array) or an attribute set of strings (mapped to a + Bash-style associative array). Note that "string" includes string-coercible + values like paths or derivations. + + Strings are translated into POSIX sh-compatible code; lists and attribute sets + assume a shell that understands Bash syntax (e.g. Bash or ZSH). + + Type: string -> (string | listOf string | attrsOf string) -> string + + Example: + '' + ${toShellVar "foo" "some string"} + [[ "$foo" == "some string" ]] + '' + */ + toShellVar = name: value: + lib.throwIfNot (isValidPosixName name) "toShellVar: ${name} is not a valid shell variable name" ( + if isAttrs value && ! isStringLike value then + "declare -A ${name}=(${ + concatStringsSep " " (lib.mapAttrsToList (n: v: + "[${escapeShellArg n}]=${escapeShellArg v}" + ) value) + })" + else if isList value then + "declare -a ${name}=(${escapeShellArgs value})" + else + "${name}=${escapeShellArg value}" + ); + + /* Translate an attribute set into corresponding shell variable declarations + using `toShellVar`. + + Type: attrsOf (string | listOf string | attrsOf string) -> string + + Example: + let + foo = "value"; + bar = foo; + in '' + ${toShellVars { inherit foo bar; }} + [[ "$foo" == "$bar" ]] + '' + */ + toShellVars = vars: concatStringsSep "\n" (lib.mapAttrsToList toShellVar vars); + + /* Turn a string into a Nix expression representing that string + + Type: string -> string + + Example: + escapeNixString "hello\${}\n" + => "\"hello\\\${}\\n\"" + */ + escapeNixString = s: escape ["$"] (toJSON s); + + /* Turn a string into an exact regular expression + + Type: string -> string + + Example: + escapeRegex "[^a-z]*" + => "\\[\\^a-z]\\*" + */ + escapeRegex = escape (stringToCharacters "\\[{()^$?*+|."); + + /* Quotes a string if it can't be used as an identifier directly. + + Type: string -> string + + Example: + escapeNixIdentifier "hello" + => "hello" + escapeNixIdentifier "0abc" + => "\"0abc\"" + */ + escapeNixIdentifier = s: + # Regex from https://github.com/NixOS/nix/blob/d048577909e383439c2549e849c5c2f2016c997e/src/libexpr/lexer.l#L91 + if match "[a-zA-Z_][a-zA-Z0-9_'-]*" s != null + then s else escapeNixString s; + + /* Escapes a string such that it is safe to include verbatim in an XML + document. + + Type: string -> string + + Example: + escapeXML ''"test" 'test' < & >'' + => ""test" 'test' < & >" + */ + escapeXML = builtins.replaceStrings + ["\"" "'" "<" ">" "&"] + [""" "'" "<" ">" "&"]; + + # warning added 12-12-2022 + replaceChars = lib.warn "replaceChars is a deprecated alias of replaceStrings, replace usages of it with replaceStrings." builtins.replaceStrings; + + # Case conversion utilities. + lowerChars = stringToCharacters "abcdefghijklmnopqrstuvwxyz"; + upperChars = stringToCharacters "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + /* Converts an ASCII string to lower-case. + + Type: toLower :: string -> string + + Example: + toLower "HOME" + => "home" + */ + toLower = replaceStrings upperChars lowerChars; + + /* Converts an ASCII string to upper-case. + + Type: toUpper :: string -> string + + Example: + toUpper "home" + => "HOME" + */ + toUpper = replaceStrings lowerChars upperChars; + + /* Appends string context from another string. This is an implementation + detail of Nix and should be used carefully. + + Strings in Nix carry an invisible `context` which is a list of strings + representing store paths. If the string is later used in a derivation + attribute, the derivation will properly populate the inputDrvs and + inputSrcs. + + Example: + pkgs = import { }; + addContextFrom pkgs.coreutils "bar" + => "bar" + */ + addContextFrom = a: b: substring 0 0 a + b; + + /* Cut a string with a separator and produces a list of strings which + were separated by this separator. + + Example: + splitString "." "foo.bar.baz" + => [ "foo" "bar" "baz" ] + splitString "/" "/usr/local/bin" + => [ "" "usr" "local" "bin" ] + */ + splitString = sep: s: + let + splits = builtins.filter builtins.isString (builtins.split (escapeRegex (toString sep)) (toString s)); + in + map (addContextFrom s) splits; + + /* Return a string without the specified prefix, if the prefix matches. + + Type: string -> string -> string + + Example: + removePrefix "foo." "foo.bar.baz" + => "bar.baz" + removePrefix "xxx" "foo.bar.baz" + => "foo.bar.baz" + */ + removePrefix = + # Prefix to remove if it matches + prefix: + # Input string + str: + # Before 23.05, paths would be copied to the store before converting them + # to strings and comparing. This was surprising and confusing. + warnIf + (isPath prefix) + '' + lib.strings.removePrefix: The first argument (${toString prefix}) is a path value, but only strings are supported. + There is almost certainly a bug in the calling code, since this function never removes any prefix in such a case. + This function also copies the path to the Nix store, which may not be what you want. + This behavior is deprecated and will throw an error in the future.'' + (let + preLen = stringLength prefix; + sLen = stringLength str; + in + if substring 0 preLen str == prefix then + substring preLen (sLen - preLen) str + else + str); + + /* Return a string without the specified suffix, if the suffix matches. + + Type: string -> string -> string + + Example: + removeSuffix "front" "homefront" + => "home" + removeSuffix "xxx" "homefront" + => "homefront" + */ + removeSuffix = + # Suffix to remove if it matches + suffix: + # Input string + str: + # Before 23.05, paths would be copied to the store before converting them + # to strings and comparing. This was surprising and confusing. + warnIf + (isPath suffix) + '' + lib.strings.removeSuffix: The first argument (${toString suffix}) is a path value, but only strings are supported. + There is almost certainly a bug in the calling code, since this function never removes any suffix in such a case. + This function also copies the path to the Nix store, which may not be what you want. + This behavior is deprecated and will throw an error in the future.'' + (let + sufLen = stringLength suffix; + sLen = stringLength str; + in + if sufLen <= sLen && suffix == substring (sLen - sufLen) sufLen str then + substring 0 (sLen - sufLen) str + else + str); + + /* Return true if string v1 denotes a version older than v2. + + Example: + versionOlder "1.1" "1.2" + => true + versionOlder "1.1" "1.1" + => false + */ + versionOlder = v1: v2: compareVersions v2 v1 == 1; + + /* Return true if string v1 denotes a version equal to or newer than v2. + + Example: + versionAtLeast "1.1" "1.0" + => true + versionAtLeast "1.1" "1.1" + => true + versionAtLeast "1.1" "1.2" + => false + */ + versionAtLeast = v1: v2: !versionOlder v1 v2; + + /* This function takes an argument that's either a derivation or a + derivation's "name" attribute and extracts the name part from that + argument. + + Example: + getName "youtube-dl-2016.01.01" + => "youtube-dl" + getName pkgs.youtube-dl + => "youtube-dl" + */ + getName = x: + let + parse = drv: (parseDrvName drv).name; + in if isString x + then parse x + else x.pname or (parse x.name); + + /* This function takes an argument that's either a derivation or a + derivation's "name" attribute and extracts the version part from that + argument. + + Example: + getVersion "youtube-dl-2016.01.01" + => "2016.01.01" + getVersion pkgs.youtube-dl + => "2016.01.01" + */ + getVersion = x: + let + parse = drv: (parseDrvName drv).version; + in if isString x + then parse x + else x.version or (parse x.name); + + /* Extract name with version from URL. Ask for separator which is + supposed to start extension. + + Example: + nameFromURL "https://nixos.org/releases/nix/nix-1.7/nix-1.7-x86_64-linux.tar.bz2" "-" + => "nix" + nameFromURL "https://nixos.org/releases/nix/nix-1.7/nix-1.7-x86_64-linux.tar.bz2" "_" + => "nix-1.7-x86" + */ + nameFromURL = url: sep: + let + components = splitString "/" url; + filename = lib.last components; + name = head (splitString sep filename); + in assert name != filename; name; + + /* Create a -D= string that can be passed to typical Meson + invocations. + + Type: mesonOption :: string -> string -> string + + @param feature The feature to be set + @param value The desired value + + Example: + mesonOption "engine" "opengl" + => "-Dengine=opengl" + */ + mesonOption = feature: value: + assert (lib.isString feature); + assert (lib.isString value); + "-D${feature}=${value}"; + + /* Create a -D={true,false} string that can be passed to typical + Meson invocations. + + Type: mesonBool :: string -> bool -> string + + @param condition The condition to be made true or false + @param flag The controlling flag of the condition + + Example: + mesonBool "hardened" true + => "-Dhardened=true" + mesonBool "static" false + => "-Dstatic=false" + */ + mesonBool = condition: flag: + assert (lib.isString condition); + assert (lib.isBool flag); + mesonOption condition (lib.boolToString flag); + + /* Create a -D={enabled,disabled} string that can be passed to + typical Meson invocations. + + Type: mesonEnable :: string -> bool -> string + + @param feature The feature to be enabled or disabled + @param flag The controlling flag + + Example: + mesonEnable "docs" true + => "-Ddocs=enabled" + mesonEnable "savage" false + => "-Dsavage=disabled" + */ + mesonEnable = feature: flag: + assert (lib.isString feature); + assert (lib.isBool flag); + mesonOption feature (if flag then "enabled" else "disabled"); + + /* Create an --{enable,disable}- string that can be passed to + standard GNU Autoconf scripts. + + Example: + enableFeature true "shared" + => "--enable-shared" + enableFeature false "shared" + => "--disable-shared" + */ + enableFeature = enable: feat: + assert isString feat; # e.g. passing openssl instead of "openssl" + "--${if enable then "enable" else "disable"}-${feat}"; + + /* Create an --{enable-=,disable-} string that can be passed to + standard GNU Autoconf scripts. + + Example: + enableFeatureAs true "shared" "foo" + => "--enable-shared=foo" + enableFeatureAs false "shared" (throw "ignored") + => "--disable-shared" + */ + enableFeatureAs = enable: feat: value: enableFeature enable feat + optionalString enable "=${value}"; + + /* Create an --{with,without}- string that can be passed to + standard GNU Autoconf scripts. + + Example: + withFeature true "shared" + => "--with-shared" + withFeature false "shared" + => "--without-shared" + */ + withFeature = with_: feat: + assert isString feat; # e.g. passing openssl instead of "openssl" + "--${if with_ then "with" else "without"}-${feat}"; + + /* Create an --{with-=,without-} string that can be passed to + standard GNU Autoconf scripts. + + Example: + withFeatureAs true "shared" "foo" + => "--with-shared=foo" + withFeatureAs false "shared" (throw "ignored") + => "--without-shared" + */ + withFeatureAs = with_: feat: value: withFeature with_ feat + optionalString with_ "=${value}"; + + /* Create a fixed width string with additional prefix to match + required width. + + This function will fail if the input string is longer than the + requested length. + + Type: fixedWidthString :: int -> string -> string -> string + + Example: + fixedWidthString 5 "0" (toString 15) + => "00015" + */ + fixedWidthString = width: filler: str: + let + strw = lib.stringLength str; + reqWidth = width - (lib.stringLength filler); + in + assert lib.assertMsg (strw <= width) + "fixedWidthString: requested string length (${ + toString width}) must not be shorter than actual length (${ + toString strw})"; + if strw == width then str else filler + fixedWidthString reqWidth filler str; + + /* Format a number adding leading zeroes up to fixed width. + + Example: + fixedWidthNumber 5 15 + => "00015" + */ + fixedWidthNumber = width: n: fixedWidthString width "0" (toString n); + + /* Convert a float to a string, but emit a warning when precision is lost + during the conversion + + Example: + floatToString 0.000001 + => "0.000001" + floatToString 0.0000001 + => trace: warning: Imprecise conversion from float to string 0.000000 + "0.000000" + */ + floatToString = float: let + result = toString float; + precise = float == fromJSON result; + in lib.warnIf (!precise) "Imprecise conversion from float to string ${result}" + result; + + /* Soft-deprecated function. While the original implementation is available as + isConvertibleWithToString, consider using isStringLike instead, if suitable. */ + isCoercibleToString = lib.warnIf (lib.isInOldestRelease 2305) + "lib.strings.isCoercibleToString is deprecated in favor of either isStringLike or isConvertibleWithToString. Only use the latter if it needs to return true for null, numbers, booleans and list of similarly coercibles." + isConvertibleWithToString; + + /* Check whether a list or other value can be passed to toString. + + Many types of value are coercible to string this way, including int, float, + null, bool, list of similarly coercible values. + */ + isConvertibleWithToString = x: + isStringLike x || + elem (typeOf x) [ "null" "int" "float" "bool" ] || + (isList x && lib.all isConvertibleWithToString x); + + /* Check whether a value can be coerced to a string. + The value must be a string, path, or attribute set. + + String-like values can be used without explicit conversion in + string interpolations and in most functions that expect a string. + */ + isStringLike = x: + isString x || + isPath x || + x ? outPath || + x ? __toString; + + /* Check whether a value is a store path. + + Example: + isStorePath "/nix/store/d945ibfx9x185xf04b890y4f9g3cbb63-python-2.7.11/bin/python" + => false + isStorePath "/nix/store/d945ibfx9x185xf04b890y4f9g3cbb63-python-2.7.11" + => true + isStorePath pkgs.python + => true + isStorePath [] || isStorePath 42 || isStorePath {} || … + => false + */ + isStorePath = x: + if isStringLike x then + let str = toString x; in + substring 0 1 str == "/" + && dirOf str == storeDir + else + false; + + /* Parse a string as an int. Does not support parsing of integers with preceding zero due to + ambiguity between zero-padded and octal numbers. See toIntBase10. + + Type: string -> int + + Example: + + toInt "1337" + => 1337 + + toInt "-4" + => -4 + + toInt " 123 " + => 123 + + toInt "00024" + => error: Ambiguity in interpretation of 00024 between octal and zero padded integer. + + toInt "3.14" + => error: floating point JSON numbers are not supported + */ + toInt = str: + let + # RegEx: Match any leading whitespace, possibly a '-', one or more digits, + # and finally match any trailing whitespace. + strippedInput = match "[[:space:]]*(-?[[:digit:]]+)[[:space:]]*" str; + + # RegEx: Match a leading '0' then one or more digits. + isLeadingZero = match "0[[:digit:]]+" (head strippedInput) == []; + + # Attempt to parse input + parsedInput = fromJSON (head strippedInput); + + generalError = "toInt: Could not convert ${escapeNixString str} to int."; + + octalAmbigError = "toInt: Ambiguity in interpretation of ${escapeNixString str}" + + " between octal and zero padded integer."; + + in + # Error on presence of non digit characters. + if strippedInput == null + then throw generalError + # Error on presence of leading zero/octal ambiguity. + else if isLeadingZero + then throw octalAmbigError + # Error if parse function fails. + else if !isInt parsedInput + then throw generalError + # Return result. + else parsedInput; + + + /* Parse a string as a base 10 int. This supports parsing of zero-padded integers. + + Type: string -> int + + Example: + toIntBase10 "1337" + => 1337 + + toIntBase10 "-4" + => -4 + + toIntBase10 " 123 " + => 123 + + toIntBase10 "00024" + => 24 + + toIntBase10 "3.14" + => error: floating point JSON numbers are not supported + */ + toIntBase10 = str: + let + # RegEx: Match any leading whitespace, then match any zero padding, + # capture possibly a '-' followed by one or more digits, + # and finally match any trailing whitespace. + strippedInput = match "[[:space:]]*0*(-?[[:digit:]]+)[[:space:]]*" str; + + # RegEx: Match at least one '0'. + isZero = match "0+" (head strippedInput) == []; + + # Attempt to parse input + parsedInput = fromJSON (head strippedInput); + + generalError = "toIntBase10: Could not convert ${escapeNixString str} to int."; + + in + # Error on presence of non digit characters. + if strippedInput == null + then throw generalError + # In the special case zero-padded zero (00000), return early. + else if isZero + then 0 + # Error if parse function fails. + else if !isInt parsedInput + then throw generalError + # Return result. + else parsedInput; + + /* Read a list of paths from `file`, relative to the `rootPath`. + Lines beginning with `#` are treated as comments and ignored. + Whitespace is significant. + + NOTE: This function is not performant and should be avoided. + + Example: + readPathsFromFile /prefix + ./pkgs/development/libraries/qt-5/5.4/qtbase/series + => [ "/prefix/dlopen-resolv.patch" "/prefix/tzdir.patch" + "/prefix/dlopen-libXcursor.patch" "/prefix/dlopen-openssl.patch" + "/prefix/dlopen-dbus.patch" "/prefix/xdg-config-dirs.patch" + "/prefix/nix-profiles-library-paths.patch" + "/prefix/compose-search-path.patch" ] + */ + readPathsFromFile = lib.warn "lib.readPathsFromFile is deprecated, use a list instead" + (rootPath: file: + let + lines = lib.splitString "\n" (readFile file); + removeComments = lib.filter (line: line != "" && !(lib.hasPrefix "#" line)); + relativePaths = removeComments lines; + absolutePaths = map (path: rootPath + "/${path}") relativePaths; + in + absolutePaths); + + /* Read the contents of a file removing the trailing \n + + Type: fileContents :: path -> string + + Example: + $ echo "1.0" > ./version + + fileContents ./version + => "1.0" + */ + fileContents = file: removeSuffix "\n" (readFile file); + + + /* Creates a valid derivation name from a potentially invalid one. + + Type: sanitizeDerivationName :: String -> String + + Example: + sanitizeDerivationName "../hello.bar # foo" + => "-hello.bar-foo" + sanitizeDerivationName "" + => "unknown" + sanitizeDerivationName pkgs.hello + => "-nix-store-2g75chlbpxlrqn15zlby2dfh8hr9qwbk-hello-2.10" + */ + sanitizeDerivationName = + let okRegex = match "[[:alnum:]+_?=-][[:alnum:]+._?=-]*"; + in + string: + # First detect the common case of already valid strings, to speed those up + if stringLength string <= 207 && okRegex string != null + then unsafeDiscardStringContext string + else lib.pipe string [ + # Get rid of string context. This is safe under the assumption that the + # resulting string is only used as a derivation name + unsafeDiscardStringContext + # Strip all leading "." + (x: elemAt (match "\\.*(.*)" x) 0) + # Split out all invalid characters + # https://github.com/NixOS/nix/blob/2.3.2/src/libstore/store-api.cc#L85-L112 + # https://github.com/NixOS/nix/blob/2242be83c61788b9c0736a92bb0b5c7bbfc40803/nix-rust/src/store/path.rs#L100-L125 + (split "[^[:alnum:]+._?=-]+") + # Replace invalid character ranges with a "-" + (concatMapStrings (s: if lib.isList s then "-" else s)) + # Limit to 211 characters (minus 4 chars for ".drv") + (x: substring (lib.max (stringLength x - 207) 0) (-1) x) + # If the result is empty, replace it with "unknown" + (x: if stringLength x == 0 then "unknown" else x) + ]; + + /* Computes the Levenshtein distance between two strings. + Complexity O(n*m) where n and m are the lengths of the strings. + Algorithm adjusted from https://stackoverflow.com/a/9750974/6605742 + + Type: levenshtein :: string -> string -> int + + Example: + levenshtein "foo" "foo" + => 0 + levenshtein "book" "hook" + => 1 + levenshtein "hello" "Heyo" + => 3 + */ + levenshtein = a: b: let + # Two dimensional array with dimensions (stringLength a + 1, stringLength b + 1) + arr = lib.genList (i: + lib.genList (j: + dist i j + ) (stringLength b + 1) + ) (stringLength a + 1); + d = x: y: lib.elemAt (lib.elemAt arr x) y; + dist = i: j: + let c = if substring (i - 1) 1 a == substring (j - 1) 1 b + then 0 else 1; + in + if j == 0 then i + else if i == 0 then j + else lib.min + ( lib.min (d (i - 1) j + 1) (d i (j - 1) + 1)) + ( d (i - 1) (j - 1) + c ); + in d (stringLength a) (stringLength b); + + /* Returns the length of the prefix common to both strings. + */ + commonPrefixLength = a: b: + let + m = lib.min (stringLength a) (stringLength b); + go = i: if i >= m then m else if substring i 1 a == substring i 1 b then go (i + 1) else i; + in go 0; + + /* Returns the length of the suffix common to both strings. + */ + commonSuffixLength = a: b: + let + m = lib.min (stringLength a) (stringLength b); + go = i: if i >= m then m else if substring (stringLength a - i - 1) 1 a == substring (stringLength b - i - 1) 1 b then go (i + 1) else i; + in go 0; + + /* Returns whether the levenshtein distance between two strings is at most some value + Complexity is O(min(n,m)) for k <= 2 and O(n*m) otherwise + + Type: levenshteinAtMost :: int -> string -> string -> bool + + Example: + levenshteinAtMost 0 "foo" "foo" + => true + levenshteinAtMost 1 "foo" "boa" + => false + levenshteinAtMost 2 "foo" "boa" + => true + levenshteinAtMost 2 "This is a sentence" "this is a sentense." + => false + levenshteinAtMost 3 "This is a sentence" "this is a sentense." + => true + + */ + levenshteinAtMost = let + infixDifferAtMost1 = x: y: stringLength x <= 1 && stringLength y <= 1; + + # This function takes two strings stripped by their common pre and suffix, + # and returns whether they differ by at most two by Levenshtein distance. + # Because of this stripping, if they do indeed differ by at most two edits, + # we know that those edits were (if at all) done at the start or the end, + # while the middle has to have stayed the same. This fact is used in the + # implementation. + infixDifferAtMost2 = x: y: + let + xlen = stringLength x; + ylen = stringLength y; + # This function is only called with |x| >= |y| and |x| - |y| <= 2, so + # diff is one of 0, 1 or 2 + diff = xlen - ylen; + + # Infix of x and y, stripped by the left and right most character + xinfix = substring 1 (xlen - 2) x; + yinfix = substring 1 (ylen - 2) y; + + # x and y but a character deleted at the left or right + xdelr = substring 0 (xlen - 1) x; + xdell = substring 1 (xlen - 1) x; + ydelr = substring 0 (ylen - 1) y; + ydell = substring 1 (ylen - 1) y; + in + # A length difference of 2 can only be gotten with 2 delete edits, + # which have to have happened at the start and end of x + # Example: "abcdef" -> "bcde" + if diff == 2 then xinfix == y + # A length difference of 1 can only be gotten with a deletion on the + # right and a replacement on the left or vice versa. + # Example: "abcdef" -> "bcdez" or "zbcde" + else if diff == 1 then xinfix == ydelr || xinfix == ydell + # No length difference can either happen through replacements on both + # sides, or a deletion on the left and an insertion on the right or + # vice versa + # Example: "abcdef" -> "zbcdez" or "bcdefz" or "zabcde" + else xinfix == yinfix || xdelr == ydell || xdell == ydelr; + + in k: if k <= 0 then a: b: a == b else + let f = a: b: + let + alen = stringLength a; + blen = stringLength b; + prelen = commonPrefixLength a b; + suflen = commonSuffixLength a b; + presuflen = prelen + suflen; + ainfix = substring prelen (alen - presuflen) a; + binfix = substring prelen (blen - presuflen) b; + in + # Make a be the bigger string + if alen < blen then f b a + # If a has over k more characters than b, even with k deletes on a, b can't be reached + else if alen - blen > k then false + else if k == 1 then infixDifferAtMost1 ainfix binfix + else if k == 2 then infixDifferAtMost2 ainfix binfix + else levenshtein ainfix binfix <= k; + in f; +} diff --git a/systems/architectures.nix b/systems/architectures.nix new file mode 100644 index 000000000..57b9184ca --- /dev/null +++ b/systems/architectures.nix @@ -0,0 +1,114 @@ +{ lib }: + +rec { + # gcc.arch to its features (as in /proc/cpuinfo) + features = { + default = [ ]; + # x86_64 Intel + westmere = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" ]; + sandybridge = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" ]; + ivybridge = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" ]; + haswell = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" "avx2" "fma" ]; + broadwell = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" "avx2" "fma" ]; + skylake = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" "avx2" "fma" ]; + skylake-avx512 = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" "avx2" "avx512" "fma" ]; + cannonlake = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" "avx2" "avx512" "fma" ]; + icelake-client = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" "avx2" "avx512" "fma" ]; + icelake-server = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" "avx2" "avx512" "fma" ]; + cascadelake = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" "avx2" "avx512" "fma" ]; + cooperlake = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" "avx2" "avx512" "fma" ]; + tigerlake = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" "avx2" "avx512" "fma" ]; + # x86_64 AMD + btver1 = [ "sse3" "ssse3" "sse4_1" "sse4_2" ]; + btver2 = [ "sse3" "ssse3" "sse4_1" "sse4_2" "aes" "avx" ]; + bdver1 = [ "sse3" "ssse3" "sse4_1" "sse4_2" "sse4a" "aes" "avx" "fma" "fma4" ]; + bdver2 = [ "sse3" "ssse3" "sse4_1" "sse4_2" "sse4a" "aes" "avx" "fma" "fma4" ]; + bdver3 = [ "sse3" "ssse3" "sse4_1" "sse4_2" "sse4a" "aes" "avx" "fma" "fma4" ]; + bdver4 = [ "sse3" "ssse3" "sse4_1" "sse4_2" "sse4a" "aes" "avx" "avx2" "fma" "fma4" ]; + znver1 = [ "sse3" "ssse3" "sse4_1" "sse4_2" "sse4a" "aes" "avx" "avx2" "fma" ]; + znver2 = [ "sse3" "ssse3" "sse4_1" "sse4_2" "sse4a" "aes" "avx" "avx2" "fma" ]; + znver3 = [ "sse3" "ssse3" "sse4_1" "sse4_2" "sse4a" "aes" "avx" "avx2" "fma" ]; + # other + armv5te = [ ]; + armv6 = [ ]; + armv7-a = [ ]; + armv8-a = [ ]; + mips32 = [ ]; + loongson2f = [ ]; + }; + + # a superior CPU has all the features of an inferior and is able to build and test code for it + inferiors = { + # x86_64 Intel + # https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html + default = [ ]; + westmere = [ ]; + sandybridge = [ "westmere" ] ++ inferiors.westmere; + ivybridge = [ "sandybridge" ] ++ inferiors.sandybridge; + haswell = [ "ivybridge" ] ++ inferiors.ivybridge; + broadwell = [ "haswell" ] ++ inferiors.haswell; + skylake = [ "broadwell" ] ++ inferiors.broadwell; + skylake-avx512 = [ "skylake" ] ++ inferiors.skylake; + cannonlake = [ "skylake-avx512" ] ++ inferiors.skylake-avx512; + icelake-client = [ "cannonlake" ] ++ inferiors.cannonlake; + icelake-server = [ "icelake-client" ] ++ inferiors.icelake-client; + cascadelake = [ "skylake-avx512" ] ++ inferiors.cannonlake; + cooperlake = [ "cascadelake" ] ++ inferiors.cascadelake; + tigerlake = [ "icelake-server" ] ++ inferiors.icelake-server; + + # x86_64 AMD + # TODO: fill this (need testing) + btver1 = [ ]; + btver2 = [ ]; + bdver1 = [ ]; + bdver2 = [ ]; + bdver3 = [ ]; + bdver4 = [ ]; + # Regarding `skylake` as inferior of `znver1`, there are reports of + # successful usage by Gentoo users and Phoronix benchmarking of different + # `-march` targets. + # + # The GCC documentation on extensions used and wikichip documentation + # regarding supperted extensions on znver1 and skylake was used to create + # this partial order. + # + # Note: + # + # - The successors of `skylake` (`cannonlake`, `icelake`, etc) use `avx512` + # which no current AMD Zen michroarch support. + # - `znver1` uses `ABM`, `CLZERO`, `CX16`, `MWAITX`, and `SSE4A` which no + # current Intel microarch support. + # + # https://www.phoronix.com/scan.php?page=article&item=amd-znver3-gcc11&num=1 + # https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html + # https://en.wikichip.org/wiki/amd/microarchitectures/zen + # https://en.wikichip.org/wiki/intel/microarchitectures/skylake + znver1 = [ "skylake" ] ++ inferiors.skylake; + znver2 = [ "znver1" ] ++ inferiors.znver1; + znver3 = [ "znver2" ] ++ inferiors.znver2; + + # other + armv5te = [ ]; + armv6 = [ ]; + armv7-a = [ ]; + armv8-a = [ ]; + mips32 = [ ]; + loongson2f = [ ]; + }; + + predicates = let + featureSupport = feature: x: builtins.elem feature features.${x} or []; + in { + sse3Support = featureSupport "sse3"; + ssse3Support = featureSupport "ssse3"; + sse4_1Support = featureSupport "sse4_1"; + sse4_2Support = featureSupport "sse4_2"; + sse4_aSupport = featureSupport "sse4a"; + avxSupport = featureSupport "avx"; + avx2Support = featureSupport "avx2"; + avx512Support = featureSupport "avx512"; + aesSupport = featureSupport "aes"; + fmaSupport = featureSupport "fma"; + fma4Support = featureSupport "fma4"; + }; +} diff --git a/systems/default.nix b/systems/default.nix new file mode 100644 index 000000000..f4784c61c --- /dev/null +++ b/systems/default.nix @@ -0,0 +1,228 @@ +{ lib }: + let inherit (lib.attrsets) mapAttrs; in + +rec { + doubles = import ./doubles.nix { inherit lib; }; + parse = import ./parse.nix { inherit lib; }; + inspect = import ./inspect.nix { inherit lib; }; + platforms = import ./platforms.nix { inherit lib; }; + examples = import ./examples.nix { inherit lib; }; + architectures = import ./architectures.nix { inherit lib; }; + + /* List of all Nix system doubles the nixpkgs flake will expose the package set + for. All systems listed here must be supported by nixpkgs as `localSystem`. + + **Warning**: This attribute is considered experimental and is subject to change. + */ + flakeExposed = import ./flake-systems.nix { }; + + # Elaborate a `localSystem` or `crossSystem` so that it contains everything + # necessary. + # + # `parsed` is inferred from args, both because there are two options with one + # clearly preferred, and to prevent cycles. A simpler fixed point where the RHS + # always just used `final.*` would fail on both counts. + elaborate = args': let + args = if lib.isString args' then { system = args'; } + else args'; + final = { + # Prefer to parse `config` as it is strictly more informative. + parsed = parse.mkSystemFromString (if args ? config then args.config else args.system); + # Either of these can be losslessly-extracted from `parsed` iff parsing succeeds. + system = parse.doubleFromSystem final.parsed; + config = parse.tripleFromSystem final.parsed; + # Determine whether we can execute binaries built for the provided platform. + canExecute = platform: + final.isAndroid == platform.isAndroid && + parse.isCompatible final.parsed.cpu platform.parsed.cpu + && final.parsed.kernel == platform.parsed.kernel; + isCompatible = _: throw "2022-05-23: isCompatible has been removed in favor of canExecute, refer to the 22.11 changelog for details"; + # Derived meta-data + libc = + /**/ if final.isDarwin then "libSystem" + else if final.isMinGW then "msvcrt" + else if final.isWasi then "wasilibc" + else if final.isRedox then "relibc" + else if final.isMusl then "musl" + else if final.isUClibc then "uclibc" + else if final.isAndroid then "bionic" + else if final.isLinux /* default */ then "glibc" + else if final.isFreeBSD then "fblibc" + else if final.isNetBSD then "nblibc" + else if final.isAvr then "avrlibc" + else if final.isGhcjs then null + else if final.isNone then "newlib" + # TODO(@Ericson2314) think more about other operating systems + else "native/impure"; + # Choose what linker we wish to use by default. Someday we might also + # choose the C compiler, runtime library, C++ standard library, etc. in + # this way, nice and orthogonally, and deprecate `useLLVM`. But due to + # the monolithic GCC build we cannot actually make those choices + # independently, so we are just doing `linker` and keeping `useLLVM` for + # now. + linker = + /**/ if final.useLLVM or false then "lld" + else if final.isDarwin then "cctools" + # "bfd" and "gold" both come from GNU binutils. The existence of Gold + # is why we use the more obscure "bfd" and not "binutils" for this + # choice. + else "bfd"; + extensions = rec { + sharedLibrary = + /**/ if final.isDarwin then ".dylib" + else if final.isWindows then ".dll" + else ".so"; + staticLibrary = + /**/ if final.isWindows then ".lib" + else ".a"; + library = + /**/ if final.isStatic then staticLibrary + else sharedLibrary; + executable = + /**/ if final.isWindows then ".exe" + else ""; + }; + # Misc boolean options + useAndroidPrebuilt = false; + useiOSPrebuilt = false; + + # Output from uname + uname = { + # uname -s + system = { + linux = "Linux"; + windows = "Windows"; + darwin = "Darwin"; + netbsd = "NetBSD"; + freebsd = "FreeBSD"; + openbsd = "OpenBSD"; + wasi = "Wasi"; + redox = "Redox"; + genode = "Genode"; + }.${final.parsed.kernel.name} or null; + + # uname -m + processor = + if final.isPower64 + then "ppc64${lib.optionalString final.isLittleEndian "le"}" + else if final.isPower + then "ppc${lib.optionalString final.isLittleEndian "le"}" + else if final.isMips64 + then "mips64" # endianness is *not* included on mips64 + else final.parsed.cpu.name; + + # uname -r + release = null; + }; + isStatic = final.isWasm || final.isRedox; + + # Just a guess, based on `system` + inherit + ({ + linux-kernel = args.linux-kernel or {}; + gcc = args.gcc or {}; + rustc = args.rustc or {}; + } // platforms.select final) + linux-kernel gcc rustc; + + linuxArch = + if final.isAarch32 then "arm" + else if final.isAarch64 then "arm64" + else if final.isx86_32 then "i386" + else if final.isx86_64 then "x86_64" + # linux kernel does not distinguish microblaze/microblazeel + else if final.isMicroBlaze then "microblaze" + else if final.isMips32 then "mips" + else if final.isMips64 then "mips" # linux kernel does not distinguish mips32/mips64 + else if final.isPower then "powerpc" + else if final.isRiscV then "riscv" + else if final.isS390 then "s390" + else if final.isLoongArch64 then "loongarch" + else final.parsed.cpu.name; + + qemuArch = + if final.isAarch32 then "arm" + else if final.isS390 && !final.isS390x then null + else if final.isx86_64 then "x86_64" + else if final.isx86 then "i386" + else if final.isMips64 then "mips64${lib.optionalString final.isLittleEndian "el"}" + else final.uname.processor; + + # Name used by UEFI for architectures. + efiArch = + if final.isx86_32 then "ia32" + else if final.isx86_64 then "x64" + else if final.isAarch32 then "arm" + else if final.isAarch64 then "aa64" + else final.parsed.cpu.name; + + darwinArch = { + armv7a = "armv7"; + aarch64 = "arm64"; + }.${final.parsed.cpu.name} or final.parsed.cpu.name; + + darwinPlatform = + if final.isMacOS then "macos" + else if final.isiOS then "ios" + else null; + # The canonical name for this attribute is darwinSdkVersion, but some + # platforms define the old name "sdkVer". + darwinSdkVersion = final.sdkVer or (if final.isAarch64 then "11.0" else "10.12"); + darwinMinVersion = final.darwinSdkVersion; + darwinMinVersionVariable = + if final.isMacOS then "MACOSX_DEPLOYMENT_TARGET" + else if final.isiOS then "IPHONEOS_DEPLOYMENT_TARGET" + else null; + } // ( + let + selectEmulator = pkgs: + let + qemu-user = pkgs.qemu.override { + smartcardSupport = false; + spiceSupport = false; + openGLSupport = false; + virglSupport = false; + vncSupport = false; + gtkSupport = false; + sdlSupport = false; + pulseSupport = false; + smbdSupport = false; + seccompSupport = false; + enableDocs = false; + hostCpuTargets = [ "${final.qemuArch}-linux-user" ]; + }; + wine = (pkgs.winePackagesFor "wine${toString final.parsed.cpu.bits}").minimal; + in + if final.parsed.kernel.name == pkgs.stdenv.hostPlatform.parsed.kernel.name && + pkgs.stdenv.hostPlatform.canExecute final + then "${pkgs.runtimeShell} -c '\"$@\"' --" + else if final.isWindows + then "${wine}/bin/wine${lib.optionalString (final.parsed.cpu.bits == 64) "64"}" + else if final.isLinux && pkgs.stdenv.hostPlatform.isLinux && final.qemuArch != null + then "${qemu-user}/bin/qemu-${final.qemuArch}" + else if final.isWasi + then "${pkgs.wasmtime}/bin/wasmtime" + else if final.isMmix + then "${pkgs.mmixware}/bin/mmix" + else null; + in { + emulatorAvailable = pkgs: (selectEmulator pkgs) != null; + + emulator = pkgs: + if (final.emulatorAvailable pkgs) + then selectEmulator pkgs + else throw "Don't know how to run ${final.config} executables."; + + }) // mapAttrs (n: v: v final.parsed) inspect.predicates + // mapAttrs (n: v: v final.gcc.arch or "default") architectures.predicates + // args; + in assert final.useAndroidPrebuilt -> final.isAndroid; + assert lib.foldl + (pass: { assertion, message }: + if assertion final + then pass + else throw message) + true + (final.parsed.abi.assertions or []); + final; +} diff --git a/systems/doubles.nix b/systems/doubles.nix new file mode 100644 index 000000000..66bf3d872 --- /dev/null +++ b/systems/doubles.nix @@ -0,0 +1,119 @@ +{ lib }: +let + inherit (lib) lists; + inherit (lib.systems) parse; + inherit (lib.systems.inspect) predicates; + inherit (lib.attrsets) matchAttrs; + + all = [ + # Cygwin + "i686-cygwin" "x86_64-cygwin" + + # Darwin + "x86_64-darwin" "i686-darwin" "aarch64-darwin" "armv7a-darwin" + + # FreeBSD + "i686-freebsd13" "x86_64-freebsd13" + + # Genode + "aarch64-genode" "i686-genode" "x86_64-genode" + + # illumos + "x86_64-solaris" + + # JS + "javascript-ghcjs" + + # Linux + "aarch64-linux" "armv5tel-linux" "armv6l-linux" "armv7a-linux" + "armv7l-linux" "i686-linux" "loongarch64-linux" "m68k-linux" "microblaze-linux" + "microblazeel-linux" "mips-linux" "mips64-linux" "mips64el-linux" + "mipsel-linux" "powerpc64-linux" "powerpc64le-linux" "riscv32-linux" + "riscv64-linux" "s390-linux" "s390x-linux" "x86_64-linux" + + # MMIXware + "mmix-mmixware" + + # NetBSD + "aarch64-netbsd" "armv6l-netbsd" "armv7a-netbsd" "armv7l-netbsd" + "i686-netbsd" "m68k-netbsd" "mipsel-netbsd" "powerpc-netbsd" + "riscv32-netbsd" "riscv64-netbsd" "x86_64-netbsd" + + # none + "aarch64_be-none" "aarch64-none" "arm-none" "armv6l-none" "avr-none" "i686-none" + "microblaze-none" "microblazeel-none" "msp430-none" "or1k-none" "m68k-none" + "powerpc-none" "powerpcle-none" "riscv32-none" "riscv64-none" "rx-none" + "s390-none" "s390x-none" "vc4-none" "x86_64-none" + + # OpenBSD + "i686-openbsd" "x86_64-openbsd" + + # Redox + "x86_64-redox" + + # WASI + "wasm64-wasi" "wasm32-wasi" + + # Windows + "x86_64-windows" "i686-windows" + ]; + + allParsed = map parse.mkSystemFromString all; + + filterDoubles = f: map parse.doubleFromSystem (lists.filter f allParsed); + +in { + inherit all; + + none = []; + + arm = filterDoubles predicates.isAarch32; + armv7 = filterDoubles predicates.isArmv7; + aarch64 = filterDoubles predicates.isAarch64; + x86 = filterDoubles predicates.isx86; + i686 = filterDoubles predicates.isi686; + x86_64 = filterDoubles predicates.isx86_64; + microblaze = filterDoubles predicates.isMicroBlaze; + mips = filterDoubles predicates.isMips; + mmix = filterDoubles predicates.isMmix; + power = filterDoubles predicates.isPower; + riscv = filterDoubles predicates.isRiscV; + riscv32 = filterDoubles predicates.isRiscV32; + riscv64 = filterDoubles predicates.isRiscV64; + rx = filterDoubles predicates.isRx; + vc4 = filterDoubles predicates.isVc4; + or1k = filterDoubles predicates.isOr1k; + m68k = filterDoubles predicates.isM68k; + s390 = filterDoubles predicates.isS390; + s390x = filterDoubles predicates.isS390x; + loongarch64 = filterDoubles predicates.isLoongArch64; + js = filterDoubles predicates.isJavaScript; + + bigEndian = filterDoubles predicates.isBigEndian; + littleEndian = filterDoubles predicates.isLittleEndian; + + cygwin = filterDoubles predicates.isCygwin; + darwin = filterDoubles predicates.isDarwin; + freebsd = filterDoubles predicates.isFreeBSD; + # Should be better, but MinGW is unclear. + gnu = filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnu; }) + ++ filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnueabi; }) + ++ filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnueabihf; }) + ++ filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnuabin32; }) + ++ filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnuabi64; }) + ++ filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnuabielfv1; }) + ++ filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnuabielfv2; }); + illumos = filterDoubles predicates.isSunOS; + linux = filterDoubles predicates.isLinux; + netbsd = filterDoubles predicates.isNetBSD; + openbsd = filterDoubles predicates.isOpenBSD; + unix = filterDoubles predicates.isUnix; + wasi = filterDoubles predicates.isWasi; + redox = filterDoubles predicates.isRedox; + windows = filterDoubles predicates.isWindows; + genode = filterDoubles predicates.isGenode; + + embedded = filterDoubles predicates.isNone; + + mesaPlatforms = ["i686-linux" "x86_64-linux" "x86_64-darwin" "armv5tel-linux" "armv6l-linux" "armv7l-linux" "armv7a-linux" "aarch64-linux" "powerpc64-linux" "powerpc64le-linux" "aarch64-darwin" "riscv64-linux"]; +} diff --git a/systems/examples.nix b/systems/examples.nix new file mode 100644 index 000000000..45b95c150 --- /dev/null +++ b/systems/examples.nix @@ -0,0 +1,335 @@ +# These can be passed to nixpkgs as either the `localSystem` or +# `crossSystem`. They are put here for user convenience, but also used by cross +# tests and linux cross stdenv building, so handle with care! +{ lib }: +let + platforms = import ./platforms.nix { inherit lib; }; + + riscv = bits: { + config = "riscv${bits}-unknown-linux-gnu"; + }; +in + +rec { + # + # Linux + # + powernv = { + config = "powerpc64le-unknown-linux-gnu"; + }; + musl-power = { + config = "powerpc64le-unknown-linux-musl"; + }; + + ppc64 = { + config = "powerpc64-unknown-linux-gnuabielfv2"; + }; + ppc64-musl = { + config = "powerpc64-unknown-linux-musl"; + gcc = { abi = "elfv2"; }; + }; + + sheevaplug = { + config = "armv5tel-unknown-linux-gnueabi"; + } // platforms.sheevaplug; + + raspberryPi = { + config = "armv6l-unknown-linux-gnueabihf"; + } // platforms.raspberrypi; + + remarkable1 = { + config = "armv7l-unknown-linux-gnueabihf"; + } // platforms.zero-gravitas; + + remarkable2 = { + config = "armv7l-unknown-linux-gnueabihf"; + } // platforms.zero-sugar; + + armv7l-hf-multiplatform = { + config = "armv7l-unknown-linux-gnueabihf"; + }; + + aarch64-multiplatform = { + config = "aarch64-unknown-linux-gnu"; + }; + + armv7a-android-prebuilt = { + config = "armv7a-unknown-linux-androideabi"; + rustc.config = "armv7-linux-androideabi"; + sdkVer = "28"; + ndkVer = "24"; + useAndroidPrebuilt = true; + } // platforms.armv7a-android; + + aarch64-android-prebuilt = { + config = "aarch64-unknown-linux-android"; + rustc.config = "aarch64-linux-android"; + sdkVer = "28"; + ndkVer = "24"; + useAndroidPrebuilt = true; + }; + + aarch64-android = { + config = "aarch64-unknown-linux-android"; + sdkVer = "30"; + ndkVer = "24"; + libc = "bionic"; + useAndroidPrebuilt = false; + useLLVM = true; + }; + + pogoplug4 = { + config = "armv5tel-unknown-linux-gnueabi"; + } // platforms.pogoplug4; + + ben-nanonote = { + config = "mipsel-unknown-linux-uclibc"; + } // platforms.ben_nanonote; + + fuloongminipc = { + config = "mipsel-unknown-linux-gnu"; + } // platforms.fuloong2f_n32; + + # can execute on 32bit chip + mips-linux-gnu = { config = "mips-unknown-linux-gnu"; } // platforms.gcc_mips32r2_o32; + mipsel-linux-gnu = { config = "mipsel-unknown-linux-gnu"; } // platforms.gcc_mips32r2_o32; + + # require 64bit chip (for more registers, 64-bit floating point, 64-bit "long long") but use 32bit pointers + mips64-linux-gnuabin32 = { config = "mips64-unknown-linux-gnuabin32"; } // platforms.gcc_mips64r2_n32; + mips64el-linux-gnuabin32 = { config = "mips64el-unknown-linux-gnuabin32"; } // platforms.gcc_mips64r2_n32; + + # 64bit pointers + mips64-linux-gnuabi64 = { config = "mips64-unknown-linux-gnuabi64"; } // platforms.gcc_mips64r2_64; + mips64el-linux-gnuabi64 = { config = "mips64el-unknown-linux-gnuabi64"; } // platforms.gcc_mips64r2_64; + + muslpi = raspberryPi // { + config = "armv6l-unknown-linux-musleabihf"; + }; + + aarch64-multiplatform-musl = { + config = "aarch64-unknown-linux-musl"; + }; + + gnu64 = { config = "x86_64-unknown-linux-gnu"; }; + gnu32 = { config = "i686-unknown-linux-gnu"; }; + + musl64 = { config = "x86_64-unknown-linux-musl"; }; + musl32 = { config = "i686-unknown-linux-musl"; }; + + riscv64 = riscv "64"; + riscv32 = riscv "32"; + + riscv64-embedded = { + config = "riscv64-none-elf"; + libc = "newlib"; + }; + + riscv32-embedded = { + config = "riscv32-none-elf"; + libc = "newlib"; + }; + + loongarch64-linux = { + config = "loongarch64-unknown-linux-gnu"; + }; + + mmix = { + config = "mmix-unknown-mmixware"; + libc = "newlib"; + }; + + rx-embedded = { + config = "rx-none-elf"; + libc = "newlib"; + }; + + msp430 = { + config = "msp430-elf"; + libc = "newlib"; + }; + + avr = { + config = "avr"; + }; + + vc4 = { + config = "vc4-elf"; + libc = "newlib"; + }; + + or1k = { + config = "or1k-elf"; + libc = "newlib"; + }; + + m68k = { + config = "m68k-unknown-linux-gnu"; + }; + + s390 = { + config = "s390-unknown-linux-gnu"; + }; + + s390x = { + config = "s390x-unknown-linux-gnu"; + }; + + arm-embedded = { + config = "arm-none-eabi"; + libc = "newlib"; + }; + armhf-embedded = { + config = "arm-none-eabihf"; + libc = "newlib"; + # GCC8+ does not build without this + # (https://www.mail-archive.com/gcc-bugs@gcc.gnu.org/msg552339.html): + gcc = { + arch = "armv5t"; + fpu = "vfp"; + }; + }; + + aarch64-embedded = { + config = "aarch64-none-elf"; + libc = "newlib"; + }; + + aarch64be-embedded = { + config = "aarch64_be-none-elf"; + libc = "newlib"; + }; + + ppc-embedded = { + config = "powerpc-none-eabi"; + libc = "newlib"; + }; + + ppcle-embedded = { + config = "powerpcle-none-eabi"; + libc = "newlib"; + }; + + i686-embedded = { + config = "i686-elf"; + libc = "newlib"; + }; + + x86_64-embedded = { + config = "x86_64-elf"; + libc = "newlib"; + }; + + # + # Redox + # + + x86_64-unknown-redox = { + config = "x86_64-unknown-redox"; + libc = "relibc"; + }; + + # + # Darwin + # + + iphone64 = { + config = "aarch64-apple-ios"; + # config = "aarch64-apple-darwin14"; + sdkVer = "14.3"; + xcodeVer = "12.3"; + xcodePlatform = "iPhoneOS"; + useiOSPrebuilt = true; + }; + + iphone32 = { + config = "armv7a-apple-ios"; + # config = "arm-apple-darwin10"; + sdkVer = "14.3"; + xcodeVer = "12.3"; + xcodePlatform = "iPhoneOS"; + useiOSPrebuilt = true; + }; + + iphone64-simulator = { + config = "x86_64-apple-ios"; + # config = "x86_64-apple-darwin14"; + sdkVer = "14.3"; + xcodeVer = "12.3"; + xcodePlatform = "iPhoneSimulator"; + darwinPlatform = "ios-simulator"; + useiOSPrebuilt = true; + }; + + iphone32-simulator = { + config = "i686-apple-ios"; + # config = "i386-apple-darwin11"; + sdkVer = "14.3"; + xcodeVer = "12.3"; + xcodePlatform = "iPhoneSimulator"; + darwinPlatform = "ios-simulator"; + useiOSPrebuilt = true; + }; + + aarch64-darwin = { + config = "aarch64-apple-darwin"; + xcodePlatform = "MacOSX"; + platform = {}; + }; + + x86_64-darwin = { + config = "x86_64-apple-darwin"; + xcodePlatform = "MacOSX"; + platform = {}; + }; + + # + # Windows + # + + # 32 bit mingw-w64 + mingw32 = { + config = "i686-w64-mingw32"; + libc = "msvcrt"; # This distinguishes the mingw (non posix) toolchain + }; + + # 64 bit mingw-w64 + mingwW64 = { + # That's the triplet they use in the mingw-w64 docs. + config = "x86_64-w64-mingw32"; + libc = "msvcrt"; # This distinguishes the mingw (non posix) toolchain + }; + + # BSDs + + x86_64-freebsd = { + config = "x86_64-unknown-freebsd13"; + useLLVM = true; + }; + + x86_64-netbsd = { + config = "x86_64-unknown-netbsd"; + }; + + # this is broken and never worked fully + x86_64-netbsd-llvm = { + config = "x86_64-unknown-netbsd"; + useLLVM = true; + }; + + # + # WASM + # + + wasi32 = { + config = "wasm32-unknown-wasi"; + useLLVM = true; + }; + + # Ghcjs + ghcjs = { + # This triple is special to GHC/Cabal/GHCJS and not recognized by autotools + # See: https://gitlab.haskell.org/ghc/ghc/-/commit/6636b670233522f01d002c9b97827d00289dbf5c + # https://github.com/ghcjs/ghcjs/issues/53 + config = "javascript-unknown-ghcjs"; + }; +} diff --git a/systems/flake-systems.nix b/systems/flake-systems.nix new file mode 100644 index 000000000..b1988c6a4 --- /dev/null +++ b/systems/flake-systems.nix @@ -0,0 +1,29 @@ +# See [RFC 46] for mandated platform support and ../../pkgs/stdenv for +# implemented platform support. This list is mainly descriptive, i.e. all +# system doubles for platforms where nixpkgs can do native compilation +# reasonably well are included. +# +# [RFC 46]: https://github.com/NixOS/rfcs/blob/master/rfcs/0046-platform-support-tiers.md +{ }: + +[ + # Tier 1 + "x86_64-linux" + # Tier 2 + "aarch64-linux" + "x86_64-darwin" + # Tier 3 + "armv6l-linux" + "armv7l-linux" + "i686-linux" + "mipsel-linux" + + # Other platforms with sufficient support in stdenv which is not formally + # mandated by their platform tier. + "aarch64-darwin" + "armv5tel-linux" + "powerpc64le-linux" + "riscv64-linux" + + # "x86_64-freebsd" is excluded because it is mostly broken +] diff --git a/systems/inspect.nix b/systems/inspect.nix new file mode 100644 index 000000000..89e9f4231 --- /dev/null +++ b/systems/inspect.nix @@ -0,0 +1,117 @@ +{ lib }: +with import ./parse.nix { inherit lib; }; +with lib.attrsets; +with lib.lists; + +let abis_ = abis; in +let abis = lib.mapAttrs (_: abi: builtins.removeAttrs abi [ "assertions" ]) abis_; in + +rec { + # these patterns are to be matched against {host,build,target}Platform.parsed + patterns = rec { + # The patterns below are lists in sum-of-products form. + # + # Each attribute is list of product conditions; non-list values are treated + # as a singleton list. If *any* product condition in the list matches then + # the predicate matches. Each product condition is tested by + # `lib.attrsets.matchAttrs`, which requires a match on *all* attributes of + # the product. + + isi686 = { cpu = cpuTypes.i686; }; + isx86_32 = { cpu = { family = "x86"; bits = 32; }; }; + isx86_64 = { cpu = { family = "x86"; bits = 64; }; }; + isPower = { cpu = { family = "power"; }; }; + isPower64 = { cpu = { family = "power"; bits = 64; }; }; + # This ABI is the default in NixOS PowerPC64 BE, but not on mainline GCC, + # so it sometimes causes issues in certain packages that makes the wrong + # assumption on the used ABI. + isAbiElfv2 = [ + { abi = { abi = "elfv2"; }; } + { abi = { name = "musl"; }; cpu = { family = "power"; bits = 64; }; } + ]; + isx86 = { cpu = { family = "x86"; }; }; + isAarch32 = { cpu = { family = "arm"; bits = 32; }; }; + isArmv7 = map ({ arch, ... }: { cpu = { inherit arch; }; }) + (lib.filter (cpu: lib.hasPrefix "armv7" cpu.arch or "") + (lib.attrValues cpuTypes)); + isAarch64 = { cpu = { family = "arm"; bits = 64; }; }; + isAarch = { cpu = { family = "arm"; }; }; + isMicroBlaze = { cpu = { family = "microblaze"; }; }; + isMips = { cpu = { family = "mips"; }; }; + isMips32 = { cpu = { family = "mips"; bits = 32; }; }; + isMips64 = { cpu = { family = "mips"; bits = 64; }; }; + isMips64n32 = { cpu = { family = "mips"; bits = 64; }; abi = { abi = "n32"; }; }; + isMips64n64 = { cpu = { family = "mips"; bits = 64; }; abi = { abi = "64"; }; }; + isMmix = { cpu = { family = "mmix"; }; }; + isRiscV = { cpu = { family = "riscv"; }; }; + isRiscV32 = { cpu = { family = "riscv"; bits = 32; }; }; + isRiscV64 = { cpu = { family = "riscv"; bits = 64; }; }; + isRx = { cpu = { family = "rx"; }; }; + isSparc = { cpu = { family = "sparc"; }; }; + isWasm = { cpu = { family = "wasm"; }; }; + isMsp430 = { cpu = { family = "msp430"; }; }; + isVc4 = { cpu = { family = "vc4"; }; }; + isAvr = { cpu = { family = "avr"; }; }; + isAlpha = { cpu = { family = "alpha"; }; }; + isOr1k = { cpu = { family = "or1k"; }; }; + isM68k = { cpu = { family = "m68k"; }; }; + isS390 = { cpu = { family = "s390"; }; }; + isS390x = { cpu = { family = "s390"; bits = 64; }; }; + isLoongArch64 = { cpu = { family = "loongarch"; bits = 64; }; }; + isJavaScript = { cpu = cpuTypes.javascript; }; + + is32bit = { cpu = { bits = 32; }; }; + is64bit = { cpu = { bits = 64; }; }; + isILP32 = map (a: { abi = { abi = a; }; }) [ "n32" "ilp32" "x32" ]; + isBigEndian = { cpu = { significantByte = significantBytes.bigEndian; }; }; + isLittleEndian = { cpu = { significantByte = significantBytes.littleEndian; }; }; + + isBSD = { kernel = { families = { inherit (kernelFamilies) bsd; }; }; }; + isDarwin = { kernel = { families = { inherit (kernelFamilies) darwin; }; }; }; + isUnix = [ isBSD isDarwin isLinux isSunOS isCygwin isRedox ]; + + isMacOS = { kernel = kernels.macos; }; + isiOS = { kernel = kernels.ios; }; + isLinux = { kernel = kernels.linux; }; + isSunOS = { kernel = kernels.solaris; }; + isFreeBSD = { kernel = { name = "freebsd"; }; }; + isNetBSD = { kernel = kernels.netbsd; }; + isOpenBSD = { kernel = kernels.openbsd; }; + isWindows = { kernel = kernels.windows; }; + isCygwin = { kernel = kernels.windows; abi = abis.cygnus; }; + isMinGW = { kernel = kernels.windows; abi = abis.gnu; }; + isWasi = { kernel = kernels.wasi; }; + isRedox = { kernel = kernels.redox; }; + isGhcjs = { kernel = kernels.ghcjs; }; + isGenode = { kernel = kernels.genode; }; + isNone = { kernel = kernels.none; }; + + isAndroid = [ { abi = abis.android; } { abi = abis.androideabi; } ]; + isGnu = with abis; map (a: { abi = a; }) [ gnuabi64 gnu gnueabi gnueabihf gnuabielfv1 gnuabielfv2 ]; + isMusl = with abis; map (a: { abi = a; }) [ musl musleabi musleabihf muslabin32 muslabi64 ]; + isUClibc = with abis; map (a: { abi = a; }) [ uclibc uclibceabi uclibceabihf ]; + + isEfi = [ + { cpu = { family = "arm"; version = "6"; }; } + { cpu = { family = "arm"; version = "7"; }; } + { cpu = { family = "arm"; version = "8"; }; } + { cpu = { family = "riscv"; }; } + { cpu = { family = "x86"; }; } + ]; + }; + + matchAnyAttrs = patterns: + if builtins.isList patterns then attrs: any (pattern: matchAttrs pattern attrs) patterns + else matchAttrs patterns; + + predicates = mapAttrs (_: matchAnyAttrs) patterns; + + # these patterns are to be matched against the entire + # {host,build,target}Platform structure; they include a `parsed={}` marker so + # that `lib.meta.availableOn` can distinguish them from the patterns which + # apply only to the `parsed` field. + + platformPatterns = mapAttrs (_: p: { parsed = {}; } // p) { + isStatic = { isStatic = true; }; + }; +} diff --git a/systems/parse.nix b/systems/parse.nix new file mode 100644 index 000000000..6eb4f27cc --- /dev/null +++ b/systems/parse.nix @@ -0,0 +1,503 @@ +# Define the list of system with their properties. +# +# See https://clang.llvm.org/docs/CrossCompilation.html and +# http://llvm.org/docs/doxygen/html/Triple_8cpp_source.html especially +# Triple::normalize. Parsing should essentially act as a more conservative +# version of that last function. +# +# Most of the types below come in "open" and "closed" pairs. The open ones +# specify what information we need to know about systems in general, and the +# closed ones are sub-types representing the whitelist of systems we support in +# practice. +# +# Code in the remainder of nixpkgs shouldn't rely on the closed ones in +# e.g. exhaustive cases. Its more a sanity check to make sure nobody defines +# systems that overlap with existing ones and won't notice something amiss. +# +{ lib }: +with lib.lists; +with lib.types; +with lib.attrsets; +with lib.strings; +with (import ./inspect.nix { inherit lib; }).predicates; + +let + inherit (lib.options) mergeOneOption; + + setTypes = type: + mapAttrs (name: value: + assert type.check value; + setType type.name ({ inherit name; } // value)); + +in + +rec { + + ################################################################################ + + types.openSignificantByte = mkOptionType { + name = "significant-byte"; + description = "Endianness"; + merge = mergeOneOption; + }; + + types.significantByte = enum (attrValues significantBytes); + + significantBytes = setTypes types.openSignificantByte { + bigEndian = {}; + littleEndian = {}; + }; + + ################################################################################ + + # Reasonable power of 2 + types.bitWidth = enum [ 8 16 32 64 128 ]; + + ################################################################################ + + types.openCpuType = mkOptionType { + name = "cpu-type"; + description = "instruction set architecture name and information"; + merge = mergeOneOption; + check = x: types.bitWidth.check x.bits + && (if 8 < x.bits + then types.significantByte.check x.significantByte + else !(x ? significantByte)); + }; + + types.cpuType = enum (attrValues cpuTypes); + + cpuTypes = with significantBytes; setTypes types.openCpuType { + arm = { bits = 32; significantByte = littleEndian; family = "arm"; }; + armv5tel = { bits = 32; significantByte = littleEndian; family = "arm"; version = "5"; arch = "armv5t"; }; + armv6m = { bits = 32; significantByte = littleEndian; family = "arm"; version = "6"; arch = "armv6-m"; }; + armv6l = { bits = 32; significantByte = littleEndian; family = "arm"; version = "6"; arch = "armv6"; }; + armv7a = { bits = 32; significantByte = littleEndian; family = "arm"; version = "7"; arch = "armv7-a"; }; + armv7r = { bits = 32; significantByte = littleEndian; family = "arm"; version = "7"; arch = "armv7-r"; }; + armv7m = { bits = 32; significantByte = littleEndian; family = "arm"; version = "7"; arch = "armv7-m"; }; + armv7l = { bits = 32; significantByte = littleEndian; family = "arm"; version = "7"; arch = "armv7"; }; + armv8a = { bits = 32; significantByte = littleEndian; family = "arm"; version = "8"; arch = "armv8-a"; }; + armv8r = { bits = 32; significantByte = littleEndian; family = "arm"; version = "8"; arch = "armv8-a"; }; + armv8m = { bits = 32; significantByte = littleEndian; family = "arm"; version = "8"; arch = "armv8-m"; }; + aarch64 = { bits = 64; significantByte = littleEndian; family = "arm"; version = "8"; arch = "armv8-a"; }; + aarch64_be = { bits = 64; significantByte = bigEndian; family = "arm"; version = "8"; arch = "armv8-a"; }; + + i386 = { bits = 32; significantByte = littleEndian; family = "x86"; arch = "i386"; }; + i486 = { bits = 32; significantByte = littleEndian; family = "x86"; arch = "i486"; }; + i586 = { bits = 32; significantByte = littleEndian; family = "x86"; arch = "i586"; }; + i686 = { bits = 32; significantByte = littleEndian; family = "x86"; arch = "i686"; }; + x86_64 = { bits = 64; significantByte = littleEndian; family = "x86"; arch = "x86-64"; }; + + microblaze = { bits = 32; significantByte = bigEndian; family = "microblaze"; }; + microblazeel = { bits = 32; significantByte = littleEndian; family = "microblaze"; }; + + mips = { bits = 32; significantByte = bigEndian; family = "mips"; }; + mipsel = { bits = 32; significantByte = littleEndian; family = "mips"; }; + mips64 = { bits = 64; significantByte = bigEndian; family = "mips"; }; + mips64el = { bits = 64; significantByte = littleEndian; family = "mips"; }; + + mmix = { bits = 64; significantByte = bigEndian; family = "mmix"; }; + + m68k = { bits = 32; significantByte = bigEndian; family = "m68k"; }; + + powerpc = { bits = 32; significantByte = bigEndian; family = "power"; }; + powerpc64 = { bits = 64; significantByte = bigEndian; family = "power"; }; + powerpc64le = { bits = 64; significantByte = littleEndian; family = "power"; }; + powerpcle = { bits = 32; significantByte = littleEndian; family = "power"; }; + + riscv32 = { bits = 32; significantByte = littleEndian; family = "riscv"; }; + riscv64 = { bits = 64; significantByte = littleEndian; family = "riscv"; }; + + s390 = { bits = 32; significantByte = bigEndian; family = "s390"; }; + s390x = { bits = 64; significantByte = bigEndian; family = "s390"; }; + + sparc = { bits = 32; significantByte = bigEndian; family = "sparc"; }; + sparc64 = { bits = 64; significantByte = bigEndian; family = "sparc"; }; + + wasm32 = { bits = 32; significantByte = littleEndian; family = "wasm"; }; + wasm64 = { bits = 64; significantByte = littleEndian; family = "wasm"; }; + + alpha = { bits = 64; significantByte = littleEndian; family = "alpha"; }; + + rx = { bits = 32; significantByte = littleEndian; family = "rx"; }; + msp430 = { bits = 16; significantByte = littleEndian; family = "msp430"; }; + avr = { bits = 8; family = "avr"; }; + + vc4 = { bits = 32; significantByte = littleEndian; family = "vc4"; }; + + or1k = { bits = 32; significantByte = bigEndian; family = "or1k"; }; + + loongarch64 = { bits = 64; significantByte = littleEndian; family = "loongarch"; }; + + javascript = { bits = 32; significantByte = littleEndian; family = "javascript"; }; + }; + + # GNU build systems assume that older NetBSD architectures are using a.out. + gnuNetBSDDefaultExecFormat = cpu: + if (cpu.family == "arm" && cpu.bits == 32) || + (cpu.family == "sparc" && cpu.bits == 32) || + (cpu.family == "m68k" && cpu.bits == 32) || + (cpu.family == "x86" && cpu.bits == 32) + then execFormats.aout + else execFormats.elf; + + # Determine when two CPUs are compatible with each other. That is, + # can code built for system B run on system A? For that to happen, + # the programs that system B accepts must be a subset of the + # programs that system A accepts. + # + # We have the following properties of the compatibility relation, + # which must be preserved when adding compatibility information for + # additional CPUs. + # - (reflexivity) + # Every CPU is compatible with itself. + # - (transitivity) + # If A is compatible with B and B is compatible with C then A is compatible with C. + # + # Note: Since 22.11 the archs of a mode switching CPU are no longer considered + # pairwise compatible. Mode switching implies that binaries built for A + # and B respectively can't be executed at the same time. + isCompatible = a: b: with cpuTypes; lib.any lib.id [ + # x86 + (b == i386 && isCompatible a i486) + (b == i486 && isCompatible a i586) + (b == i586 && isCompatible a i686) + + # XXX: Not true in some cases. Like in WSL mode. + (b == i686 && isCompatible a x86_64) + + # ARMv4 + (b == arm && isCompatible a armv5tel) + + # ARMv5 + (b == armv5tel && isCompatible a armv6l) + + # ARMv6 + (b == armv6l && isCompatible a armv6m) + (b == armv6m && isCompatible a armv7l) + + # ARMv7 + (b == armv7l && isCompatible a armv7a) + (b == armv7l && isCompatible a armv7r) + (b == armv7l && isCompatible a armv7m) + + # ARMv8 + (b == aarch64 && a == armv8a) + (b == armv8a && isCompatible a aarch64) + (b == armv8r && isCompatible a armv8a) + (b == armv8m && isCompatible a armv8a) + + # PowerPC + (b == powerpc && isCompatible a powerpc64) + (b == powerpcle && isCompatible a powerpc64le) + + # MIPS + (b == mips && isCompatible a mips64) + (b == mipsel && isCompatible a mips64el) + + # RISCV + (b == riscv32 && isCompatible a riscv64) + + # SPARC + (b == sparc && isCompatible a sparc64) + + # WASM + (b == wasm32 && isCompatible a wasm64) + + # identity + (b == a) + ]; + + ################################################################################ + + types.openVendor = mkOptionType { + name = "vendor"; + description = "vendor for the platform"; + merge = mergeOneOption; + }; + + types.vendor = enum (attrValues vendors); + + vendors = setTypes types.openVendor { + apple = {}; + pc = {}; + # Actually matters, unlocking some MinGW-w64-specific options in GCC. See + # bottom of https://sourceforge.net/p/mingw-w64/wiki2/Unicode%20apps/ + w64 = {}; + + none = {}; + unknown = {}; + }; + + ################################################################################ + + types.openExecFormat = mkOptionType { + name = "exec-format"; + description = "executable container used by the kernel"; + merge = mergeOneOption; + }; + + types.execFormat = enum (attrValues execFormats); + + execFormats = setTypes types.openExecFormat { + aout = {}; # a.out + elf = {}; + macho = {}; + pe = {}; + wasm = {}; + + unknown = {}; + }; + + ################################################################################ + + types.openKernelFamily = mkOptionType { + name = "exec-format"; + description = "executable container used by the kernel"; + merge = mergeOneOption; + }; + + types.kernelFamily = enum (attrValues kernelFamilies); + + kernelFamilies = setTypes types.openKernelFamily { + bsd = {}; + darwin = {}; + }; + + ################################################################################ + + types.openKernel = mkOptionType { + name = "kernel"; + description = "kernel name and information"; + merge = mergeOneOption; + check = x: types.execFormat.check x.execFormat + && all types.kernelFamily.check (attrValues x.families); + }; + + types.kernel = enum (attrValues kernels); + + kernels = with execFormats; with kernelFamilies; setTypes types.openKernel { + # TODO(@Ericson2314): Don't want to mass-rebuild yet to keeping 'darwin' as + # the normalized name for macOS. + macos = { execFormat = macho; families = { inherit darwin; }; name = "darwin"; }; + ios = { execFormat = macho; families = { inherit darwin; }; }; + # A tricky thing about FreeBSD is that there is no stable ABI across + # versions. That means that putting in the version as part of the + # config string is paramount. + freebsd12 = { execFormat = elf; families = { inherit bsd; }; name = "freebsd"; version = 12; }; + freebsd13 = { execFormat = elf; families = { inherit bsd; }; name = "freebsd"; version = 13; }; + linux = { execFormat = elf; families = { }; }; + netbsd = { execFormat = elf; families = { inherit bsd; }; }; + none = { execFormat = unknown; families = { }; }; + openbsd = { execFormat = elf; families = { inherit bsd; }; }; + solaris = { execFormat = elf; families = { }; }; + wasi = { execFormat = wasm; families = { }; }; + redox = { execFormat = elf; families = { }; }; + windows = { execFormat = pe; families = { }; }; + ghcjs = { execFormat = unknown; families = { }; }; + genode = { execFormat = elf; families = { }; }; + mmixware = { execFormat = unknown; families = { }; }; + } // { # aliases + # 'darwin' is the kernel for all of them. We choose macOS by default. + darwin = kernels.macos; + watchos = kernels.ios; + tvos = kernels.ios; + win32 = kernels.windows; + }; + + ################################################################################ + + types.openAbi = mkOptionType { + name = "abi"; + description = "binary interface for compiled code and syscalls"; + merge = mergeOneOption; + }; + + types.abi = enum (attrValues abis); + + abis = setTypes types.openAbi { + cygnus = {}; + msvc = {}; + + # Note: eabi is specific to ARM and PowerPC. + # On PowerPC, this corresponds to PPCEABI. + # On ARM, this corresponds to ARMEABI. + eabi = { float = "soft"; }; + eabihf = { float = "hard"; }; + + # Other architectures should use ELF in embedded situations. + elf = {}; + + androideabi = {}; + android = { + assertions = [ + { assertion = platform: !platform.isAarch32; + message = '' + The "android" ABI is not for 32-bit ARM. Use "androideabi" instead. + ''; + } + ]; + }; + + gnueabi = { float = "soft"; }; + gnueabihf = { float = "hard"; }; + gnu = { + assertions = [ + { assertion = platform: !platform.isAarch32; + message = '' + The "gnu" ABI is ambiguous on 32-bit ARM. Use "gnueabi" or "gnueabihf" instead. + ''; + } + { assertion = platform: with platform; !(isPower64 && isBigEndian); + message = '' + The "gnu" ABI is ambiguous on big-endian 64-bit PowerPC. Use "gnuabielfv2" or "gnuabielfv1" instead. + ''; + } + ]; + }; + gnuabi64 = { abi = "64"; }; + muslabi64 = { abi = "64"; }; + + # NOTE: abi=n32 requires a 64-bit MIPS chip! That is not a typo. + # It is basically the 64-bit abi with 32-bit pointers. Details: + # https://www.linux-mips.org/pub/linux/mips/doc/ABI/MIPS-N32-ABI-Handbook.pdf + gnuabin32 = { abi = "n32"; }; + muslabin32 = { abi = "n32"; }; + + gnuabielfv2 = { abi = "elfv2"; }; + gnuabielfv1 = { abi = "elfv1"; }; + + musleabi = { float = "soft"; }; + musleabihf = { float = "hard"; }; + musl = {}; + + uclibceabi = { float = "soft"; }; + uclibceabihf = { float = "hard"; }; + uclibc = {}; + + unknown = {}; + }; + + ################################################################################ + + types.parsedPlatform = mkOptionType { + name = "system"; + description = "fully parsed representation of llvm- or nix-style platform tuple"; + merge = mergeOneOption; + check = { cpu, vendor, kernel, abi }: + types.cpuType.check cpu + && types.vendor.check vendor + && types.kernel.check kernel + && types.abi.check abi; + }; + + isSystem = isType "system"; + + mkSystem = components: + assert types.parsedPlatform.check components; + setType "system" components; + + mkSkeletonFromList = l: { + "1" = if elemAt l 0 == "avr" + then { cpu = elemAt l 0; kernel = "none"; abi = "unknown"; } + else throw "Target specification with 1 components is ambiguous"; + "2" = # We only do 2-part hacks for things Nix already supports + if elemAt l 1 == "cygwin" + then { cpu = elemAt l 0; kernel = "windows"; abi = "cygnus"; } + # MSVC ought to be the default ABI so this case isn't needed. But then it + # becomes difficult to handle the gnu* variants for Aarch32 correctly for + # minGW. So it's easier to make gnu* the default for the MinGW, but + # hack-in MSVC for the non-MinGW case right here. + else if elemAt l 1 == "windows" + then { cpu = elemAt l 0; kernel = "windows"; abi = "msvc"; } + else if (elemAt l 1) == "elf" + then { cpu = elemAt l 0; vendor = "unknown"; kernel = "none"; abi = elemAt l 1; } + else { cpu = elemAt l 0; kernel = elemAt l 1; }; + "3" = + # cpu-kernel-environment + if elemAt l 1 == "linux" || + elem (elemAt l 2) ["eabi" "eabihf" "elf" "gnu"] + then { + cpu = elemAt l 0; + kernel = elemAt l 1; + abi = elemAt l 2; + vendor = "unknown"; + } + # cpu-vendor-os + else if elemAt l 1 == "apple" || + elem (elemAt l 2) [ "wasi" "redox" "mmixware" "ghcjs" "mingw32" ] || + hasPrefix "freebsd" (elemAt l 2) || + hasPrefix "netbsd" (elemAt l 2) || + hasPrefix "genode" (elemAt l 2) + then { + cpu = elemAt l 0; + vendor = elemAt l 1; + kernel = if elemAt l 2 == "mingw32" + then "windows" # autotools breaks on -gnu for window + else elemAt l 2; + } + else throw "Target specification with 3 components is ambiguous"; + "4" = { cpu = elemAt l 0; vendor = elemAt l 1; kernel = elemAt l 2; abi = elemAt l 3; }; + }.${toString (length l)} + or (throw "system string has invalid number of hyphen-separated components"); + + # This should revert the job done by config.guess from the gcc compiler. + mkSystemFromSkeleton = { cpu + , # Optional, but fallback too complex for here. + # Inferred below instead. + vendor ? assert false; null + , kernel + , # Also inferred below + abi ? assert false; null + } @ args: let + getCpu = name: cpuTypes.${name} or (throw "Unknown CPU type: ${name}"); + getVendor = name: vendors.${name} or (throw "Unknown vendor: ${name}"); + getKernel = name: kernels.${name} or (throw "Unknown kernel: ${name}"); + getAbi = name: abis.${name} or (throw "Unknown ABI: ${name}"); + + parsed = { + cpu = getCpu args.cpu; + vendor = + /**/ if args ? vendor then getVendor args.vendor + else if isDarwin parsed then vendors.apple + else if isWindows parsed then vendors.pc + else vendors.unknown; + kernel = if hasPrefix "darwin" args.kernel then getKernel "darwin" + else if hasPrefix "netbsd" args.kernel then getKernel "netbsd" + else getKernel args.kernel; + abi = + /**/ if args ? abi then getAbi args.abi + else if isLinux parsed || isWindows parsed then + if isAarch32 parsed then + if lib.versionAtLeast (parsed.cpu.version or "0") "6" + then abis.gnueabihf + else abis.gnueabi + # Default ppc64 BE to ELFv2 + else if isPower64 parsed && isBigEndian parsed then abis.gnuabielfv2 + else abis.gnu + else abis.unknown; + }; + + in mkSystem parsed; + + mkSystemFromString = s: mkSystemFromSkeleton (mkSkeletonFromList (lib.splitString "-" s)); + + kernelName = kernel: + kernel.name + toString (kernel.version or ""); + + doubleFromSystem = { cpu, kernel, abi, ... }: + /**/ if abi == abis.cygnus then "${cpu.name}-cygwin" + else if kernel.families ? darwin then "${cpu.name}-darwin" + else "${cpu.name}-${kernelName kernel}"; + + tripleFromSystem = { cpu, vendor, kernel, abi, ... } @ sys: assert isSystem sys; let + optExecFormat = + lib.optionalString (kernel.name == "netbsd" && + gnuNetBSDDefaultExecFormat cpu != kernel.execFormat) + kernel.execFormat.name; + optAbi = lib.optionalString (abi != abis.unknown) "-${abi.name}"; + in "${cpu.name}-${vendor.name}-${kernelName kernel}${optExecFormat}${optAbi}"; + + ################################################################################ + +} diff --git a/systems/platforms.nix b/systems/platforms.nix new file mode 100644 index 000000000..d574943e4 --- /dev/null +++ b/systems/platforms.nix @@ -0,0 +1,565 @@ +# Note: lib/systems/default.nix takes care of producing valid, +# fully-formed "platform" values (e.g. hostPlatform, buildPlatform, +# targetPlatform, etc) containing at least the minimal set of attrs +# required (see types.parsedPlatform in lib/systems/parse.nix). This +# file takes an already-valid platform and further elaborates it with +# optional fields; currently these are: linux-kernel, gcc, and rustc. + +{ lib }: +rec { + pc = { + linux-kernel = { + name = "pc"; + + baseConfig = "defconfig"; + # Build whatever possible as a module, if not stated in the extra config. + autoModules = true; + target = "bzImage"; + }; + }; + + pc_simplekernel = lib.recursiveUpdate pc { + linux-kernel.autoModules = false; + }; + + powernv = { + linux-kernel = { + name = "PowerNV"; + + baseConfig = "powernv_defconfig"; + target = "vmlinux"; + autoModules = true; + # avoid driver/FS trouble arising from unusual page size + extraConfig = '' + PPC_64K_PAGES n + PPC_4K_PAGES y + IPV6 y + + ATA_BMDMA y + ATA_SFF y + VIRTIO_MENU y + ''; + }; + }; + + ## + ## ARM + ## + + pogoplug4 = { + linux-kernel = { + name = "pogoplug4"; + + baseConfig = "multi_v5_defconfig"; + autoModules = false; + extraConfig = '' + # Ubi for the mtd + MTD_UBI y + UBIFS_FS y + UBIFS_FS_XATTR y + UBIFS_FS_ADVANCED_COMPR y + UBIFS_FS_LZO y + UBIFS_FS_ZLIB y + UBIFS_FS_DEBUG n + ''; + makeFlags = [ "LOADADDR=0x8000" ]; + target = "uImage"; + # TODO reenable once manual-config's config actually builds a .dtb and this is checked to be working + #DTB = true; + }; + gcc = { + arch = "armv5te"; + }; + }; + + sheevaplug = { + linux-kernel = { + name = "sheevaplug"; + + baseConfig = "multi_v5_defconfig"; + autoModules = false; + extraConfig = '' + BLK_DEV_RAM y + BLK_DEV_INITRD y + BLK_DEV_CRYPTOLOOP m + BLK_DEV_DM m + DM_CRYPT m + MD y + REISERFS_FS m + BTRFS_FS m + XFS_FS m + JFS_FS m + EXT4_FS m + USB_STORAGE_CYPRESS_ATACB m + + # mv cesa requires this sw fallback, for mv-sha1 + CRYPTO_SHA1 y + # Fast crypto + CRYPTO_TWOFISH y + CRYPTO_TWOFISH_COMMON y + CRYPTO_BLOWFISH y + CRYPTO_BLOWFISH_COMMON y + + IP_PNP y + IP_PNP_DHCP y + NFS_FS y + ROOT_NFS y + TUN m + NFS_V4 y + NFS_V4_1 y + NFS_FSCACHE y + NFSD m + NFSD_V2_ACL y + NFSD_V3 y + NFSD_V3_ACL y + NFSD_V4 y + NETFILTER y + IP_NF_IPTABLES y + IP_NF_FILTER y + IP_NF_MATCH_ADDRTYPE y + IP_NF_TARGET_LOG y + IP_NF_MANGLE y + IPV6 m + VLAN_8021Q m + + CIFS y + CIFS_XATTR y + CIFS_POSIX y + CIFS_FSCACHE y + CIFS_ACL y + + WATCHDOG y + WATCHDOG_CORE y + ORION_WATCHDOG m + + ZRAM m + NETCONSOLE m + + # Disable OABI to have seccomp_filter (required for systemd) + # https://github.com/raspberrypi/firmware/issues/651 + OABI_COMPAT n + + # Fail to build + DRM n + SCSI_ADVANSYS n + USB_ISP1362_HCD n + SND_SOC n + SND_ALI5451 n + FB_SAVAGE n + SCSI_NSP32 n + ATA_SFF n + SUNGEM n + IRDA n + ATM_HE n + SCSI_ACARD n + BLK_DEV_CMD640_ENHANCED n + + FUSE_FS m + + # systemd uses cgroups + CGROUPS y + + # Latencytop + LATENCYTOP y + + # Ubi for the mtd + MTD_UBI y + UBIFS_FS y + UBIFS_FS_XATTR y + UBIFS_FS_ADVANCED_COMPR y + UBIFS_FS_LZO y + UBIFS_FS_ZLIB y + UBIFS_FS_DEBUG n + + # Kdb, for kernel troubles + KGDB y + KGDB_SERIAL_CONSOLE y + KGDB_KDB y + ''; + makeFlags = [ "LOADADDR=0x0200000" ]; + target = "uImage"; + DTB = true; # Beyond 3.10 + }; + gcc = { + arch = "armv5te"; + }; + }; + + raspberrypi = { + linux-kernel = { + name = "raspberrypi"; + + baseConfig = "bcm2835_defconfig"; + DTB = true; + autoModules = true; + preferBuiltin = true; + extraConfig = '' + # Disable OABI to have seccomp_filter (required for systemd) + # https://github.com/raspberrypi/firmware/issues/651 + OABI_COMPAT n + ''; + target = "zImage"; + }; + gcc = { + arch = "armv6"; + fpu = "vfp"; + }; + }; + + # Legacy attribute, for compatibility with existing configs only. + raspberrypi2 = armv7l-hf-multiplatform; + + zero-gravitas = { + linux-kernel = { + name = "zero-gravitas"; + + baseConfig = "zero-gravitas_defconfig"; + # Target verified by checking /boot on reMarkable 1 device + target = "zImage"; + autoModules = false; + DTB = true; + }; + gcc = { + fpu = "neon"; + cpu = "cortex-a9"; + }; + }; + + zero-sugar = { + linux-kernel = { + name = "zero-sugar"; + + baseConfig = "zero-sugar_defconfig"; + DTB = true; + autoModules = false; + preferBuiltin = true; + target = "zImage"; + }; + gcc = { + cpu = "cortex-a7"; + fpu = "neon-vfpv4"; + float-abi = "hard"; + }; + }; + + utilite = { + linux-kernel = { + name = "utilite"; + maseConfig = "multi_v7_defconfig"; + autoModules = false; + extraConfig = '' + # Ubi for the mtd + MTD_UBI y + UBIFS_FS y + UBIFS_FS_XATTR y + UBIFS_FS_ADVANCED_COMPR y + UBIFS_FS_LZO y + UBIFS_FS_ZLIB y + UBIFS_FS_DEBUG n + ''; + makeFlags = [ "LOADADDR=0x10800000" ]; + target = "uImage"; + DTB = true; + }; + gcc = { + cpu = "cortex-a9"; + fpu = "neon"; + }; + }; + + guruplug = lib.recursiveUpdate sheevaplug { + # Define `CONFIG_MACH_GURUPLUG' (see + # ) + # and other GuruPlug-specific things. Requires the `guruplug-defconfig' + # patch. + linux-kernel.baseConfig = "guruplug_defconfig"; + }; + + beaglebone = lib.recursiveUpdate armv7l-hf-multiplatform { + linux-kernel = { + name = "beaglebone"; + baseConfig = "bb.org_defconfig"; + autoModules = false; + extraConfig = ""; # TBD kernel config + target = "zImage"; + }; + }; + + # https://developer.android.com/ndk/guides/abis#v7a + armv7a-android = { + linux-kernel.name = "armeabi-v7a"; + gcc = { + arch = "armv7-a"; + float-abi = "softfp"; + fpu = "vfpv3-d16"; + }; + }; + + armv7l-hf-multiplatform = { + linux-kernel = { + name = "armv7l-hf-multiplatform"; + Major = "2.6"; # Using "2.6" enables 2.6 kernel syscalls in glibc. + baseConfig = "multi_v7_defconfig"; + DTB = true; + autoModules = true; + preferBuiltin = true; + target = "zImage"; + extraConfig = '' + # Serial port for Raspberry Pi 3. Wasn't included in ARMv7 defconfig + # until 4.17. + SERIAL_8250_BCM2835AUX y + SERIAL_8250_EXTENDED y + SERIAL_8250_SHARE_IRQ y + + # Hangs ODROID-XU4 + ARM_BIG_LITTLE_CPUIDLE n + + # Disable OABI to have seccomp_filter (required for systemd) + # https://github.com/raspberrypi/firmware/issues/651 + OABI_COMPAT n + + # >=5.12 fails with: + # drivers/net/ethernet/micrel/ks8851_common.o: in function `ks8851_probe_common': + # ks8851_common.c:(.text+0x179c): undefined reference to `__this_module' + # See: https://lore.kernel.org/netdev/20210116164828.40545-1-marex@denx.de/T/ + KS8851_MLL y + ''; + }; + gcc = { + # Some table about fpu flags: + # http://community.arm.com/servlet/JiveServlet/showImage/38-1981-3827/blogentry-103749-004812900+1365712953_thumb.png + # Cortex-A5: -mfpu=neon-fp16 + # Cortex-A7 (rpi2): -mfpu=neon-vfpv4 + # Cortex-A8 (beaglebone): -mfpu=neon + # Cortex-A9: -mfpu=neon-fp16 + # Cortex-A15: -mfpu=neon-vfpv4 + + # More about FPU: + # https://wiki.debian.org/ArmHardFloatPort/VfpComparison + + # vfpv3-d16 is what Debian uses and seems to be the best compromise: NEON is not supported in e.g. Scaleway or Tegra 2, + # and the above page suggests NEON is only an improvement with hand-written assembly. + arch = "armv7-a"; + fpu = "vfpv3-d16"; + + # For Raspberry Pi the 2 the best would be: + # cpu = "cortex-a7"; + # fpu = "neon-vfpv4"; + }; + }; + + aarch64-multiplatform = { + linux-kernel = { + name = "aarch64-multiplatform"; + baseConfig = "defconfig"; + DTB = true; + autoModules = true; + preferBuiltin = true; + extraConfig = '' + # Raspberry Pi 3 stuff. Not needed for s >= 4.10. + ARCH_BCM2835 y + BCM2835_MBOX y + BCM2835_WDT y + RASPBERRYPI_FIRMWARE y + RASPBERRYPI_POWER y + SERIAL_8250_BCM2835AUX y + SERIAL_8250_EXTENDED y + SERIAL_8250_SHARE_IRQ y + + # Cavium ThunderX stuff. + PCI_HOST_THUNDER_ECAM y + + # Nvidia Tegra stuff. + PCI_TEGRA y + + # The default (=y) forces us to have the XHCI firmware available in initrd, + # which our initrd builder can't currently do easily. + USB_XHCI_TEGRA m + ''; + target = "Image"; + }; + gcc = { + arch = "armv8-a"; + }; + }; + + apple-m1 = { + gcc = { + arch = "armv8.3-a+crypto+sha2+aes+crc+fp16+lse+simd+ras+rdm+rcpc"; + cpu = "apple-a13"; + }; + }; + + ## + ## MIPS + ## + + ben_nanonote = { + linux-kernel = { + name = "ben_nanonote"; + }; + gcc = { + arch = "mips32"; + float = "soft"; + }; + }; + + fuloong2f_n32 = { + linux-kernel = { + name = "fuloong2f_n32"; + baseConfig = "lemote2f_defconfig"; + autoModules = false; + extraConfig = '' + MIGRATION n + COMPACTION n + + # nixos mounts some cgroup + CGROUPS y + + BLK_DEV_RAM y + BLK_DEV_INITRD y + BLK_DEV_CRYPTOLOOP m + BLK_DEV_DM m + DM_CRYPT m + MD y + REISERFS_FS m + EXT4_FS m + USB_STORAGE_CYPRESS_ATACB m + + IP_PNP y + IP_PNP_DHCP y + IP_PNP_BOOTP y + NFS_FS y + ROOT_NFS y + TUN m + NFS_V4 y + NFS_V4_1 y + NFS_FSCACHE y + NFSD m + NFSD_V2_ACL y + NFSD_V3 y + NFSD_V3_ACL y + NFSD_V4 y + + # Fail to build + DRM n + SCSI_ADVANSYS n + USB_ISP1362_HCD n + SND_SOC n + SND_ALI5451 n + FB_SAVAGE n + SCSI_NSP32 n + ATA_SFF n + SUNGEM n + IRDA n + ATM_HE n + SCSI_ACARD n + BLK_DEV_CMD640_ENHANCED n + + FUSE_FS m + + # Needed for udev >= 150 + SYSFS_DEPRECATED_V2 n + + VGA_CONSOLE n + VT_HW_CONSOLE_BINDING y + SERIAL_8250_CONSOLE y + FRAMEBUFFER_CONSOLE y + EXT2_FS y + EXT3_FS y + REISERFS_FS y + MAGIC_SYSRQ y + + # The kernel doesn't boot at all, with FTRACE + FTRACE n + ''; + target = "vmlinux"; + }; + gcc = { + arch = "loongson2f"; + float = "hard"; + abi = "n32"; + }; + }; + + # can execute on 32bit chip + gcc_mips32r2_o32 = { gcc = { arch = "mips32r2"; abi = "32"; }; }; + gcc_mips32r6_o32 = { gcc = { arch = "mips32r6"; abi = "32"; }; }; + gcc_mips64r2_n32 = { gcc = { arch = "mips64r2"; abi = "n32"; }; }; + gcc_mips64r6_n32 = { gcc = { arch = "mips64r6"; abi = "n32"; }; }; + gcc_mips64r2_64 = { gcc = { arch = "mips64r2"; abi = "64"; }; }; + gcc_mips64r6_64 = { gcc = { arch = "mips64r6"; abi = "64"; }; }; + + # based on: + # https://www.mail-archive.com/qemu-discuss@nongnu.org/msg05179.html + # https://gmplib.org/~tege/qemu.html#mips64-debian + mips64el-qemu-linux-gnuabi64 = { + linux-kernel = { + name = "mips64el"; + baseConfig = "64r2el_defconfig"; + target = "vmlinuz"; + autoModules = false; + DTB = true; + # for qemu 9p passthrough filesystem + extraConfig = '' + MIPS_MALTA y + PAGE_SIZE_4KB y + CPU_LITTLE_ENDIAN y + CPU_MIPS64_R2 y + 64BIT y + CPU_MIPS64_R2 y + + NET_9P y + NET_9P_VIRTIO y + 9P_FS y + 9P_FS_POSIX_ACL y + PCI y + VIRTIO_PCI y + ''; + }; + }; + + ## + ## Other + ## + + riscv-multiplatform = { + linux-kernel = { + name = "riscv-multiplatform"; + target = "Image"; + autoModules = true; + baseConfig = "defconfig"; + DTB = true; + extraConfig = '' + SERIAL_OF_PLATFORM y + ''; + }; + }; + + # This function takes a minimally-valid "platform" and returns an + # attrset containing zero or more additional attrs which should be + # included in the platform in order to further elaborate it. + select = platform: + # x86 + /**/ if platform.isx86 then pc + + # ARM + else if platform.isAarch32 then let + version = platform.parsed.cpu.version or null; + in if version == null then pc + else if lib.versionOlder version "6" then sheevaplug + else if lib.versionOlder version "7" then raspberrypi + else armv7l-hf-multiplatform + + else if platform.isAarch64 then + if platform.isDarwin then apple-m1 + else aarch64-multiplatform + + else if platform.isRiscV then riscv-multiplatform + + else if platform.parsed.cpu == lib.systems.parse.cpuTypes.mipsel then (import ./examples.nix { inherit lib; }).mipsel-linux-gnu + + else if platform.parsed.cpu == lib.systems.parse.cpuTypes.powerpc64le then powernv + + else { }; +} diff --git a/tests/check-eval.nix b/tests/check-eval.nix new file mode 100644 index 000000000..8bd7b605a --- /dev/null +++ b/tests/check-eval.nix @@ -0,0 +1,7 @@ +# Throws an error if any of our lib tests fail. + +let tests = [ "misc" "systems" ]; + all = builtins.concatLists (map (f: import (./. + "/${f}.nix")) tests); +in if all == [] + then null + else throw (builtins.toJSON all) diff --git a/tests/filesystem.sh b/tests/filesystem.sh new file mode 100755 index 000000000..cfd333d00 --- /dev/null +++ b/tests/filesystem.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +# Tests lib/filesystem.nix +# Run: +# [nixpkgs]$ lib/tests/filesystem.sh +# or: +# [nixpkgs]$ nix-build lib/tests/release.nix + +set -euo pipefail +shopt -s inherit_errexit + +# Use +# || die +die() { + echo >&2 "test case failed: " "$@" + exit 1 +} + +if test -n "${TEST_LIB:-}"; then + NIX_PATH=nixpkgs="$(dirname "$TEST_LIB")" +else + NIX_PATH=nixpkgs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.."; pwd)" +fi +export NIX_PATH + +work="$(mktemp -d)" +clean_up() { + rm -rf "$work" +} +trap clean_up EXIT +cd "$work" + +mkdir directory +touch regular +ln -s target symlink +mkfifo fifo + +expectSuccess() { + local expr=$1 + local expectedResultRegex=$2 + if ! result=$(nix-instantiate --eval --strict --json \ + --expr "with (import ).filesystem; $expr"); then + die "$expr failed to evaluate, but it was expected to succeed" + fi + if [[ ! "$result" =~ $expectedResultRegex ]]; then + die "$expr == $result, but $expectedResultRegex was expected" + fi +} + +expectFailure() { + local expr=$1 + local expectedErrorRegex=$2 + if result=$(nix-instantiate --eval --strict --json 2>"$work/stderr" \ + --expr "with (import ).filesystem; $expr"); then + die "$expr evaluated successfully to $result, but it was expected to fail" + fi + if [[ ! "$(<"$work/stderr")" =~ $expectedErrorRegex ]]; then + die "Error was $(<"$work/stderr"), but $expectedErrorRegex was expected" + fi +} + +expectSuccess "pathType /." '"directory"' +expectSuccess "pathType $PWD/directory" '"directory"' +expectSuccess "pathType $PWD/regular" '"regular"' +expectSuccess "pathType $PWD/symlink" '"symlink"' +expectSuccess "pathType $PWD/fifo" '"unknown"' +# Different errors depending on whether the builtins.readFilePath primop is available or not +expectFailure "pathType $PWD/non-existent" "error: (evaluation aborted with the following error message: 'lib.filesystem.pathType: Path $PWD/non-existent does not exist.'|getting status of '$PWD/non-existent': No such file or directory)" + +expectSuccess "pathIsDirectory /." "true" +expectSuccess "pathIsDirectory $PWD/directory" "true" +expectSuccess "pathIsDirectory $PWD/regular" "false" +expectSuccess "pathIsDirectory $PWD/symlink" "false" +expectSuccess "pathIsDirectory $PWD/fifo" "false" +expectSuccess "pathIsDirectory $PWD/non-existent" "false" + +expectSuccess "pathIsRegularFile /." "false" +expectSuccess "pathIsRegularFile $PWD/directory" "false" +expectSuccess "pathIsRegularFile $PWD/regular" "true" +expectSuccess "pathIsRegularFile $PWD/symlink" "false" +expectSuccess "pathIsRegularFile $PWD/fifo" "false" +expectSuccess "pathIsRegularFile $PWD/non-existent" "false" + +echo >&2 tests ok diff --git a/tests/maintainer-module.nix b/tests/maintainer-module.nix new file mode 100644 index 000000000..afa12587a --- /dev/null +++ b/tests/maintainer-module.nix @@ -0,0 +1,32 @@ +{ lib, ... }: +let + inherit (lib) types; +in { + options = { + name = lib.mkOption { + type = types.str; + }; + email = lib.mkOption { + type = types.nullOr types.str; + default = null; + }; + matrix = lib.mkOption { + type = types.nullOr types.str; + default = null; + }; + github = lib.mkOption { + type = types.nullOr types.str; + default = null; + }; + githubId = lib.mkOption { + type = types.nullOr types.ints.unsigned; + default = null; + }; + keys = lib.mkOption { + type = types.listOf (types.submodule { + options.fingerprint = lib.mkOption { type = types.str; }; + }); + default = []; + }; + }; +} diff --git a/tests/maintainers.nix b/tests/maintainers.nix new file mode 100644 index 000000000..be1c8aaa8 --- /dev/null +++ b/tests/maintainers.nix @@ -0,0 +1,53 @@ +# to run these tests (and the others) +# nix-build nixpkgs/lib/tests/release.nix +# These tests should stay in sync with the comment in maintainers/maintainers-list.nix +{ # The pkgs used for dependencies for the testing itself + pkgs ? import ../.. {} +, lib ? pkgs.lib +}: + +let + checkMaintainer = handle: uncheckedAttrs: + let + prefix = [ "lib" "maintainers" handle ]; + checkedAttrs = (lib.modules.evalModules { + inherit prefix; + modules = [ + ./maintainer-module.nix + { + _file = toString ../../maintainers/maintainer-list.nix; + config = uncheckedAttrs; + } + ]; + }).config; + + checks = lib.optional (checkedAttrs.github != null && checkedAttrs.githubId == null) '' + echo ${lib.escapeShellArg (lib.showOption prefix)}': If `github` is specified, `githubId` must be too.' + # Calling this too often would hit non-authenticated API limits, but this + # shouldn't happen since such errors will get fixed rather quickly + info=$(curl -sS https://api.github.com/users/${checkedAttrs.github}) + id=$(jq -r '.id' <<< "$info") + echo "The GitHub ID for GitHub user ${checkedAttrs.github} is $id:" + echo -e " githubId = $id;\n" + '' ++ lib.optional (checkedAttrs.email == null && checkedAttrs.github == null && checkedAttrs.matrix == null) '' + echo ${lib.escapeShellArg (lib.showOption prefix)}': At least one of `email`, `github` or `matrix` must be specified, so that users know how to reach you.' + '' ++ lib.optional (checkedAttrs.email != null && lib.hasSuffix "noreply.github.com" checkedAttrs.email) '' + echo ${lib.escapeShellArg (lib.showOption prefix)}': If an email address is given, it should allow people to reach you. If you do not want that, you can just provide `github` or `matrix` instead.' + ''; + in lib.deepSeq checkedAttrs checks; + + missingGithubIds = lib.concatLists (lib.mapAttrsToList checkMaintainer lib.maintainers); + + success = pkgs.runCommand "checked-maintainers-success" {} ">$out"; + + failure = pkgs.runCommand "checked-maintainers-failure" { + nativeBuildInputs = [ pkgs.curl pkgs.jq ]; + outputHash = "sha256:${lib.fakeSha256}"; + outputHAlgo = "sha256"; + outputHashMode = "flat"; + SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; + } '' + ${lib.concatStringsSep "\n" missingGithubIds} + exit 1 + ''; +in if missingGithubIds == [] then success else failure diff --git a/tests/misc.nix b/tests/misc.nix new file mode 100644 index 000000000..ce980436c --- /dev/null +++ b/tests/misc.nix @@ -0,0 +1,1657 @@ +# to run these tests: +# nix-instantiate --eval --strict nixpkgs/lib/tests/misc.nix +# if the resulting list is empty, all tests passed +with import ../default.nix; + +let + testingThrow = expr: { + expr = (builtins.tryEval (builtins.seq expr "didn't throw")); + expected = { success = false; value = false; }; + }; + testingDeepThrow = expr: testingThrow (builtins.deepSeq expr expr); + + testSanitizeDerivationName = { name, expected }: + let + drv = derivation { + name = strings.sanitizeDerivationName name; + builder = "x"; + system = "x"; + }; + in { + # Evaluate the derivation so an invalid name would be caught + expr = builtins.seq drv.drvPath drv.name; + inherit expected; + }; + +in + +runTests { + +# TRIVIAL + + testId = { + expr = id 1; + expected = 1; + }; + + testConst = { + expr = const 2 3; + expected = 2; + }; + + testPipe = { + expr = pipe 2 [ + (x: x + 2) # 2 + 2 = 4 + (x: x * 2) # 4 * 2 = 8 + ]; + expected = 8; + }; + + testPipeEmpty = { + expr = pipe 2 []; + expected = 2; + }; + + testPipeStrings = { + expr = pipe [ 3 4 ] [ + (map toString) + (map (s: s + "\n")) + concatStrings + ]; + expected = '' + 3 + 4 + ''; + }; + + /* + testOr = { + expr = or true false; + expected = true; + }; + */ + + testAnd = { + expr = and true false; + expected = false; + }; + + testFix = { + expr = fix (x: {a = if x ? a then "a" else "b";}); + expected = {a = "a";}; + }; + + testComposeExtensions = { + expr = let obj = makeExtensible (self: { foo = self.bar; }); + f = self: super: { bar = false; baz = true; }; + g = self: super: { bar = super.baz or false; }; + f_o_g = composeExtensions f g; + composed = obj.extend f_o_g; + in composed.foo; + expected = true; + }; + + testComposeManyExtensions0 = { + expr = let obj = makeExtensible (self: { foo = true; }); + emptyComposition = composeManyExtensions []; + composed = obj.extend emptyComposition; + in composed.foo; + expected = true; + }; + + testComposeManyExtensions = + let f = self: super: { bar = false; baz = true; }; + g = self: super: { bar = super.baz or false; }; + h = self: super: { qux = super.bar or false; }; + obj = makeExtensible (self: { foo = self.qux; }); + in { + expr = let composition = composeManyExtensions [f g h]; + composed = obj.extend composition; + in composed.foo; + expected = (obj.extend (composeExtensions f (composeExtensions g h))).foo; + }; + + testBitAnd = { + expr = (bitAnd 3 10); + expected = 2; + }; + + testBitOr = { + expr = (bitOr 3 10); + expected = 11; + }; + + testBitXor = { + expr = (bitXor 3 10); + expected = 9; + }; + + testToHexString = { + expr = toHexString 250; + expected = "FA"; + }; + + testToBaseDigits = { + expr = toBaseDigits 2 6; + expected = [ 1 1 0 ]; + }; + + testFunctionArgsFunctor = { + expr = functionArgs { __functor = self: { a, b }: null; }; + expected = { a = false; b = false; }; + }; + + testFunctionArgsSetFunctionArgs = { + expr = functionArgs (setFunctionArgs (args: args.x) { x = false; }); + expected = { x = false; }; + }; + +# STRINGS + + testConcatMapStrings = { + expr = concatMapStrings (x: x + ";") ["a" "b" "c"]; + expected = "a;b;c;"; + }; + + testConcatStringsSep = { + expr = concatStringsSep "," ["a" "b" "c"]; + expected = "a,b,c"; + }; + + testConcatLines = { + expr = concatLines ["a" "b" "c"]; + expected = "a\nb\nc\n"; + }; + + testSplitStringsSimple = { + expr = strings.splitString "." "a.b.c.d"; + expected = [ "a" "b" "c" "d" ]; + }; + + testSplitStringsEmpty = { + expr = strings.splitString "." "a..b"; + expected = [ "a" "" "b" ]; + }; + + testSplitStringsOne = { + expr = strings.splitString ":" "a.b"; + expected = [ "a.b" ]; + }; + + testSplitStringsNone = { + expr = strings.splitString "." ""; + expected = [ "" ]; + }; + + testSplitStringsFirstEmpty = { + expr = strings.splitString "/" "/a/b/c"; + expected = [ "" "a" "b" "c" ]; + }; + + testSplitStringsLastEmpty = { + expr = strings.splitString ":" "2001:db8:0:0042::8a2e:370:"; + expected = [ "2001" "db8" "0" "0042" "" "8a2e" "370" "" ]; + }; + + testSplitStringsRegex = { + expr = strings.splitString "\\[{}]()^$?*+|." "A\\[{}]()^$?*+|.B"; + expected = [ "A" "B" ]; + }; + + testSplitStringsDerivation = { + expr = take 3 (strings.splitString "/" (derivation { + name = "name"; + builder = "builder"; + system = "system"; + })); + expected = ["" "nix" "store"]; + }; + + testSplitVersionSingle = { + expr = versions.splitVersion "1"; + expected = [ "1" ]; + }; + + testSplitVersionDouble = { + expr = versions.splitVersion "1.2"; + expected = [ "1" "2" ]; + }; + + testSplitVersionTriple = { + expr = versions.splitVersion "1.2.3"; + expected = [ "1" "2" "3" ]; + }; + + testPadVersionLess = { + expr = versions.pad 3 "1.2"; + expected = "1.2.0"; + }; + + testPadVersionLessExtra = { + expr = versions.pad 3 "1.3-rc1"; + expected = "1.3.0-rc1"; + }; + + testPadVersionMore = { + expr = versions.pad 3 "1.2.3.4"; + expected = "1.2.3"; + }; + + testIsStorePath = { + expr = + let goodPath = + "${builtins.storeDir}/d945ibfx9x185xf04b890y4f9g3cbb63-python-2.7.11"; + in { + storePath = isStorePath goodPath; + storePathDerivation = isStorePath (import ../.. { system = "x86_64-linux"; }).hello; + storePathAppendix = isStorePath + "${goodPath}/bin/python"; + nonAbsolute = isStorePath (concatStrings (tail (stringToCharacters goodPath))); + asPath = isStorePath (/. + goodPath); + otherPath = isStorePath "/something/else"; + otherVals = { + attrset = isStorePath {}; + list = isStorePath []; + int = isStorePath 42; + }; + }; + expected = { + storePath = true; + storePathDerivation = true; + storePathAppendix = false; + nonAbsolute = false; + asPath = true; + otherPath = false; + otherVals = { + attrset = false; + list = false; + int = false; + }; + }; + }; + + testEscapeXML = { + expr = escapeXML ''"test" 'test' < & >''; + expected = ""test" 'test' < & >"; + }; + + testToShellVars = { + expr = '' + ${toShellVars { + STRing01 = "just a 'string'"; + _array_ = [ "with" "more strings" ]; + assoc."with some" = '' + strings + possibly newlines + ''; + drv = { + outPath = "/drv"; + foo = "ignored attribute"; + }; + path = /path; + stringable = { + __toString = _: "hello toString"; + bar = "ignored attribute"; + }; + }} + ''; + expected = '' + STRing01='just a '\'''string'\'''' + declare -a _array_=('with' 'more strings') + declare -A assoc=(['with some']='strings + possibly newlines + ') + drv='/drv' + path='/path' + stringable='hello toString' + ''; + }; + + testHasInfixFalse = { + expr = hasInfix "c" "abde"; + expected = false; + }; + + testHasInfixTrue = { + expr = hasInfix "c" "abcde"; + expected = true; + }; + + testHasInfixDerivation = { + expr = hasInfix "hello" (import ../.. { system = "x86_64-linux"; }).hello; + expected = true; + }; + + testHasInfixPath = { + expr = hasInfix "tests" ./.; + expected = true; + }; + + testHasInfixPathStoreDir = { + expr = hasInfix builtins.storeDir ./.; + expected = true; + }; + + testHasInfixToString = { + expr = hasInfix "a" { __toString = _: "a"; }; + expected = true; + }; + + testNormalizePath = { + expr = strings.normalizePath "//a/b//c////d/"; + expected = "/a/b/c/d/"; + }; + + testCharToInt = { + expr = strings.charToInt "A"; + expected = 65; + }; + + testEscapeC = { + expr = strings.escapeC [ " " ] "Hello World"; + expected = "Hello\\x20World"; + }; + + testEscapeURL = testAllTrue [ + ("" == strings.escapeURL "") + ("Hello" == strings.escapeURL "Hello") + ("Hello%20World" == strings.escapeURL "Hello World") + ("Hello%2FWorld" == strings.escapeURL "Hello/World") + ("42%25" == strings.escapeURL "42%") + ("%20%3F%26%3D%23%2B%25%21%3C%3E%23%22%7B%7D%7C%5C%5E%5B%5D%60%09%3A%2F%40%24%27%28%29%2A%2C%3B" == strings.escapeURL " ?&=#+%!<>#\"{}|\\^[]`\t:/@$'()*,;") + ]; + + testToInt = testAllTrue [ + # Naive + (123 == toInt "123") + (0 == toInt "0") + # Whitespace Padding + (123 == toInt " 123") + (123 == toInt "123 ") + (123 == toInt " 123 ") + (123 == toInt " 123 ") + (0 == toInt " 0") + (0 == toInt "0 ") + (0 == toInt " 0 ") + (-1 == toInt "-1") + (-1 == toInt " -1 ") + ]; + + testToIntFails = testAllTrue [ + ( builtins.tryEval (toInt "") == { success = false; value = false; } ) + ( builtins.tryEval (toInt "123 123") == { success = false; value = false; } ) + ( builtins.tryEval (toInt "0 123") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " 0d ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " 1d ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " d0 ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt "00") == { success = false; value = false; } ) + ( builtins.tryEval (toInt "01") == { success = false; value = false; } ) + ( builtins.tryEval (toInt "002") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " 002 ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " foo ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " foo 123 ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " foo123 ") == { success = false; value = false; } ) + ]; + + testToIntBase10 = testAllTrue [ + # Naive + (123 == toIntBase10 "123") + (0 == toIntBase10 "0") + # Whitespace Padding + (123 == toIntBase10 " 123") + (123 == toIntBase10 "123 ") + (123 == toIntBase10 " 123 ") + (123 == toIntBase10 " 123 ") + (0 == toIntBase10 " 0") + (0 == toIntBase10 "0 ") + (0 == toIntBase10 " 0 ") + # Zero Padding + (123 == toIntBase10 "0123") + (123 == toIntBase10 "0000123") + (0 == toIntBase10 "000000") + # Whitespace and Zero Padding + (123 == toIntBase10 " 0123") + (123 == toIntBase10 "0123 ") + (123 == toIntBase10 " 0123 ") + (123 == toIntBase10 " 0000123") + (123 == toIntBase10 "0000123 ") + (123 == toIntBase10 " 0000123 ") + (0 == toIntBase10 " 000000") + (0 == toIntBase10 "000000 ") + (0 == toIntBase10 " 000000 ") + (-1 == toIntBase10 "-1") + (-1 == toIntBase10 " -1 ") + ]; + + testToIntBase10Fails = testAllTrue [ + ( builtins.tryEval (toIntBase10 "") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 "123 123") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 "0 123") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " 0d ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " 1d ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " d0 ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " foo ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " foo 123 ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " foo 00123 ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " foo00123 ") == { success = false; value = false; } ) + ]; + +# LISTS + + testFilter = { + expr = filter (x: x != "a") ["a" "b" "c" "a"]; + expected = ["b" "c"]; + }; + + testFold = + let + f = op: fold: fold op 0 (range 0 100); + # fold with associative operator + assoc = f builtins.add; + # fold with non-associative operator + nonAssoc = f builtins.sub; + in { + expr = { + assocRight = assoc foldr; + # right fold with assoc operator is same as left fold + assocRightIsLeft = assoc foldr == assoc foldl; + nonAssocRight = nonAssoc foldr; + nonAssocLeft = nonAssoc foldl; + # with non-assoc operator the fold results are not the same + nonAssocRightIsNotLeft = nonAssoc foldl != nonAssoc foldr; + # fold is an alias for foldr + foldIsRight = nonAssoc fold == nonAssoc foldr; + }; + expected = { + assocRight = 5050; + assocRightIsLeft = true; + nonAssocRight = 50; + nonAssocLeft = (-5050); + nonAssocRightIsNotLeft = true; + foldIsRight = true; + }; + }; + + testTake = testAllTrue [ + ([] == (take 0 [ 1 2 3 ])) + ([1] == (take 1 [ 1 2 3 ])) + ([ 1 2 ] == (take 2 [ 1 2 3 ])) + ([ 1 2 3 ] == (take 3 [ 1 2 3 ])) + ([ 1 2 3 ] == (take 4 [ 1 2 3 ])) + ]; + + testFoldAttrs = { + expr = foldAttrs (n: a: [n] ++ a) [] [ + { a = 2; b = 7; } + { a = 3; c = 8; } + ]; + expected = { a = [ 2 3 ]; b = [7]; c = [8];}; + }; + + testSort = { + expr = sort builtins.lessThan [ 40 2 30 42 ]; + expected = [2 30 40 42]; + }; + + testReplicate = { + expr = replicate 3 "a"; + expected = ["a" "a" "a"]; + }; + + testToIntShouldConvertStringToInt = { + expr = toInt "27"; + expected = 27; + }; + + testToIntShouldThrowErrorIfItCouldNotConvertToInt = { + expr = builtins.tryEval (toInt "\"foo\""); + expected = { success = false; value = false; }; + }; + + testHasAttrByPathTrue = { + expr = hasAttrByPath ["a" "b"] { a = { b = "yey"; }; }; + expected = true; + }; + + testHasAttrByPathFalse = { + expr = hasAttrByPath ["a" "b"] { a = { c = "yey"; }; }; + expected = false; + }; + + testFindFirstExample1 = { + expr = findFirst (x: x > 3) 7 [ 1 6 4 ]; + expected = 6; + }; + + testFindFirstExample2 = { + expr = findFirst (x: x > 9) 7 [ 1 6 4 ]; + expected = 7; + }; + + testFindFirstEmpty = { + expr = findFirst (abort "when the list is empty, the predicate is not needed") null []; + expected = null; + }; + + testFindFirstSingleMatch = { + expr = findFirst (x: x == 5) null [ 5 ]; + expected = 5; + }; + + testFindFirstSingleDefault = { + expr = findFirst (x: false) null [ (abort "if the predicate doesn't access the value, it must not be evaluated") ]; + expected = null; + }; + + testFindFirstNone = { + expr = builtins.tryEval (findFirst (x: x == 2) null [ 1 (throw "the last element must be evaluated when there's no match") ]); + expected = { success = false; value = false; }; + }; + + # Makes sure that the implementation doesn't cause a stack overflow + testFindFirstBig = { + expr = findFirst (x: x == 1000000) null (range 0 1000000); + expected = 1000000; + }; + + testFindFirstLazy = { + expr = findFirst (x: x == 1) 7 [ 1 (abort "list elements after the match must not be evaluated") ]; + expected = 1; + }; + +# ATTRSETS + + testConcatMapAttrs = { + expr = concatMapAttrs + (name: value: { + ${name} = value; + ${name + value} = value; + }) + { + foo = "bar"; + foobar = "baz"; + }; + expected = { + foo = "bar"; + foobar = "baz"; + foobarbaz = "baz"; + }; + }; + + # code from example + testFoldlAttrs = { + expr = { + example = foldlAttrs + (acc: name: value: { + sum = acc.sum + value; + names = acc.names ++ [ name ]; + }) + { sum = 0; names = [ ]; } + { + foo = 1; + bar = 10; + }; + # should just return the initial value + emptySet = foldlAttrs (throw "function not needed") 123 { }; + # should just evaluate to the last value + accNotNeeded = foldlAttrs (_acc: _name: v: v) (throw "accumulator not needed") { z = 3; a = 2; }; + # the accumulator doesnt have to be an attrset it can be as trivial as being just a number or string + trivialAcc = foldlAttrs (acc: _name: v: acc * 10 + v) 1 { z = 1; a = 2; }; + }; + expected = { + example = { + sum = 11; + names = [ "bar" "foo" ]; + }; + emptySet = 123; + accNotNeeded = 3; + trivialAcc = 121; + }; + }; + + # code from the example + testRecursiveUpdateUntil = { + expr = recursiveUpdateUntil (path: l: r: path == ["foo"]) { + # first attribute set + foo.bar = 1; + foo.baz = 2; + bar = 3; + } { + #second attribute set + foo.bar = 1; + foo.quz = 2; + baz = 4; + }; + expected = { + foo.bar = 1; # 'foo.*' from the second set + foo.quz = 2; # + bar = 3; # 'bar' from the first set + baz = 4; # 'baz' from the second set + }; + }; + + testOverrideExistingEmpty = { + expr = overrideExisting {} { a = 1; }; + expected = {}; + }; + + testOverrideExistingDisjoint = { + expr = overrideExisting { b = 2; } { a = 1; }; + expected = { b = 2; }; + }; + + testOverrideExistingOverride = { + expr = overrideExisting { a = 3; b = 2; } { a = 1; }; + expected = { a = 1; b = 2; }; + }; + +# GENERATORS +# these tests assume attributes are converted to lists +# in alphabetical order + + testMkKeyValueDefault = { + expr = generators.mkKeyValueDefault {} ":" "f:oo" "bar"; + expected = ''f\:oo:bar''; + }; + + testMkValueString = { + expr = let + vals = { + int = 42; + string = ''fo"o''; + bool = true; + bool2 = false; + null = null; + # float = 42.23; # floats are strange + }; + in mapAttrs + (const (generators.mkValueStringDefault {})) + vals; + expected = { + int = "42"; + string = ''fo"o''; + bool = "true"; + bool2 = "false"; + null = "null"; + # float = "42.23" true false [ "bar" ] ]''; + }; + }; + + testToKeyValue = { + expr = generators.toKeyValue {} { + key = "value"; + "other=key" = "baz"; + }; + expected = '' + key=value + other\=key=baz + ''; + }; + + testToINIEmpty = { + expr = generators.toINI {} {}; + expected = ""; + }; + + testToINIEmptySection = { + expr = generators.toINI {} { foo = {}; bar = {}; }; + expected = '' + [bar] + + [foo] + ''; + }; + + testToINIDuplicateKeys = { + expr = generators.toINI { listsAsDuplicateKeys = true; } { foo.bar = true; baz.qux = [ 1 false ]; }; + expected = '' + [baz] + qux=1 + qux=false + + [foo] + bar=true + ''; + }; + + testToINIDefaultEscapes = { + expr = generators.toINI {} { + "no [ and ] allowed unescaped" = { + "and also no = in keys" = 42; + }; + }; + expected = '' + [no \[ and \] allowed unescaped] + and also no \= in keys=42 + ''; + }; + + testToINIDefaultFull = { + expr = generators.toINI {} { + "section 1" = { + attribute1 = 5; + x = "Me-se JarJar Binx"; + # booleans are converted verbatim by default + boolean = false; + }; + "foo[]" = { + "he\\h=he" = "this is okay"; + }; + }; + expected = '' + [foo\[\]] + he\h\=he=this is okay + + [section 1] + attribute1=5 + boolean=false + x=Me-se JarJar Binx + ''; + }; + + testToINIWithGlobalSectionEmpty = { + expr = generators.toINIWithGlobalSection {} { + globalSection = { + }; + sections = { + }; + }; + expected = '' + ''; + }; + + testToINIWithGlobalSectionGlobalEmptyIsTheSameAsToINI = + let + sections = { + "section 1" = { + attribute1 = 5; + x = "Me-se JarJar Binx"; + }; + "foo" = { + "he\\h=he" = "this is okay"; + }; + }; + in { + expr = + generators.toINIWithGlobalSection {} { + globalSection = {}; + sections = sections; + }; + expected = generators.toINI {} sections; + }; + + testToINIWithGlobalSectionFull = { + expr = generators.toINIWithGlobalSection {} { + globalSection = { + foo = "bar"; + test = false; + }; + sections = { + "section 1" = { + attribute1 = 5; + x = "Me-se JarJar Binx"; + }; + "foo" = { + "he\\h=he" = "this is okay"; + }; + }; + }; + expected = '' + foo=bar + test=false + + [foo] + he\h\=he=this is okay + + [section 1] + attribute1=5 + x=Me-se JarJar Binx + ''; + }; + + /* right now only invocation check */ + testToJSONSimple = + let val = { + foobar = [ "baz" 1 2 3 ]; + }; + in { + expr = generators.toJSON {} val; + # trivial implementation + expected = builtins.toJSON val; + }; + + /* right now only invocation check */ + testToYAMLSimple = + let val = { + list = [ { one = 1; } { two = 2; } ]; + all = 42; + }; + in { + expr = generators.toYAML {} val; + # trivial implementation + expected = builtins.toJSON val; + }; + + testToPretty = + let + deriv = derivation { name = "test"; builder = "/bin/sh"; system = "aarch64-linux"; }; + in { + expr = mapAttrs (const (generators.toPretty { multiline = false; })) rec { + int = 42; + float = 0.1337; + bool = true; + emptystring = ""; + string = "fn\${o}\"r\\d"; + newlinestring = "\n"; + path = /. + "/foo"; + null_ = null; + function = x: x; + functionArgs = { arg ? 4, foo }: arg; + list = [ 3 4 function [ false ] ]; + emptylist = []; + attrs = { foo = null; "foo b/ar" = "baz"; }; + emptyattrs = {}; + drv = deriv; + }; + expected = rec { + int = "42"; + float = "0.1337"; + bool = "true"; + emptystring = ''""''; + string = ''"fn\''${o}\"r\\d"''; + newlinestring = "\"\\n\""; + path = "/foo"; + null_ = "null"; + function = ""; + functionArgs = ""; + list = "[ 3 4 ${function} [ false ] ]"; + emptylist = "[ ]"; + attrs = "{ foo = null; \"foo b/ar\" = \"baz\"; }"; + emptyattrs = "{ }"; + drv = ""; + }; + }; + + testToPrettyLimit = + let + a.b = 1; + a.c = a; + in { + expr = generators.toPretty { } (generators.withRecursion { throwOnDepthLimit = false; depthLimit = 2; } a); + expected = "{\n b = 1;\n c = {\n b = \"\";\n c = {\n b = \"\";\n c = \"\";\n };\n };\n}"; + }; + + testToPrettyLimitThrow = + let + a.b = 1; + a.c = a; + in { + expr = (builtins.tryEval + (generators.toPretty { } (generators.withRecursion { depthLimit = 2; } a))).success; + expected = false; + }; + + testWithRecursionDealsWithFunctors = + let + functor = { + __functor = self: { a, b, }: null; + }; + a = { + value = "1234"; + b = functor; + c.d = functor; + }; + in { + expr = generators.toPretty { } (generators.withRecursion { depthLimit = 1; throwOnDepthLimit = false; } a); + expected = "{\n b = ;\n c = {\n d = \"\";\n };\n value = \"\";\n}"; + }; + + testToPrettyMultiline = { + expr = mapAttrs (const (generators.toPretty { })) rec { + list = [ 3 4 [ false ] ]; + attrs = { foo = null; bar.foo = "baz"; }; + newlinestring = "\n"; + multilinestring = '' + hello + ''${there} + te'''st + ''; + multilinestring' = '' + hello + there + test''; + }; + expected = rec { + list = '' + [ + 3 + 4 + [ + false + ] + ]''; + attrs = '' + { + bar = { + foo = "baz"; + }; + foo = null; + }''; + newlinestring = "''\n \n''"; + multilinestring = '' + ''' + hello + '''''${there} + te''''st + '''''; + multilinestring' = '' + ''' + hello + there + test'''''; + + }; + }; + + testToPrettyAllowPrettyValues = { + expr = generators.toPretty { allowPrettyValues = true; } + { __pretty = v: "«" + v + "»"; val = "foo"; }; + expected = "«foo»"; + }; + + testToPlist = + let + deriv = derivation { name = "test"; builder = "/bin/sh"; system = "aarch64-linux"; }; + in { + expr = mapAttrs (const (generators.toPlist { })) { + value = { + nested.values = rec { + int = 42; + float = 0.1337; + bool = true; + emptystring = ""; + string = "fn\${o}\"r\\d"; + newlinestring = "\n"; + path = /. + "/foo"; + null_ = null; + list = [ 3 4 "test" ]; + emptylist = []; + attrs = { foo = null; "foo b/ar" = "baz"; }; + emptyattrs = {}; + }; + }; + }; + expected = { value = builtins.readFile ./test-to-plist-expected.plist; }; + }; + + testToLuaEmptyAttrSet = { + expr = generators.toLua {} {}; + expected = ''{}''; + }; + + testToLuaEmptyList = { + expr = generators.toLua {} []; + expected = ''{}''; + }; + + testToLuaListOfVariousTypes = { + expr = generators.toLua {} [ null 43 3.14159 true ]; + expected = '' + { + nil, + 43, + 3.14159, + true + }''; + }; + + testToLuaString = { + expr = generators.toLua {} ''double-quote (") and single quotes (')''; + expected = ''"double-quote (\") and single quotes (')"''; + }; + + testToLuaAttrsetWithLuaInline = { + expr = generators.toLua {} { x = generators.mkLuaInline ''"abc" .. "def"''; }; + expected = '' + { + ["x"] = ("abc" .. "def") + }''; + }; + + testToLuaAttrsetWithSpaceInKey = { + expr = generators.toLua {} { "some space and double-quote (\")" = 42; }; + expected = '' + { + ["some space and double-quote (\")"] = 42 + }''; + }; + + testToLuaWithoutMultiline = { + expr = generators.toLua { multiline = false; } [ 41 43 ]; + expected = ''{ 41, 43 }''; + }; + + testToLuaEmptyBindings = { + expr = generators.toLua { asBindings = true; } {}; + expected = ""; + }; + + testToLuaBindings = { + expr = generators.toLua { asBindings = true; } { x1 = 41; _y = { a = 43; }; }; + expected = '' + _y = { + ["a"] = 43 + } + x1 = 41 + ''; + }; + + testToLuaPartialTableBindings = { + expr = generators.toLua { asBindings = true; } { "x.y" = 42; }; + expected = '' + x.y = 42 + ''; + }; + + testToLuaIndentedBindings = { + expr = generators.toLua { asBindings = true; indent = " "; } { x = { y = 42; }; }; + expected = " x = {\n [\"y\"] = 42\n }\n"; + }; + + testToLuaBindingsWithSpace = testingThrow ( + generators.toLua { asBindings = true; } { "with space" = 42; } + ); + + testToLuaBindingsWithLeadingDigit = testingThrow ( + generators.toLua { asBindings = true; } { "11eleven" = 42; } + ); + + testToLuaBasicExample = { + expr = generators.toLua {} { + cmd = [ "typescript-language-server" "--stdio" ]; + settings.workspace.library = generators.mkLuaInline ''vim.api.nvim_get_runtime_file("", true)''; + }; + expected = '' + { + ["cmd"] = { + "typescript-language-server", + "--stdio" + }, + ["settings"] = { + ["workspace"] = { + ["library"] = (vim.api.nvim_get_runtime_file("", true)) + } + } + }''; + }; + +# CLI + + testToGNUCommandLine = { + expr = cli.toGNUCommandLine {} { + data = builtins.toJSON { id = 0; }; + X = "PUT"; + retry = 3; + retry-delay = null; + url = [ "https://example.com/foo" "https://example.com/bar" ]; + silent = false; + verbose = true; + }; + + expected = [ + "-X" "PUT" + "--data" "{\"id\":0}" + "--retry" "3" + "--url" "https://example.com/foo" + "--url" "https://example.com/bar" + "--verbose" + ]; + }; + + testToGNUCommandLineShell = { + expr = cli.toGNUCommandLineShell {} { + data = builtins.toJSON { id = 0; }; + X = "PUT"; + retry = 3; + retry-delay = null; + url = [ "https://example.com/foo" "https://example.com/bar" ]; + silent = false; + verbose = true; + }; + + expected = "'-X' 'PUT' '--data' '{\"id\":0}' '--retry' '3' '--url' 'https://example.com/foo' '--url' 'https://example.com/bar' '--verbose'"; + }; + + testSanitizeDerivationNameLeadingDots = testSanitizeDerivationName { + name = "..foo"; + expected = "foo"; + }; + + testSanitizeDerivationNameUnicode = testSanitizeDerivationName { + name = "fö"; + expected = "f-"; + }; + + testSanitizeDerivationNameAscii = testSanitizeDerivationName { + name = " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"; + expected = "-+--.-0123456789-=-?-ABCDEFGHIJKLMNOPQRSTUVWXYZ-_-abcdefghijklmnopqrstuvwxyz-"; + }; + + testSanitizeDerivationNameTooLong = testSanitizeDerivationName { + name = "This string is loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong"; + expected = "loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong"; + }; + + testSanitizeDerivationNameTooLongWithInvalid = testSanitizeDerivationName { + name = "Hello there aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa &&&&&&&&"; + expected = "there-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-"; + }; + + testSanitizeDerivationNameEmpty = testSanitizeDerivationName { + name = ""; + expected = "unknown"; + }; + + testFreeformOptions = { + expr = + let + submodule = { lib, ... }: { + freeformType = lib.types.attrsOf (lib.types.submodule { + options.bar = lib.mkOption {}; + }); + options.bar = lib.mkOption {}; + }; + + module = { lib, ... }: { + options.foo = lib.mkOption { + type = lib.types.submodule submodule; + }; + }; + + options = (evalModules { + modules = [ module ]; + }).options; + + locs = filter (o: ! o.internal) (optionAttrSetToDocList options); + in map (o: o.loc) locs; + expected = [ [ "_module" "args" ] [ "foo" ] [ "foo" "" "bar" ] [ "foo" "bar" ] ]; + }; + + testCartesianProductOfEmptySet = { + expr = cartesianProductOfSets {}; + expected = [ {} ]; + }; + + testCartesianProductOfOneSet = { + expr = cartesianProductOfSets { a = [ 1 2 3 ]; }; + expected = [ { a = 1; } { a = 2; } { a = 3; } ]; + }; + + testCartesianProductOfTwoSets = { + expr = cartesianProductOfSets { a = [ 1 ]; b = [ 10 20 ]; }; + expected = [ + { a = 1; b = 10; } + { a = 1; b = 20; } + ]; + }; + + testCartesianProductOfTwoSetsWithOneEmpty = { + expr = cartesianProductOfSets { a = [ ]; b = [ 10 20 ]; }; + expected = [ ]; + }; + + testCartesianProductOfThreeSets = { + expr = cartesianProductOfSets { + a = [ 1 2 3 ]; + b = [ 10 20 30 ]; + c = [ 100 200 300 ]; + }; + expected = [ + { a = 1; b = 10; c = 100; } + { a = 1; b = 10; c = 200; } + { a = 1; b = 10; c = 300; } + + { a = 1; b = 20; c = 100; } + { a = 1; b = 20; c = 200; } + { a = 1; b = 20; c = 300; } + + { a = 1; b = 30; c = 100; } + { a = 1; b = 30; c = 200; } + { a = 1; b = 30; c = 300; } + + { a = 2; b = 10; c = 100; } + { a = 2; b = 10; c = 200; } + { a = 2; b = 10; c = 300; } + + { a = 2; b = 20; c = 100; } + { a = 2; b = 20; c = 200; } + { a = 2; b = 20; c = 300; } + + { a = 2; b = 30; c = 100; } + { a = 2; b = 30; c = 200; } + { a = 2; b = 30; c = 300; } + + { a = 3; b = 10; c = 100; } + { a = 3; b = 10; c = 200; } + { a = 3; b = 10; c = 300; } + + { a = 3; b = 20; c = 100; } + { a = 3; b = 20; c = 200; } + { a = 3; b = 20; c = 300; } + + { a = 3; b = 30; c = 100; } + { a = 3; b = 30; c = 200; } + { a = 3; b = 30; c = 300; } + ]; + }; + + # The example from the showAttrPath documentation + testShowAttrPathExample = { + expr = showAttrPath [ "foo" "10" "bar" ]; + expected = "foo.\"10\".bar"; + }; + + testShowAttrPathEmpty = { + expr = showAttrPath []; + expected = ""; + }; + + testShowAttrPathVarious = { + expr = showAttrPath [ + "." + "foo" + "2" + "a2-b" + "_bc'de" + ]; + expected = ''".".foo."2".a2-b._bc'de''; + }; + + testGroupBy = { + expr = groupBy (n: toString (mod n 5)) (range 0 16); + expected = { + "0" = [ 0 5 10 15 ]; + "1" = [ 1 6 11 16 ]; + "2" = [ 2 7 12 ]; + "3" = [ 3 8 13 ]; + "4" = [ 4 9 14 ]; + }; + }; + + testGroupBy' = { + expr = groupBy' builtins.add 0 (x: boolToString (x > 2)) [ 5 1 2 3 4 ]; + expected = { false = 3; true = 12; }; + }; + + # The example from the updateManyAttrsByPath documentation + testUpdateManyAttrsByPathExample = { + expr = updateManyAttrsByPath [ + { + path = [ "a" "b" ]; + update = old: { d = old.c; }; + } + { + path = [ "a" "b" "c" ]; + update = old: old + 1; + } + { + path = [ "x" "y" ]; + update = old: "xy"; + } + ] { a.b.c = 0; }; + expected = { a = { b = { d = 1; }; }; x = { y = "xy"; }; }; + }; + + # If there are no updates, the value is passed through + testUpdateManyAttrsByPathNone = { + expr = updateManyAttrsByPath [] "something"; + expected = "something"; + }; + + # A single update to the root path is just like applying the function directly + testUpdateManyAttrsByPathSingleIncrement = { + expr = updateManyAttrsByPath [ + { + path = [ ]; + update = old: old + 1; + } + ] 0; + expected = 1; + }; + + # Multiple updates can be applied are done in order + testUpdateManyAttrsByPathMultipleIncrements = { + expr = updateManyAttrsByPath [ + { + path = [ ]; + update = old: old + "a"; + } + { + path = [ ]; + update = old: old + "b"; + } + { + path = [ ]; + update = old: old + "c"; + } + ] ""; + expected = "abc"; + }; + + # If an update doesn't use the value, all previous updates are not evaluated + testUpdateManyAttrsByPathLazy = { + expr = updateManyAttrsByPath [ + { + path = [ ]; + update = old: old + throw "nope"; + } + { + path = [ ]; + update = old: "untainted"; + } + ] (throw "start"); + expected = "untainted"; + }; + + # Deeply nested attributes can be updated without affecting others + testUpdateManyAttrsByPathDeep = { + expr = updateManyAttrsByPath [ + { + path = [ "a" "b" "c" ]; + update = old: old + 1; + } + ] { + a.b.c = 0; + + a.b.z = 0; + a.y.z = 0; + x.y.z = 0; + }; + expected = { + a.b.c = 1; + + a.b.z = 0; + a.y.z = 0; + x.y.z = 0; + }; + }; + + # Nested attributes are updated first + testUpdateManyAttrsByPathNestedBeforehand = { + expr = updateManyAttrsByPath [ + { + path = [ "a" ]; + update = old: old // { x = old.b; }; + } + { + path = [ "a" "b" ]; + update = old: old + 1; + } + ] { + a.b = 0; + }; + expected = { + a.b = 1; + a.x = 1; + }; + }; + + ## Levenshtein distance functions and co. + testCommonPrefixLengthEmpty = { + expr = strings.commonPrefixLength "" "hello"; + expected = 0; + }; + + testCommonPrefixLengthSame = { + expr = strings.commonPrefixLength "hello" "hello"; + expected = 5; + }; + + testCommonPrefixLengthDiffering = { + expr = strings.commonPrefixLength "hello" "hey"; + expected = 2; + }; + + testCommonSuffixLengthEmpty = { + expr = strings.commonSuffixLength "" "hello"; + expected = 0; + }; + + testCommonSuffixLengthSame = { + expr = strings.commonSuffixLength "hello" "hello"; + expected = 5; + }; + + testCommonSuffixLengthDiffering = { + expr = strings.commonSuffixLength "test" "rest"; + expected = 3; + }; + + testLevenshteinEmpty = { + expr = strings.levenshtein "" ""; + expected = 0; + }; + + testLevenshteinOnlyAdd = { + expr = strings.levenshtein "" "hello there"; + expected = 11; + }; + + testLevenshteinOnlyRemove = { + expr = strings.levenshtein "hello there" ""; + expected = 11; + }; + + testLevenshteinOnlyTransform = { + expr = strings.levenshtein "abcdef" "ghijkl"; + expected = 6; + }; + + testLevenshteinMixed = { + expr = strings.levenshtein "kitchen" "sitting"; + expected = 5; + }; + + testLevenshteinAtMostZeroFalse = { + expr = strings.levenshteinAtMost 0 "foo" "boo"; + expected = false; + }; + + testLevenshteinAtMostZeroTrue = { + expr = strings.levenshteinAtMost 0 "foo" "foo"; + expected = true; + }; + + testLevenshteinAtMostOneFalse = { + expr = strings.levenshteinAtMost 1 "car" "ct"; + expected = false; + }; + + testLevenshteinAtMostOneTrue = { + expr = strings.levenshteinAtMost 1 "car" "cr"; + expected = true; + }; + + # We test levenshteinAtMost 2 particularly well because it uses a complicated + # implementation + testLevenshteinAtMostTwoIsEmpty = { + expr = strings.levenshteinAtMost 2 "" ""; + expected = true; + }; + + testLevenshteinAtMostTwoIsZero = { + expr = strings.levenshteinAtMost 2 "abcdef" "abcdef"; + expected = true; + }; + + testLevenshteinAtMostTwoIsOne = { + expr = strings.levenshteinAtMost 2 "abcdef" "abddef"; + expected = true; + }; + + testLevenshteinAtMostTwoDiff0False = { + expr = strings.levenshteinAtMost 2 "abcdef" "aczyef"; + expected = false; + }; + + testLevenshteinAtMostTwoDiff0Outer = { + expr = strings.levenshteinAtMost 2 "abcdef" "zbcdez"; + expected = true; + }; + + testLevenshteinAtMostTwoDiff0DelLeft = { + expr = strings.levenshteinAtMost 2 "abcdef" "bcdefz"; + expected = true; + }; + + testLevenshteinAtMostTwoDiff0DelRight = { + expr = strings.levenshteinAtMost 2 "abcdef" "zabcde"; + expected = true; + }; + + testLevenshteinAtMostTwoDiff1False = { + expr = strings.levenshteinAtMost 2 "abcdef" "bddez"; + expected = false; + }; + + testLevenshteinAtMostTwoDiff1DelLeft = { + expr = strings.levenshteinAtMost 2 "abcdef" "bcdez"; + expected = true; + }; + + testLevenshteinAtMostTwoDiff1DelRight = { + expr = strings.levenshteinAtMost 2 "abcdef" "zbcde"; + expected = true; + }; + + testLevenshteinAtMostTwoDiff2False = { + expr = strings.levenshteinAtMost 2 "hello" "hxo"; + expected = false; + }; + + testLevenshteinAtMostTwoDiff2True = { + expr = strings.levenshteinAtMost 2 "hello" "heo"; + expected = true; + }; + + testLevenshteinAtMostTwoDiff3 = { + expr = strings.levenshteinAtMost 2 "hello" "ho"; + expected = false; + }; + + testLevenshteinAtMostThreeFalse = { + expr = strings.levenshteinAtMost 3 "hello" "Holla!"; + expected = false; + }; + + testLevenshteinAtMostThreeTrue = { + expr = strings.levenshteinAtMost 3 "hello" "Holla"; + expected = true; + }; + + # lazyDerivation + + testLazyDerivationIsLazyInDerivationForAttrNames = { + expr = attrNames (lazyDerivation { + derivation = throw "not lazy enough"; + }); + # It's ok to add attribute names here when lazyDerivation is improved + # in accordance with its inline comments. + expected = [ "drvPath" "meta" "name" "out" "outPath" "outputName" "outputs" "system" "type" ]; + }; + + testLazyDerivationIsLazyInDerivationForPassthruAttr = { + expr = (lazyDerivation { + derivation = throw "not lazy enough"; + passthru.tests = "whatever is in tests"; + }).tests; + expected = "whatever is in tests"; + }; + + testLazyDerivationIsLazyInDerivationForPassthruAttr2 = { + # passthru.tests is not a special case. It works for any attr. + expr = (lazyDerivation { + derivation = throw "not lazy enough"; + passthru.foo = "whatever is in foo"; + }).foo; + expected = "whatever is in foo"; + }; + + testLazyDerivationIsLazyInDerivationForMeta = { + expr = (lazyDerivation { + derivation = throw "not lazy enough"; + meta = "whatever is in meta"; + }).meta; + expected = "whatever is in meta"; + }; + + testLazyDerivationReturnsDerivationAttrs = let + derivation = { + type = "derivation"; + outputs = ["out"]; + out = "test out"; + outPath = "test outPath"; + outputName = "out"; + drvPath = "test drvPath"; + name = "test name"; + system = "test system"; + meta = "test meta"; + }; + in { + expr = lazyDerivation { inherit derivation; }; + expected = derivation; + }; + + testTypeDescriptionInt = { + expr = (with types; int).description; + expected = "signed integer"; + }; + testTypeDescriptionListOfInt = { + expr = (with types; listOf int).description; + expected = "list of signed integer"; + }; + testTypeDescriptionListOfListOfInt = { + expr = (with types; listOf (listOf int)).description; + expected = "list of list of signed integer"; + }; + testTypeDescriptionListOfEitherStrOrBool = { + expr = (with types; listOf (either str bool)).description; + expected = "list of (string or boolean)"; + }; + testTypeDescriptionEitherListOfStrOrBool = { + expr = (with types; either (listOf bool) str).description; + expected = "(list of boolean) or string"; + }; + testTypeDescriptionEitherStrOrListOfBool = { + expr = (with types; either str (listOf bool)).description; + expected = "string or list of boolean"; + }; + testTypeDescriptionOneOfListOfStrOrBool = { + expr = (with types; oneOf [ (listOf bool) str ]).description; + expected = "(list of boolean) or string"; + }; + testTypeDescriptionOneOfListOfStrOrBoolOrNumber = { + expr = (with types; oneOf [ (listOf bool) str number ]).description; + expected = "(list of boolean) or string or signed integer or floating point number"; + }; + testTypeDescriptionEitherListOfBoolOrEitherStringOrNumber = { + expr = (with types; either (listOf bool) (either str number)).description; + expected = "(list of boolean) or string or signed integer or floating point number"; + }; + testTypeDescriptionEitherEitherListOfBoolOrStringOrNumber = { + expr = (with types; either (either (listOf bool) str) number).description; + expected = "(list of boolean) or string or signed integer or floating point number"; + }; + testTypeDescriptionEitherNullOrBoolOrString = { + expr = (with types; either (nullOr bool) str).description; + expected = "null or boolean or string"; + }; + testTypeDescriptionEitherListOfEitherBoolOrStrOrInt = { + expr = (with types; either (listOf (either bool str)) int).description; + expected = "(list of (boolean or string)) or signed integer"; + }; + testTypeDescriptionEitherIntOrListOrEitherBoolOrStr = { + expr = (with types; either int (listOf (either bool str))).description; + expected = "signed integer or list of (boolean or string)"; + }; +} diff --git a/tests/modules.sh b/tests/modules.sh new file mode 100755 index 000000000..7aebba6b5 --- /dev/null +++ b/tests/modules.sh @@ -0,0 +1,408 @@ +#!/usr/bin/env bash +# +# This script is used to test that the module system is working as expected. +# By default it test the version of nixpkgs which is defined in the NIX_PATH. + +set -o errexit -o noclobber -o nounset -o pipefail +shopt -s failglob inherit_errexit + +# https://stackoverflow.com/a/246128/6605742 +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +cd "$DIR"/modules + +pass=0 +fail=0 + +evalConfig() { + local attr=$1 + shift + local script="import ./default.nix { modules = [ $* ];}" + nix-instantiate --timeout 1 -E "$script" -A "$attr" --eval-only --show-trace --read-write-mode +} + +reportFailure() { + local attr=$1 + shift + local script="import ./default.nix { modules = [ $* ];}" + echo 2>&1 "$ nix-instantiate -E '$script' -A '$attr' --eval-only" + evalConfig "$attr" "$@" || true + ((++fail)) +} + +checkConfigOutput() { + local outputContains=$1 + shift + if evalConfig "$@" 2>/dev/null | grep --silent "$outputContains" ; then + ((++pass)) + else + echo 2>&1 "error: Expected result matching '$outputContains', while evaluating" + reportFailure "$@" + fi +} + +checkConfigError() { + local errorContains=$1 + local err="" + shift + if err="$(evalConfig "$@" 2>&1 >/dev/null)"; then + echo 2>&1 "error: Expected error code, got exit code 0, while evaluating" + reportFailure "$@" + else + if echo "$err" | grep -zP --silent "$errorContains" ; then + ((++pass)) + else + echo 2>&1 "error: Expected error matching '$errorContains', while evaluating" + reportFailure "$@" + fi + fi +} + +# Shorthand meta attribute does not duplicate the config +checkConfigOutput '^"one two"$' config.result ./shorthand-meta.nix + +# Check boolean option. +checkConfigOutput '^false$' config.enable ./declare-enable.nix +checkConfigError 'The option .* does not exist. Definition values:\n\s*- In .*: true' config.enable ./define-enable.nix +checkConfigError 'The option .* does not exist. Definition values:\n\s*- In .*' config.enable ./define-enable-throw.nix +checkConfigError 'while evaluating a definition from `.*/define-enable-abort.nix' config.enable ./define-enable-abort.nix +checkConfigError 'while evaluating the error message for definitions for .enable., which is an option that does not exist' config.enable ./define-enable-abort.nix + +checkConfigOutput '^1$' config.bare-submodule.nested ./declare-bare-submodule.nix ./declare-bare-submodule-nested-option.nix +checkConfigOutput '^2$' config.bare-submodule.deep ./declare-bare-submodule.nix ./declare-bare-submodule-deep-option.nix +checkConfigOutput '^42$' config.bare-submodule.nested ./declare-bare-submodule.nix ./declare-bare-submodule-nested-option.nix ./declare-bare-submodule-deep-option.nix ./define-bare-submodule-values.nix +checkConfigOutput '^420$' config.bare-submodule.deep ./declare-bare-submodule.nix ./declare-bare-submodule-nested-option.nix ./declare-bare-submodule-deep-option.nix ./define-bare-submodule-values.nix +checkConfigOutput '^2$' config.bare-submodule.deep ./declare-bare-submodule.nix ./declare-bare-submodule-deep-option.nix ./define-shorthandOnlyDefinesConfig-true.nix +checkConfigError 'The option .bare-submodule.deep. in .*/declare-bare-submodule-deep-option.nix. is already declared in .*/declare-bare-submodule-deep-option-duplicate.nix' config.bare-submodule.deep ./declare-bare-submodule.nix ./declare-bare-submodule-deep-option.nix ./declare-bare-submodule-deep-option-duplicate.nix + +# Check integer types. +# unsigned +checkConfigOutput '^42$' config.value ./declare-int-unsigned-value.nix ./define-value-int-positive.nix +checkConfigError 'A definition for option .* is not of type.*unsigned integer.*. Definition values:\n\s*- In .*: -23' config.value ./declare-int-unsigned-value.nix ./define-value-int-negative.nix +# positive +checkConfigError 'A definition for option .* is not of type.*positive integer.*. Definition values:\n\s*- In .*: 0' config.value ./declare-int-positive-value.nix ./define-value-int-zero.nix +# between +checkConfigOutput '^42$' config.value ./declare-int-between-value.nix ./define-value-int-positive.nix +checkConfigError 'A definition for option .* is not of type.*between.*-21 and 43.*inclusive.*. Definition values:\n\s*- In .*: -23' config.value ./declare-int-between-value.nix ./define-value-int-negative.nix + +# Check either types +# types.either +checkConfigOutput '^42$' config.value ./declare-either.nix ./define-value-int-positive.nix +checkConfigOutput '^"24"$' config.value ./declare-either.nix ./define-value-string.nix +# types.oneOf +checkConfigOutput '^42$' config.value ./declare-oneOf.nix ./define-value-int-positive.nix +checkConfigOutput '^\[ \]$' config.value ./declare-oneOf.nix ./define-value-list.nix +checkConfigOutput '^"24"$' config.value ./declare-oneOf.nix ./define-value-string.nix + +# Check mkForce without submodules. +set -- config.enable ./declare-enable.nix ./define-enable.nix +checkConfigOutput '^true$' "$@" +checkConfigOutput '^false$' "$@" ./define-force-enable.nix +checkConfigOutput '^false$' "$@" ./define-enable-force.nix + +# Check mkForce with option and submodules. +checkConfigError 'attribute .*foo.* .* not found' config.attrsOfSub.foo.enable ./declare-attrsOfSub-any-enable.nix +checkConfigOutput '^false$' config.attrsOfSub.foo.enable ./declare-attrsOfSub-any-enable.nix ./define-attrsOfSub-foo.nix +set -- config.attrsOfSub.foo.enable ./declare-attrsOfSub-any-enable.nix ./define-attrsOfSub-foo-enable.nix +checkConfigOutput '^true$' "$@" +checkConfigOutput '^false$' "$@" ./define-force-attrsOfSub-foo-enable.nix +checkConfigOutput '^false$' "$@" ./define-attrsOfSub-force-foo-enable.nix +checkConfigOutput '^false$' "$@" ./define-attrsOfSub-foo-force-enable.nix +checkConfigOutput '^false$' "$@" ./define-attrsOfSub-foo-enable-force.nix + +# Check overriding effect of mkForce on submodule definitions. +checkConfigError 'attribute .*bar.* .* not found' config.attrsOfSub.bar.enable ./declare-attrsOfSub-any-enable.nix ./define-attrsOfSub-foo.nix +checkConfigOutput '^false$' config.attrsOfSub.bar.enable ./declare-attrsOfSub-any-enable.nix ./define-attrsOfSub-foo.nix ./define-attrsOfSub-bar.nix +set -- config.attrsOfSub.bar.enable ./declare-attrsOfSub-any-enable.nix ./define-attrsOfSub-foo.nix ./define-attrsOfSub-bar-enable.nix +checkConfigOutput '^true$' "$@" +checkConfigError 'attribute .*bar.* .* not found' "$@" ./define-force-attrsOfSub-foo-enable.nix +checkConfigError 'attribute .*bar.* .* not found' "$@" ./define-attrsOfSub-force-foo-enable.nix +checkConfigOutput '^true$' "$@" ./define-attrsOfSub-foo-force-enable.nix +checkConfigOutput '^true$' "$@" ./define-attrsOfSub-foo-enable-force.nix + +# Check mkIf with submodules. +checkConfigError 'attribute .*foo.* .* not found' config.attrsOfSub.foo.enable ./declare-enable.nix ./declare-attrsOfSub-any-enable.nix +set -- config.attrsOfSub.foo.enable ./declare-enable.nix ./declare-attrsOfSub-any-enable.nix +checkConfigError 'attribute .*foo.* .* not found' "$@" ./define-if-attrsOfSub-foo-enable.nix +checkConfigError 'attribute .*foo.* .* not found' "$@" ./define-attrsOfSub-if-foo-enable.nix +checkConfigError 'attribute .*foo.* .* not found' "$@" ./define-attrsOfSub-foo-if-enable.nix +checkConfigOutput '^false$' "$@" ./define-attrsOfSub-foo-enable-if.nix +checkConfigOutput '^true$' "$@" ./define-enable.nix ./define-if-attrsOfSub-foo-enable.nix +checkConfigOutput '^true$' "$@" ./define-enable.nix ./define-attrsOfSub-if-foo-enable.nix +checkConfigOutput '^true$' "$@" ./define-enable.nix ./define-attrsOfSub-foo-if-enable.nix +checkConfigOutput '^true$' "$@" ./define-enable.nix ./define-attrsOfSub-foo-enable-if.nix + +# Check disabledModules with config definitions and option declarations. +set -- config.enable ./define-enable.nix ./declare-enable.nix +checkConfigOutput '^true$' "$@" +checkConfigOutput '^false$' "$@" ./disable-define-enable.nix +checkConfigOutput '^false$' "$@" ./disable-define-enable-string-path.nix +checkConfigError "The option .*enable.* does not exist. Definition values:\n\s*- In .*: true" "$@" ./disable-declare-enable.nix +checkConfigError "attribute .*enable.* in selection path .*config.enable.* not found" "$@" ./disable-define-enable.nix ./disable-declare-enable.nix +checkConfigError "attribute .*enable.* in selection path .*config.enable.* not found" "$@" ./disable-enable-modules.nix + +checkConfigOutput '^true$' 'config.positive.enable' ./disable-module-with-key.nix +checkConfigOutput '^false$' 'config.negative.enable' ./disable-module-with-key.nix +checkConfigError 'Module ..*disable-module-bad-key.nix. contains a disabledModules item that is an attribute set, presumably a module, that does not have a .key. attribute. .*' 'config.enable' ./disable-module-bad-key.nix + +# Not sure if we want to keep supporting module keys that aren't strings, paths or v?key, but we shouldn't remove support accidentally. +checkConfigOutput '^true$' 'config.positive.enable' ./disable-module-with-toString-key.nix +checkConfigOutput '^false$' 'config.negative.enable' ./disable-module-with-toString-key.nix + +# Check _module.args. +set -- config.enable ./declare-enable.nix ./define-enable-with-custom-arg.nix +checkConfigError 'while evaluating the module argument .*custom.* in .*define-enable-with-custom-arg.nix.*:' "$@" +checkConfigOutput '^true$' "$@" ./define-_module-args-custom.nix + +# Check that using _module.args on imports cause infinite recursions, with +# the proper error context. +set -- "$@" ./define-_module-args-custom.nix ./import-custom-arg.nix +checkConfigError 'while evaluating the module argument .*custom.* in .*import-custom-arg.nix.*:' "$@" +checkConfigError 'infinite recursion encountered' "$@" + +# Check _module.check. +set -- config.enable ./declare-enable.nix ./define-enable.nix ./define-attrsOfSub-foo.nix +checkConfigError 'The option .* does not exist. Definition values:\n\s*- In .*' "$@" +checkConfigOutput '^true$' "$@" ./define-module-check.nix + +# Check coerced value. +set -- +checkConfigOutput '^"42"$' config.value ./declare-coerced-value.nix +checkConfigOutput '^"24"$' config.value ./declare-coerced-value.nix ./define-value-string.nix +checkConfigError 'A definition for option .* is not.*string or signed integer convertible to it.*. Definition values:\n\s*- In .*: \[ \]' config.value ./declare-coerced-value.nix ./define-value-list.nix + +# Check coerced value with unsound coercion +checkConfigOutput '^12$' config.value ./declare-coerced-value-unsound.nix +checkConfigError 'A definition for option .* is not of type .*. Definition values:\n\s*- In .*: "1000"' config.value ./declare-coerced-value-unsound.nix ./define-value-string-bigint.nix +checkConfigError 'toInt: Could not convert .* to int' config.value ./declare-coerced-value-unsound.nix ./define-value-string-arbitrary.nix + +# Check mkAliasOptionModule. +checkConfigOutput '^true$' config.enable ./alias-with-priority.nix +checkConfigOutput '^true$' config.enableAlias ./alias-with-priority.nix +checkConfigOutput '^false$' config.enable ./alias-with-priority-can-override.nix +checkConfigOutput '^false$' config.enableAlias ./alias-with-priority-can-override.nix + +# Check mkPackageOption +checkConfigOutput '^"hello"$' config.package.pname ./declare-mkPackageOption.nix +checkConfigError 'The option .undefinedPackage. is used but not defined' config.undefinedPackage ./declare-mkPackageOption.nix +checkConfigOutput '^null$' config.nullablePackage ./declare-mkPackageOption.nix + +# submoduleWith + +## specialArgs should work +checkConfigOutput '^"foo"$' config.submodule.foo ./declare-submoduleWith-special.nix + +## shorthandOnlyDefines config behaves as expected +checkConfigOutput '^true$' config.submodule.config ./declare-submoduleWith-shorthand.nix ./define-submoduleWith-shorthand.nix +checkConfigError 'is not of type `boolean' config.submodule.config ./declare-submoduleWith-shorthand.nix ./define-submoduleWith-noshorthand.nix +checkConfigError "You're trying to define a value of type \`bool'\n\s*rather than an attribute set for the option" config.submodule.config ./declare-submoduleWith-noshorthand.nix ./define-submoduleWith-shorthand.nix +checkConfigOutput '^true$' config.submodule.config ./declare-submoduleWith-noshorthand.nix ./define-submoduleWith-noshorthand.nix + +## submoduleWith should merge all modules in one swoop +checkConfigOutput '^true$' config.submodule.inner ./declare-submoduleWith-modules.nix +checkConfigOutput '^true$' config.submodule.outer ./declare-submoduleWith-modules.nix +# Should also be able to evaluate the type name (which evaluates freeformType, +# which evaluates all the modules defined by the type) +checkConfigOutput '^"submodule"$' options.submodule.type.description ./declare-submoduleWith-modules.nix + +## submodules can be declared using (evalModules {...}).type +checkConfigOutput '^true$' config.submodule.inner ./declare-submodule-via-evalModules.nix +checkConfigOutput '^true$' config.submodule.outer ./declare-submodule-via-evalModules.nix +# Should also be able to evaluate the type name (which evaluates freeformType, +# which evaluates all the modules defined by the type) +checkConfigOutput '^"submodule"$' options.submodule.type.description ./declare-submodule-via-evalModules.nix + +## Paths should be allowed as values and work as expected +checkConfigOutput '^true$' config.submodule.enable ./declare-submoduleWith-path.nix + +## deferredModule +# default module is merged into nodes.foo +checkConfigOutput '"beta"' config.nodes.foo.settingsDict.c ./deferred-module.nix +# errors from the default module are reported with accurate location +checkConfigError 'In `the-file-that-contains-the-bad-config.nix, via option default'\'': "bogus"' config.nodes.foo.bottom ./deferred-module.nix +checkConfigError '.*lib/tests/modules/deferred-module-error.nix, via option deferred [(]:anon-1:anon-1:anon-1[)] does not look like a module.' config.result ./deferred-module-error.nix + +# Check the file location information is propagated into submodules +checkConfigOutput the-file.nix config.submodule.internalFiles.0 ./submoduleFiles.nix + + +# Check that disabledModules works recursively and correctly +checkConfigOutput '^true$' config.enable ./disable-recursive/main.nix +checkConfigOutput '^true$' config.enable ./disable-recursive/{main.nix,disable-foo.nix} +checkConfigOutput '^true$' config.enable ./disable-recursive/{main.nix,disable-bar.nix} +checkConfigError 'The option .* does not exist. Definition values:\n\s*- In .*: true' config.enable ./disable-recursive/{main.nix,disable-foo.nix,disable-bar.nix} + +# Check that imports can depend on derivations +checkConfigOutput '^true$' config.enable ./import-from-store.nix + +# Check that configs can be conditional on option existence +checkConfigOutput '^true$' config.enable ./define-option-dependently.nix ./declare-enable.nix ./declare-int-positive-value.nix +checkConfigOutput '^360$' config.value ./define-option-dependently.nix ./declare-enable.nix ./declare-int-positive-value.nix +checkConfigOutput '^7$' config.value ./define-option-dependently.nix ./declare-int-positive-value.nix +checkConfigOutput '^true$' config.set.enable ./define-option-dependently-nested.nix ./declare-enable-nested.nix ./declare-int-positive-value-nested.nix +checkConfigOutput '^360$' config.set.value ./define-option-dependently-nested.nix ./declare-enable-nested.nix ./declare-int-positive-value-nested.nix +checkConfigOutput '^7$' config.set.value ./define-option-dependently-nested.nix ./declare-int-positive-value-nested.nix + +# Check attrsOf and lazyAttrsOf. Only lazyAttrsOf should be lazy, and only +# attrsOf should work with conditional definitions +# In addition, lazyAttrsOf should honor an options emptyValue +checkConfigError "is not lazy" config.isLazy ./declare-attrsOf.nix ./attrsOf-lazy-check.nix +checkConfigOutput '^true$' config.isLazy ./declare-lazyAttrsOf.nix ./attrsOf-lazy-check.nix +checkConfigOutput '^true$' config.conditionalWorks ./declare-attrsOf.nix ./attrsOf-conditional-check.nix +checkConfigOutput '^false$' config.conditionalWorks ./declare-lazyAttrsOf.nix ./attrsOf-conditional-check.nix +checkConfigOutput '^"empty"$' config.value.foo ./declare-lazyAttrsOf.nix ./attrsOf-conditional-check.nix + + +# Even with multiple assignments, a type error should be thrown if any of them aren't valid +checkConfigError 'A definition for option .* is not of type .*' \ + config.value ./declare-int-unsigned-value.nix ./define-value-list.nix ./define-value-int-positive.nix + +## Freeform modules +# Assigning without a declared option should work +checkConfigOutput '^"24"$' config.value ./freeform-attrsOf.nix ./define-value-string.nix +# Shorthand modules interpret `meta` and `class` as config items +checkConfigOutput '^true$' options._module.args.value.result ./freeform-attrsOf.nix ./define-freeform-keywords-shorthand.nix +# No freeform assignments shouldn't make it error +checkConfigOutput '^{ }$' config ./freeform-attrsOf.nix +# but only if the type matches +checkConfigError 'A definition for option .* is not of type .*' config.value ./freeform-attrsOf.nix ./define-value-list.nix +# and properties should be applied +checkConfigOutput '^"yes"$' config.value ./freeform-attrsOf.nix ./define-value-string-properties.nix +# Options should still be declarable, and be able to have a type that doesn't match the freeform type +checkConfigOutput '^false$' config.enable ./freeform-attrsOf.nix ./define-value-string.nix ./declare-enable.nix +checkConfigOutput '^"24"$' config.value ./freeform-attrsOf.nix ./define-value-string.nix ./declare-enable.nix +# and this should work too with nested values +checkConfigOutput '^false$' config.nest.foo ./freeform-attrsOf.nix ./freeform-nested.nix +checkConfigOutput '^"bar"$' config.nest.bar ./freeform-attrsOf.nix ./freeform-nested.nix +# Check whether a declared option can depend on an freeform-typed one +checkConfigOutput '^null$' config.foo ./freeform-attrsOf.nix ./freeform-str-dep-unstr.nix +checkConfigOutput '^"24"$' config.foo ./freeform-attrsOf.nix ./freeform-str-dep-unstr.nix ./define-value-string.nix +# Check whether an freeform-typed value can depend on a declared option, this can only work with lazyAttrsOf +checkConfigError 'infinite recursion encountered' config.foo ./freeform-attrsOf.nix ./freeform-unstr-dep-str.nix +checkConfigError 'The option .* is used but not defined' config.foo ./freeform-lazyAttrsOf.nix ./freeform-unstr-dep-str.nix +checkConfigOutput '^"24"$' config.foo ./freeform-lazyAttrsOf.nix ./freeform-unstr-dep-str.nix ./define-value-string.nix +# submodules in freeformTypes should have their locations annotated +checkConfigOutput '/freeform-submodules.nix"$' config.fooDeclarations.0 ./freeform-submodules.nix +# freeformTypes can get merged using `types.type`, including submodules +checkConfigOutput '^10$' config.free.xxx.foo ./freeform-submodules.nix +checkConfigOutput '^10$' config.free.yyy.bar ./freeform-submodules.nix + +## types.anything +# Check that attribute sets are merged recursively +checkConfigOutput '^null$' config.value.foo ./types-anything/nested-attrs.nix +checkConfigOutput '^null$' config.value.l1.foo ./types-anything/nested-attrs.nix +checkConfigOutput '^null$' config.value.l1.l2.foo ./types-anything/nested-attrs.nix +checkConfigOutput '^null$' config.value.l1.l2.l3.foo ./types-anything/nested-attrs.nix +# Attribute sets that are coercible to strings shouldn't be recursed into +checkConfigOutput '^"foo"$' config.value.outPath ./types-anything/attrs-coercible.nix +# Multiple lists aren't concatenated together +checkConfigError 'The option .* has conflicting definitions' config.value ./types-anything/lists.nix +# Check that all equalizable atoms can be used as long as all definitions are equal +checkConfigOutput '^0$' config.value.int ./types-anything/equal-atoms.nix +checkConfigOutput '^false$' config.value.bool ./types-anything/equal-atoms.nix +checkConfigOutput '^""$' config.value.string ./types-anything/equal-atoms.nix +checkConfigOutput '^/$' config.value.path ./types-anything/equal-atoms.nix +checkConfigOutput '^null$' config.value.null ./types-anything/equal-atoms.nix +checkConfigOutput '^0.1$' config.value.float ./types-anything/equal-atoms.nix +# Functions can't be merged together +checkConfigError "The option .value.multiple-lambdas.. has conflicting option types" config.applied.multiple-lambdas ./types-anything/functions.nix +checkConfigOutput '^$' config.value.single-lambda ./types-anything/functions.nix +checkConfigOutput '^null$' config.applied.merging-lambdas.x ./types-anything/functions.nix +checkConfigOutput '^null$' config.applied.merging-lambdas.y ./types-anything/functions.nix +# Check that all mk* modifiers are applied +checkConfigError 'attribute .* not found' config.value.mkiffalse ./types-anything/mk-mods.nix +checkConfigOutput '^{ }$' config.value.mkiftrue ./types-anything/mk-mods.nix +checkConfigOutput '^1$' config.value.mkdefault ./types-anything/mk-mods.nix +checkConfigOutput '^{ }$' config.value.mkmerge ./types-anything/mk-mods.nix +checkConfigOutput '^true$' config.value.mkbefore ./types-anything/mk-mods.nix +checkConfigOutput '^1$' config.value.nested.foo ./types-anything/mk-mods.nix +checkConfigOutput '^"baz"$' config.value.nested.bar.baz ./types-anything/mk-mods.nix + +## types.functionTo +checkConfigOutput '^"input is input"$' config.result ./functionTo/trivial.nix +checkConfigOutput '^"a b"$' config.result ./functionTo/merging-list.nix +checkConfigError 'A definition for option .fun.. is not of type .string.. Definition values:\n\s*- In .*wrong-type.nix' config.result ./functionTo/wrong-type.nix +checkConfigOutput '^"b a"$' config.result ./functionTo/list-order.nix +checkConfigOutput '^"a c"$' config.result ./functionTo/merging-attrs.nix +checkConfigOutput '^"a bee"$' config.result ./functionTo/submodule-options.nix +checkConfigOutput '^"fun..a fun..b"$' config.optionsResult ./functionTo/submodule-options.nix + +# moduleType +checkConfigOutput '^"a b"$' config.resultFoo ./declare-variants.nix ./define-variant.nix +checkConfigOutput '^"a b y z"$' config.resultFooBar ./declare-variants.nix ./define-variant.nix +checkConfigOutput '^"a b c"$' config.resultFooFoo ./declare-variants.nix ./define-variant.nix + +## emptyValue's +checkConfigOutput "[ ]" config.list.a ./emptyValues.nix +checkConfigOutput "{ }" config.attrs.a ./emptyValues.nix +checkConfigOutput "null" config.null.a ./emptyValues.nix +checkConfigOutput "{ }" config.submodule.a ./emptyValues.nix +# These types don't have empty values +checkConfigError 'The option .int.a. is used but not defined' config.int.a ./emptyValues.nix +checkConfigError 'The option .nonEmptyList.a. is used but not defined' config.nonEmptyList.a ./emptyValues.nix + +## types.raw +checkConfigOutput "{ foo = ; }" config.unprocessedNesting ./raw.nix +checkConfigOutput "10" config.processedToplevel ./raw.nix +checkConfigError "The option .multiple. is defined multiple times" config.multiple ./raw.nix +checkConfigOutput "bar" config.priorities ./raw.nix + +## Option collision +checkConfigError \ + 'The option .set. in module .*/declare-set.nix. would be a parent of the following options, but its type .attribute set of signed integer. does not support nested options.\n\s*- option[(]s[)] with prefix .set.enable. in module .*/declare-enable-nested.nix.' \ + config.set \ + ./declare-set.nix ./declare-enable-nested.nix + +# Test that types.optionType merges types correctly +checkConfigOutput '^10$' config.theOption.int ./optionTypeMerging.nix +checkConfigOutput '^"hello"$' config.theOption.str ./optionTypeMerging.nix + +# Test that types.optionType correctly annotates option locations +checkConfigError 'The option .theOption.nested. in .other.nix. is already declared in .optionTypeFile.nix.' config.theOption.nested ./optionTypeFile.nix + +# Test that types.optionType leaves types untouched as long as they don't need to be merged +checkConfigOutput 'ok' config.freeformItems.foo.bar ./adhoc-freeformType-survives-type-merge.nix + +# Anonymous submodules don't get nixed by import resolution/deduplication +# because of an `extendModules` bug, issue 168767. +checkConfigOutput '^1$' config.sub.specialisation.value ./extendModules-168767-imports.nix + +# Class checks, evalModules +checkConfigOutput '^{ }$' config.ok.config ./class-check.nix +checkConfigOutput '"nixos"' config.ok.class ./class-check.nix +checkConfigError 'The module .*/module-class-is-darwin.nix was imported into nixos instead of darwin.' config.fail.config ./class-check.nix +checkConfigError 'The module foo.nix#darwinModules.default was imported into nixos instead of darwin.' config.fail-anon.config ./class-check.nix + +# Class checks, submoduleWith +checkConfigOutput '^{ }$' config.sub.nixosOk ./class-check.nix +checkConfigError 'The module .*/module-class-is-darwin.nix was imported into nixos instead of darwin.' config.sub.nixosFail.config ./class-check.nix + +# submoduleWith type merge with different class +checkConfigError 'A submoduleWith option is declared multiple times with conflicting class values "darwin" and "nixos".' config.sub.mergeFail.config ./class-check.nix + +# _type check +checkConfigError 'Could not load a value as a module, because it is of type "flake", in file .*/module-imports-_type-check.nix' config.ok.config ./module-imports-_type-check.nix +checkConfigOutput '^true$' "$@" config.enable ./declare-enable.nix ./define-enable-with-top-level-mkIf.nix +checkConfigError 'Could not load a value as a module, because it is of type "configuration", in file .*/import-configuration.nix.*please only import the modules that make up the configuration.*' config ./import-configuration.nix + +# doRename works when `warnings` does not exist. +checkConfigOutput '^1234$' config.c.d.e ./doRename-basic.nix +# doRename adds a warning. +checkConfigOutput '^"The option `a\.b. defined in `.*/doRename-warnings\.nix. has been renamed to `c\.d\.e.\."$' \ + config.result \ + ./doRename-warnings.nix + +# Anonymous modules get deduplicated by key +checkConfigOutput '^"pear"$' config.once.raw ./merge-module-with-key.nix +checkConfigOutput '^"pear\\npear"$' config.twice.raw ./merge-module-with-key.nix + +cat < + # - default + # where all nodes include the default + ({ config, ... }: { + _file = "generic.nix"; + options.nodes = mkOption { + type = lazyAttrsOf (submodule { imports = [ config.default ]; }); + default = {}; + }; + options.default = mkOption { + type = deferredModule; + default = { }; + description = '' + Module that is included in all nodes. + ''; + }; + }) + + { + _file = "default-1.nix"; + default = { config, ... }: { + options.settingsDict = lib.mkOption { type = lazyAttrsOf str; default = {}; }; + options.bottom = lib.mkOption { type = enum []; }; + }; + } + + { + _file = "default-a-is-b.nix"; + default = ./define-settingsDict-a-is-b.nix; + } + + { + _file = "nodes-foo.nix"; + nodes.foo.settingsDict.b = "beta"; + } + + { + _file = "the-file-that-contains-the-bad-config.nix"; + default.bottom = "bogus"; + } + + { + _file = "nodes-foo-c-is-a.nix"; + nodes.foo = { config, ... }: { + settingsDict.c = config.settingsDict.a; + }; + } + + ]; +} diff --git a/tests/modules/define-_module-args-custom.nix b/tests/modules/define-_module-args-custom.nix new file mode 100644 index 000000000..e565fd215 --- /dev/null +++ b/tests/modules/define-_module-args-custom.nix @@ -0,0 +1,7 @@ +{ lib, ... }: + +{ + config = { + _module.args.custom = true; + }; +} diff --git a/tests/modules/define-attrsOfSub-bar-enable.nix b/tests/modules/define-attrsOfSub-bar-enable.nix new file mode 100644 index 000000000..99c55d8b3 --- /dev/null +++ b/tests/modules/define-attrsOfSub-bar-enable.nix @@ -0,0 +1,3 @@ +{ + attrsOfSub.bar.enable = true; +} diff --git a/tests/modules/define-attrsOfSub-bar.nix b/tests/modules/define-attrsOfSub-bar.nix new file mode 100644 index 000000000..2a33068a5 --- /dev/null +++ b/tests/modules/define-attrsOfSub-bar.nix @@ -0,0 +1,3 @@ +{ + attrsOfSub.bar = {}; +} diff --git a/tests/modules/define-attrsOfSub-foo-enable-force.nix b/tests/modules/define-attrsOfSub-foo-enable-force.nix new file mode 100644 index 000000000..c9ee36446 --- /dev/null +++ b/tests/modules/define-attrsOfSub-foo-enable-force.nix @@ -0,0 +1,5 @@ +{ lib, ... }: + +{ + attrsOfSub.foo.enable = lib.mkForce false; +} diff --git a/tests/modules/define-attrsOfSub-foo-enable-if.nix b/tests/modules/define-attrsOfSub-foo-enable-if.nix new file mode 100644 index 000000000..0b3baddb5 --- /dev/null +++ b/tests/modules/define-attrsOfSub-foo-enable-if.nix @@ -0,0 +1,5 @@ +{ config, lib, ... }: + +{ + attrsOfSub.foo.enable = lib.mkIf config.enable true; +} diff --git a/tests/modules/define-attrsOfSub-foo-enable.nix b/tests/modules/define-attrsOfSub-foo-enable.nix new file mode 100644 index 000000000..39cd63cef --- /dev/null +++ b/tests/modules/define-attrsOfSub-foo-enable.nix @@ -0,0 +1,3 @@ +{ + attrsOfSub.foo.enable = true; +} diff --git a/tests/modules/define-attrsOfSub-foo-force-enable.nix b/tests/modules/define-attrsOfSub-foo-force-enable.nix new file mode 100644 index 000000000..009da7c77 --- /dev/null +++ b/tests/modules/define-attrsOfSub-foo-force-enable.nix @@ -0,0 +1,7 @@ +{ lib, ... }: + +{ + attrsOfSub.foo = lib.mkForce { + enable = false; + }; +} diff --git a/tests/modules/define-attrsOfSub-foo-if-enable.nix b/tests/modules/define-attrsOfSub-foo-if-enable.nix new file mode 100644 index 000000000..93702dfa8 --- /dev/null +++ b/tests/modules/define-attrsOfSub-foo-if-enable.nix @@ -0,0 +1,7 @@ +{ config, lib, ... }: + +{ + attrsOfSub.foo = lib.mkIf config.enable { + enable = true; + }; +} diff --git a/tests/modules/define-attrsOfSub-foo.nix b/tests/modules/define-attrsOfSub-foo.nix new file mode 100644 index 000000000..e6bb531de --- /dev/null +++ b/tests/modules/define-attrsOfSub-foo.nix @@ -0,0 +1,3 @@ +{ + attrsOfSub.foo = {}; +} diff --git a/tests/modules/define-attrsOfSub-force-foo-enable.nix b/tests/modules/define-attrsOfSub-force-foo-enable.nix new file mode 100644 index 000000000..5c02dd343 --- /dev/null +++ b/tests/modules/define-attrsOfSub-force-foo-enable.nix @@ -0,0 +1,7 @@ +{ lib, ... }: + +{ + attrsOfSub = lib.mkForce { + foo.enable = false; + }; +} diff --git a/tests/modules/define-attrsOfSub-if-foo-enable.nix b/tests/modules/define-attrsOfSub-if-foo-enable.nix new file mode 100644 index 000000000..a3fe6051d --- /dev/null +++ b/tests/modules/define-attrsOfSub-if-foo-enable.nix @@ -0,0 +1,7 @@ +{ config, lib, ... }: + +{ + attrsOfSub = lib.mkIf config.enable { + foo.enable = true; + }; +} diff --git a/tests/modules/define-bare-submodule-values.nix b/tests/modules/define-bare-submodule-values.nix new file mode 100644 index 000000000..00ede929e --- /dev/null +++ b/tests/modules/define-bare-submodule-values.nix @@ -0,0 +1,4 @@ +{ + bare-submodule.nested = 42; + bare-submodule.deep = 420; +} diff --git a/tests/modules/define-enable-abort.nix b/tests/modules/define-enable-abort.nix new file mode 100644 index 000000000..85b58a567 --- /dev/null +++ b/tests/modules/define-enable-abort.nix @@ -0,0 +1,3 @@ +{ + config.enable = abort "oops"; +} diff --git a/tests/modules/define-enable-force.nix b/tests/modules/define-enable-force.nix new file mode 100644 index 000000000..f4990a328 --- /dev/null +++ b/tests/modules/define-enable-force.nix @@ -0,0 +1,5 @@ +{ lib, ... }: + +{ + enable = lib.mkForce false; +} diff --git a/tests/modules/define-enable-throw.nix b/tests/modules/define-enable-throw.nix new file mode 100644 index 000000000..16a59b781 --- /dev/null +++ b/tests/modules/define-enable-throw.nix @@ -0,0 +1,3 @@ +{ + config.enable = throw "oops"; +} diff --git a/tests/modules/define-enable-with-custom-arg.nix b/tests/modules/define-enable-with-custom-arg.nix new file mode 100644 index 000000000..7da74671d --- /dev/null +++ b/tests/modules/define-enable-with-custom-arg.nix @@ -0,0 +1,7 @@ +{ lib, custom, ... }: + +{ + config = { + enable = custom; + }; +} diff --git a/tests/modules/define-enable-with-top-level-mkIf.nix b/tests/modules/define-enable-with-top-level-mkIf.nix new file mode 100644 index 000000000..4909c16d8 --- /dev/null +++ b/tests/modules/define-enable-with-top-level-mkIf.nix @@ -0,0 +1,5 @@ +{ lib, ... }: +# I think this might occur more realistically in a submodule +{ + imports = [ (lib.mkIf true { enable = true; }) ]; +} diff --git a/tests/modules/define-enable.nix b/tests/modules/define-enable.nix new file mode 100644 index 000000000..7dc26010a --- /dev/null +++ b/tests/modules/define-enable.nix @@ -0,0 +1,3 @@ +{ + enable = true; +} diff --git a/tests/modules/define-force-attrsOfSub-foo-enable.nix b/tests/modules/define-force-attrsOfSub-foo-enable.nix new file mode 100644 index 000000000..dafb2360e --- /dev/null +++ b/tests/modules/define-force-attrsOfSub-foo-enable.nix @@ -0,0 +1,5 @@ +{ lib, ... }: + +lib.mkForce { + attrsOfSub.foo.enable = false; +} diff --git a/tests/modules/define-force-enable.nix b/tests/modules/define-force-enable.nix new file mode 100644 index 000000000..978caa2a8 --- /dev/null +++ b/tests/modules/define-force-enable.nix @@ -0,0 +1,5 @@ +{ lib, ... }: + +lib.mkForce { + enable = false; +} diff --git a/tests/modules/define-freeform-keywords-shorthand.nix b/tests/modules/define-freeform-keywords-shorthand.nix new file mode 100644 index 000000000..8de1ec6a7 --- /dev/null +++ b/tests/modules/define-freeform-keywords-shorthand.nix @@ -0,0 +1,15 @@ +{ config, ... }: { + class = { "just" = "data"; }; + a = "one"; + b = "two"; + meta = "meta"; + + _module.args.result = + let r = builtins.removeAttrs config [ "_module" ]; + in builtins.trace (builtins.deepSeq r r) (r == { + a = "one"; + b = "two"; + class = { "just" = "data"; }; + meta = "meta"; + }); +} diff --git a/tests/modules/define-if-attrsOfSub-foo-enable.nix b/tests/modules/define-if-attrsOfSub-foo-enable.nix new file mode 100644 index 000000000..6a8e32e80 --- /dev/null +++ b/tests/modules/define-if-attrsOfSub-foo-enable.nix @@ -0,0 +1,5 @@ +{ config, lib, ... }: + +lib.mkIf config.enable { + attrsOfSub.foo.enable = true; +} diff --git a/tests/modules/define-module-check.nix b/tests/modules/define-module-check.nix new file mode 100644 index 000000000..5a0707c97 --- /dev/null +++ b/tests/modules/define-module-check.nix @@ -0,0 +1,3 @@ +{ + _module.check = false; +} diff --git a/tests/modules/define-option-dependently-nested.nix b/tests/modules/define-option-dependently-nested.nix new file mode 100644 index 000000000..69ee42555 --- /dev/null +++ b/tests/modules/define-option-dependently-nested.nix @@ -0,0 +1,16 @@ +{ lib, options, ... }: + +# Some modules may be distributed separately and need to adapt to other modules +# that are distributed and versioned separately. +{ + + # Always defined, but the value depends on the presence of an option. + config.set = { + value = if options ? set.enable then 360 else 7; + } + # Only define if possible. + // lib.optionalAttrs (options ? set.enable) { + enable = true; + }; + +} diff --git a/tests/modules/define-option-dependently.nix b/tests/modules/define-option-dependently.nix new file mode 100644 index 000000000..ad85f99a9 --- /dev/null +++ b/tests/modules/define-option-dependently.nix @@ -0,0 +1,16 @@ +{ lib, options, ... }: + +# Some modules may be distributed separately and need to adapt to other modules +# that are distributed and versioned separately. +{ + + # Always defined, but the value depends on the presence of an option. + config = { + value = if options ? enable then 360 else 7; + } + # Only define if possible. + // lib.optionalAttrs (options ? enable) { + enable = true; + }; + +} diff --git a/tests/modules/define-settingsDict-a-is-b.nix b/tests/modules/define-settingsDict-a-is-b.nix new file mode 100644 index 000000000..42363f45f --- /dev/null +++ b/tests/modules/define-settingsDict-a-is-b.nix @@ -0,0 +1,3 @@ +{ config, ... }: { + settingsDict.a = config.settingsDict.b; +} diff --git a/tests/modules/define-shorthandOnlyDefinesConfig-true.nix b/tests/modules/define-shorthandOnlyDefinesConfig-true.nix new file mode 100644 index 000000000..bd3a73dce --- /dev/null +++ b/tests/modules/define-shorthandOnlyDefinesConfig-true.nix @@ -0,0 +1 @@ +{ shorthandOnlyDefinesConfig = true; } diff --git a/tests/modules/define-submoduleWith-noshorthand.nix b/tests/modules/define-submoduleWith-noshorthand.nix new file mode 100644 index 000000000..35e1607b6 --- /dev/null +++ b/tests/modules/define-submoduleWith-noshorthand.nix @@ -0,0 +1,3 @@ +{ + submodule.config.config = true; +} diff --git a/tests/modules/define-submoduleWith-shorthand.nix b/tests/modules/define-submoduleWith-shorthand.nix new file mode 100644 index 000000000..17df248db --- /dev/null +++ b/tests/modules/define-submoduleWith-shorthand.nix @@ -0,0 +1,3 @@ +{ + submodule.config = true; +} diff --git a/tests/modules/define-value-int-negative.nix b/tests/modules/define-value-int-negative.nix new file mode 100644 index 000000000..a04122298 --- /dev/null +++ b/tests/modules/define-value-int-negative.nix @@ -0,0 +1,3 @@ +{ + value = -23; +} diff --git a/tests/modules/define-value-int-positive.nix b/tests/modules/define-value-int-positive.nix new file mode 100644 index 000000000..5803de172 --- /dev/null +++ b/tests/modules/define-value-int-positive.nix @@ -0,0 +1,3 @@ +{ + value = 42; +} diff --git a/tests/modules/define-value-int-zero.nix b/tests/modules/define-value-int-zero.nix new file mode 100644 index 000000000..68bb9f415 --- /dev/null +++ b/tests/modules/define-value-int-zero.nix @@ -0,0 +1,3 @@ +{ + value = 0; +} diff --git a/tests/modules/define-value-list.nix b/tests/modules/define-value-list.nix new file mode 100644 index 000000000..4831c1cc0 --- /dev/null +++ b/tests/modules/define-value-list.nix @@ -0,0 +1,3 @@ +{ + value = []; +} diff --git a/tests/modules/define-value-string-arbitrary.nix b/tests/modules/define-value-string-arbitrary.nix new file mode 100644 index 000000000..8e3abaf53 --- /dev/null +++ b/tests/modules/define-value-string-arbitrary.nix @@ -0,0 +1,3 @@ +{ + value = "foobar"; +} diff --git a/tests/modules/define-value-string-bigint.nix b/tests/modules/define-value-string-bigint.nix new file mode 100644 index 000000000..f27e31985 --- /dev/null +++ b/tests/modules/define-value-string-bigint.nix @@ -0,0 +1,3 @@ +{ + value = "1000"; +} diff --git a/tests/modules/define-value-string-properties.nix b/tests/modules/define-value-string-properties.nix new file mode 100644 index 000000000..972304c01 --- /dev/null +++ b/tests/modules/define-value-string-properties.nix @@ -0,0 +1,12 @@ +{ lib, ... }: { + + imports = [{ + value = lib.mkDefault "def"; + }]; + + value = lib.mkMerge [ + (lib.mkIf false "nope") + "yes" + ]; + +} diff --git a/tests/modules/define-value-string.nix b/tests/modules/define-value-string.nix new file mode 100644 index 000000000..e7a166965 --- /dev/null +++ b/tests/modules/define-value-string.nix @@ -0,0 +1,3 @@ +{ + value = "24"; +} diff --git a/tests/modules/define-variant.nix b/tests/modules/define-variant.nix new file mode 100644 index 000000000..423cb0e37 --- /dev/null +++ b/tests/modules/define-variant.nix @@ -0,0 +1,22 @@ +{ config, lib, ... }: +let inherit (lib) types mkOption attrNames; +in +{ + options = { + attrs = mkOption { type = types.attrsOf lib.types.int; }; + result = mkOption { }; + resultFoo = mkOption { }; + resultFooBar = mkOption { }; + resultFooFoo = mkOption { }; + }; + config = { + attrs.a = 1; + variants.foo.attrs.b = 1; + variants.bar.attrs.y = 1; + variants.foo.variants.bar.attrs.z = 1; + variants.foo.variants.foo.attrs.c = 3; + resultFoo = lib.concatMapStringsSep " " toString (attrNames config.variants.foo.attrs); + resultFooBar = lib.concatMapStringsSep " " toString (attrNames config.variants.foo.variants.bar.attrs); + resultFooFoo = lib.concatMapStringsSep " " toString (attrNames config.variants.foo.variants.foo.attrs); + }; +} diff --git a/tests/modules/disable-declare-enable.nix b/tests/modules/disable-declare-enable.nix new file mode 100644 index 000000000..a373ee7e5 --- /dev/null +++ b/tests/modules/disable-declare-enable.nix @@ -0,0 +1,5 @@ +{ lib, ... }: + +{ + disabledModules = [ ./declare-enable.nix ]; +} diff --git a/tests/modules/disable-define-enable-string-path.nix b/tests/modules/disable-define-enable-string-path.nix new file mode 100644 index 000000000..6429a6d63 --- /dev/null +++ b/tests/modules/disable-define-enable-string-path.nix @@ -0,0 +1,5 @@ +{ lib, ... }: + +{ + disabledModules = [ (toString ./define-enable.nix) ]; +} diff --git a/tests/modules/disable-define-enable.nix b/tests/modules/disable-define-enable.nix new file mode 100644 index 000000000..0d84a7c3c --- /dev/null +++ b/tests/modules/disable-define-enable.nix @@ -0,0 +1,5 @@ +{ lib, ... }: + +{ + disabledModules = [ ./define-enable.nix ]; +} diff --git a/tests/modules/disable-enable-modules.nix b/tests/modules/disable-enable-modules.nix new file mode 100644 index 000000000..c325f4e07 --- /dev/null +++ b/tests/modules/disable-enable-modules.nix @@ -0,0 +1,5 @@ +{ lib, ... }: + +{ + disabledModules = [ "define-enable.nix" "declare-enable.nix" ]; +} diff --git a/tests/modules/disable-module-bad-key.nix b/tests/modules/disable-module-bad-key.nix new file mode 100644 index 000000000..f50d06f2f --- /dev/null +++ b/tests/modules/disable-module-bad-key.nix @@ -0,0 +1,16 @@ +{ lib, ... }: +let + inherit (lib) mkOption types; + + moduleWithKey = { config, ... }: { + config = { + enable = true; + }; + }; +in +{ + imports = [ + ./declare-enable.nix + ]; + disabledModules = [ { } ]; +} diff --git a/tests/modules/disable-module-with-key.nix b/tests/modules/disable-module-with-key.nix new file mode 100644 index 000000000..ea2a60aa8 --- /dev/null +++ b/tests/modules/disable-module-with-key.nix @@ -0,0 +1,34 @@ +{ lib, ... }: +let + inherit (lib) mkOption types; + + moduleWithKey = { + key = "disable-module-with-key.nix#moduleWithKey"; + config = { + enable = true; + }; + }; +in +{ + options = { + positive = mkOption { + type = types.submodule { + imports = [ + ./declare-enable.nix + moduleWithKey + ]; + }; + default = {}; + }; + negative = mkOption { + type = types.submodule { + imports = [ + ./declare-enable.nix + moduleWithKey + ]; + disabledModules = [ moduleWithKey ]; + }; + default = {}; + }; + }; +} diff --git a/tests/modules/disable-module-with-toString-key.nix b/tests/modules/disable-module-with-toString-key.nix new file mode 100644 index 000000000..3f8c81904 --- /dev/null +++ b/tests/modules/disable-module-with-toString-key.nix @@ -0,0 +1,34 @@ +{ lib, ... }: +let + inherit (lib) mkOption types; + + moduleWithKey = { + key = 123; + config = { + enable = true; + }; + }; +in +{ + options = { + positive = mkOption { + type = types.submodule { + imports = [ + ./declare-enable.nix + moduleWithKey + ]; + }; + default = {}; + }; + negative = mkOption { + type = types.submodule { + imports = [ + ./declare-enable.nix + moduleWithKey + ]; + disabledModules = [ 123 ]; + }; + default = {}; + }; + }; +} diff --git a/tests/modules/disable-recursive/bar.nix b/tests/modules/disable-recursive/bar.nix new file mode 100644 index 000000000..4d9240a43 --- /dev/null +++ b/tests/modules/disable-recursive/bar.nix @@ -0,0 +1,5 @@ +{ + imports = [ + ../declare-enable.nix + ]; +} diff --git a/tests/modules/disable-recursive/disable-bar.nix b/tests/modules/disable-recursive/disable-bar.nix new file mode 100644 index 000000000..987b2802a --- /dev/null +++ b/tests/modules/disable-recursive/disable-bar.nix @@ -0,0 +1,7 @@ +{ + + disabledModules = [ + ./bar.nix + ]; + +} diff --git a/tests/modules/disable-recursive/disable-foo.nix b/tests/modules/disable-recursive/disable-foo.nix new file mode 100644 index 000000000..5b68a3c46 --- /dev/null +++ b/tests/modules/disable-recursive/disable-foo.nix @@ -0,0 +1,7 @@ +{ + + disabledModules = [ + ./foo.nix + ]; + +} diff --git a/tests/modules/disable-recursive/foo.nix b/tests/modules/disable-recursive/foo.nix new file mode 100644 index 000000000..4d9240a43 --- /dev/null +++ b/tests/modules/disable-recursive/foo.nix @@ -0,0 +1,5 @@ +{ + imports = [ + ../declare-enable.nix + ]; +} diff --git a/tests/modules/disable-recursive/main.nix b/tests/modules/disable-recursive/main.nix new file mode 100644 index 000000000..48a3c6218 --- /dev/null +++ b/tests/modules/disable-recursive/main.nix @@ -0,0 +1,8 @@ +{ + imports = [ + ./foo.nix + ./bar.nix + ]; + + enable = true; +} diff --git a/tests/modules/doRename-basic.nix b/tests/modules/doRename-basic.nix new file mode 100644 index 000000000..9d79fa4f2 --- /dev/null +++ b/tests/modules/doRename-basic.nix @@ -0,0 +1,11 @@ +{ lib, ... }: { + imports = [ + (lib.doRename { from = ["a" "b"]; to = ["c" "d" "e"]; warn = true; use = x: x; visible = true; }) + ]; + options = { + c.d.e = lib.mkOption {}; + }; + config = { + a.b = 1234; + }; +} diff --git a/tests/modules/doRename-warnings.nix b/tests/modules/doRename-warnings.nix new file mode 100644 index 000000000..6f0f1e87e --- /dev/null +++ b/tests/modules/doRename-warnings.nix @@ -0,0 +1,14 @@ +{ lib, config, ... }: { + imports = [ + (lib.doRename { from = ["a" "b"]; to = ["c" "d" "e"]; warn = true; use = x: x; visible = true; }) + ]; + options = { + warnings = lib.mkOption { type = lib.types.listOf lib.types.str; }; + c.d.e = lib.mkOption {}; + result = lib.mkOption {}; + }; + config = { + a.b = 1234; + result = lib.concatStringsSep "%" config.warnings; + }; +} diff --git a/tests/modules/emptyValues.nix b/tests/modules/emptyValues.nix new file mode 100644 index 000000000..77825de32 --- /dev/null +++ b/tests/modules/emptyValues.nix @@ -0,0 +1,36 @@ +{ lib, ... }: +let + inherit (lib) types; +in { + + options = { + int = lib.mkOption { + type = types.lazyAttrsOf types.int; + }; + list = lib.mkOption { + type = types.lazyAttrsOf (types.listOf types.int); + }; + nonEmptyList = lib.mkOption { + type = types.lazyAttrsOf (types.nonEmptyListOf types.int); + }; + attrs = lib.mkOption { + type = types.lazyAttrsOf (types.attrsOf types.int); + }; + null = lib.mkOption { + type = types.lazyAttrsOf (types.nullOr types.int); + }; + submodule = lib.mkOption { + type = types.lazyAttrsOf (types.submodule {}); + }; + }; + + config = { + int.a = lib.mkIf false null; + list.a = lib.mkIf false null; + nonEmptyList.a = lib.mkIf false null; + attrs.a = lib.mkIf false null; + null.a = lib.mkIf false null; + submodule.a = lib.mkIf false null; + }; + +} diff --git a/tests/modules/extendModules-168767-imports.nix b/tests/modules/extendModules-168767-imports.nix new file mode 100644 index 000000000..489e6b5a5 --- /dev/null +++ b/tests/modules/extendModules-168767-imports.nix @@ -0,0 +1,41 @@ +{ lib +, extendModules +, ... +}: +with lib; +{ + imports = [ + + { + options.sub = mkOption { + default = { }; + type = types.submodule ( + { config + , extendModules + , ... + }: + { + options.value = mkOption { + type = types.int; + }; + + options.specialisation = mkOption { + default = { }; + inherit + (extendModules { + modules = [{ + specialisation = mkOverride 0 { }; + }]; + }) + type; + }; + } + ); + }; + } + + { config.sub.value = 1; } + + + ]; +} diff --git a/tests/modules/freeform-attrsOf.nix b/tests/modules/freeform-attrsOf.nix new file mode 100644 index 000000000..8cc577f38 --- /dev/null +++ b/tests/modules/freeform-attrsOf.nix @@ -0,0 +1,3 @@ +{ lib, ... }: { + freeformType = with lib.types; attrsOf (either str (attrsOf str)); +} diff --git a/tests/modules/freeform-lazyAttrsOf.nix b/tests/modules/freeform-lazyAttrsOf.nix new file mode 100644 index 000000000..36d6c0b13 --- /dev/null +++ b/tests/modules/freeform-lazyAttrsOf.nix @@ -0,0 +1,3 @@ +{ lib, ... }: { + freeformType = with lib.types; lazyAttrsOf (either str (lazyAttrsOf str)); +} diff --git a/tests/modules/freeform-nested.nix b/tests/modules/freeform-nested.nix new file mode 100644 index 000000000..b81fa7f0d --- /dev/null +++ b/tests/modules/freeform-nested.nix @@ -0,0 +1,14 @@ +{ lib, ... }: +let + deathtrapArgs = lib.mapAttrs + (k: _: throw "The module system is too strict, accessing an unused option's ${k} mkOption-attribute.") + (lib.functionArgs lib.mkOption); +in +{ + options.nest.foo = lib.mkOption { + type = lib.types.bool; + default = false; + }; + options.nest.unused = lib.mkOption deathtrapArgs; + config.nest.bar = "bar"; +} diff --git a/tests/modules/freeform-str-dep-unstr.nix b/tests/modules/freeform-str-dep-unstr.nix new file mode 100644 index 000000000..a2dfbc80c --- /dev/null +++ b/tests/modules/freeform-str-dep-unstr.nix @@ -0,0 +1,8 @@ +{ lib, config, ... }: { + options.foo = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + }; + + config.foo = lib.mkIf (config ? value) config.value; +} diff --git a/tests/modules/freeform-submodules.nix b/tests/modules/freeform-submodules.nix new file mode 100644 index 000000000..3910435a7 --- /dev/null +++ b/tests/modules/freeform-submodules.nix @@ -0,0 +1,22 @@ +{ lib, options, ... }: with lib.types; { + + options.fooDeclarations = lib.mkOption { + default = (options.free.type.getSubOptions [])._freeformOptions.foo.declarations; + }; + + options.free = lib.mkOption { + type = submodule { + config._module.freeformType = lib.mkMerge [ + (attrsOf (submodule { + options.foo = lib.mkOption {}; + })) + (attrsOf (submodule { + options.bar = lib.mkOption {}; + })) + ]; + }; + }; + + config.free.xxx.foo = 10; + config.free.yyy.bar = 10; +} diff --git a/tests/modules/freeform-unstr-dep-str.nix b/tests/modules/freeform-unstr-dep-str.nix new file mode 100644 index 000000000..549d89afe --- /dev/null +++ b/tests/modules/freeform-unstr-dep-str.nix @@ -0,0 +1,8 @@ +{ lib, config, ... }: { + options.value = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + }; + + config.foo = lib.mkIf (config.value != null) config.value; +} diff --git a/tests/modules/functionTo/list-order.nix b/tests/modules/functionTo/list-order.nix new file mode 100644 index 000000000..77a1a43a8 --- /dev/null +++ b/tests/modules/functionTo/list-order.nix @@ -0,0 +1,25 @@ + +{ lib, config, ... }: +let + inherit (lib) types; +in { + options = { + fun = lib.mkOption { + type = types.functionTo (types.listOf types.str); + }; + + result = lib.mkOption { + type = types.str; + default = toString (config.fun { + a = "a"; + b = "b"; + c = "c"; + }); + }; + }; + + config.fun = lib.mkMerge [ + (input: lib.mkAfter [ input.a ]) + (input: [ input.b ]) + ]; +} diff --git a/tests/modules/functionTo/merging-attrs.nix b/tests/modules/functionTo/merging-attrs.nix new file mode 100644 index 000000000..97c015f92 --- /dev/null +++ b/tests/modules/functionTo/merging-attrs.nix @@ -0,0 +1,27 @@ +{ lib, config, ... }: +let + inherit (lib) types; +in { + options = { + fun = lib.mkOption { + type = types.functionTo (types.attrsOf types.str); + }; + + result = lib.mkOption { + type = types.str; + default = toString (lib.attrValues (config.fun { + a = "a"; + b = "b"; + c = "c"; + })); + }; + }; + + config.fun = lib.mkMerge [ + (input: { inherit (input) a; }) + (input: { inherit (input) b; }) + (input: { + b = lib.mkForce input.c; + }) + ]; +} diff --git a/tests/modules/functionTo/merging-list.nix b/tests/modules/functionTo/merging-list.nix new file mode 100644 index 000000000..15fcd2bdc --- /dev/null +++ b/tests/modules/functionTo/merging-list.nix @@ -0,0 +1,24 @@ +{ lib, config, ... }: +let + inherit (lib) types; +in { + options = { + fun = lib.mkOption { + type = types.functionTo (types.listOf types.str); + }; + + result = lib.mkOption { + type = types.str; + default = toString (config.fun { + a = "a"; + b = "b"; + c = "c"; + }); + }; + }; + + config.fun = lib.mkMerge [ + (input: [ input.a ]) + (input: [ input.b ]) + ]; +} diff --git a/tests/modules/functionTo/submodule-options.nix b/tests/modules/functionTo/submodule-options.nix new file mode 100644 index 000000000..b884892ef --- /dev/null +++ b/tests/modules/functionTo/submodule-options.nix @@ -0,0 +1,61 @@ +{ lib, config, options, ... }: +let + inherit (lib) types; +in +{ + imports = [ + + # fun..a + ({ ... }: { + options = { + fun = lib.mkOption { + type = types.functionTo (types.submodule { + options.a = lib.mkOption { default = "a"; }; + }); + }; + }; + }) + + # fun..b + ({ ... }: { + options = { + fun = lib.mkOption { + type = types.functionTo (types.submodule { + options.b = lib.mkOption { default = "b"; }; + }); + }; + }; + }) + ]; + + options = { + result = lib.mkOption + { + type = types.str; + default = lib.concatStringsSep " " (lib.attrValues (config.fun (throw "shouldn't use input param"))); + }; + + optionsResult = lib.mkOption + { + type = types.str; + default = lib.concatStringsSep " " + (lib.concatLists + (lib.mapAttrsToList + (k: v: + if k == "_module" + then [ ] + else [ (lib.showOption v.loc) ] + ) + ( + (options.fun.type.getSubOptions [ "fun" ]) + ) + ) + ); + }; + }; + + config.fun = lib.mkMerge + [ + (input: { b = "bee"; }) + ]; +} diff --git a/tests/modules/functionTo/trivial.nix b/tests/modules/functionTo/trivial.nix new file mode 100644 index 000000000..0962a0cf8 --- /dev/null +++ b/tests/modules/functionTo/trivial.nix @@ -0,0 +1,17 @@ +{ lib, config, ... }: +let + inherit (lib) types; +in { + options = { + fun = lib.mkOption { + type = types.functionTo types.str; + }; + + result = lib.mkOption { + type = types.str; + default = config.fun "input"; + }; + }; + + config.fun = input: "input is ${input}"; +} diff --git a/tests/modules/functionTo/wrong-type.nix b/tests/modules/functionTo/wrong-type.nix new file mode 100644 index 000000000..fd65b7508 --- /dev/null +++ b/tests/modules/functionTo/wrong-type.nix @@ -0,0 +1,18 @@ + +{ lib, config, ... }: +let + inherit (lib) types; +in { + options = { + fun = lib.mkOption { + type = types.functionTo types.str; + }; + + result = lib.mkOption { + type = types.str; + default = config.fun 0; + }; + }; + + config.fun = input: input + 1; +} diff --git a/tests/modules/import-configuration.nix b/tests/modules/import-configuration.nix new file mode 100644 index 000000000..a2a32bbee --- /dev/null +++ b/tests/modules/import-configuration.nix @@ -0,0 +1,12 @@ +{ lib, ... }: +let + myconf = lib.evalModules { modules = [ { } ]; }; +in +{ + imports = [ + # We can't do this. A configuration is not equal to its set of a modules. + # Equating those would lead to a mess, as specialArgs, anonymous modules + # that can't be deduplicated, and possibly more come into play. + myconf + ]; +} diff --git a/tests/modules/import-custom-arg.nix b/tests/modules/import-custom-arg.nix new file mode 100644 index 000000000..3e687b661 --- /dev/null +++ b/tests/modules/import-custom-arg.nix @@ -0,0 +1,6 @@ +{ lib, custom, ... }: + +{ + imports = [] + ++ lib.optional custom ./define-enable-force.nix; +} diff --git a/tests/modules/import-from-store.nix b/tests/modules/import-from-store.nix new file mode 100644 index 000000000..f5af22432 --- /dev/null +++ b/tests/modules/import-from-store.nix @@ -0,0 +1,11 @@ +{ lib, ... }: +{ + + imports = [ + "${builtins.toFile "drv" "{}"}" + ./declare-enable.nix + ./define-enable.nix + ]; + +} + diff --git a/tests/modules/merge-module-with-key.nix b/tests/modules/merge-module-with-key.nix new file mode 100644 index 000000000..21f00e6ef --- /dev/null +++ b/tests/modules/merge-module-with-key.nix @@ -0,0 +1,49 @@ +{ lib, ... }: +let + inherit (lib) mkOption types; + + moduleWithoutKey = { + config = { + raw = "pear"; + }; + }; + + moduleWithKey = { + key = __curPos.file + "#moduleWithKey"; + config = { + raw = "pear"; + }; + }; + + decl = { + options = { + raw = mkOption { + type = types.lines; + }; + }; + }; +in +{ + options = { + once = mkOption { + type = types.submodule { + imports = [ + decl + moduleWithKey + moduleWithKey + ]; + }; + default = {}; + }; + twice = mkOption { + type = types.submodule { + imports = [ + decl + moduleWithoutKey + moduleWithoutKey + ]; + }; + default = {}; + }; + }; +} diff --git a/tests/modules/module-class-is-darwin.nix b/tests/modules/module-class-is-darwin.nix new file mode 100644 index 000000000..bacf45626 --- /dev/null +++ b/tests/modules/module-class-is-darwin.nix @@ -0,0 +1,4 @@ +{ + _class = "darwin"; + config = {}; +} diff --git a/tests/modules/module-class-is-nixos.nix b/tests/modules/module-class-is-nixos.nix new file mode 100644 index 000000000..6d62feeda --- /dev/null +++ b/tests/modules/module-class-is-nixos.nix @@ -0,0 +1,4 @@ +{ + _class = "nixos"; + config = {}; +} diff --git a/tests/modules/module-imports-_type-check.nix b/tests/modules/module-imports-_type-check.nix new file mode 100644 index 000000000..1e29c469d --- /dev/null +++ b/tests/modules/module-imports-_type-check.nix @@ -0,0 +1,3 @@ +{ + imports = [ { _type = "flake"; } ]; +} diff --git a/tests/modules/optionTypeFile.nix b/tests/modules/optionTypeFile.nix new file mode 100644 index 000000000..6015d59a7 --- /dev/null +++ b/tests/modules/optionTypeFile.nix @@ -0,0 +1,28 @@ +{ config, lib, ... }: { + + _file = "optionTypeFile.nix"; + + options.theType = lib.mkOption { + type = lib.types.optionType; + }; + + options.theOption = lib.mkOption { + type = config.theType; + default = {}; + }; + + config.theType = lib.mkMerge [ + (lib.types.submodule { + options.nested = lib.mkOption { + type = lib.types.int; + }; + }) + (lib.types.submodule { + _file = "other.nix"; + options.nested = lib.mkOption { + type = lib.types.str; + }; + }) + ]; + +} diff --git a/tests/modules/optionTypeMerging.nix b/tests/modules/optionTypeMerging.nix new file mode 100644 index 000000000..74a620c46 --- /dev/null +++ b/tests/modules/optionTypeMerging.nix @@ -0,0 +1,27 @@ +{ config, lib, ... }: { + + options.theType = lib.mkOption { + type = lib.types.optionType; + }; + + options.theOption = lib.mkOption { + type = config.theType; + }; + + config.theType = lib.mkMerge [ + (lib.types.submodule { + options.int = lib.mkOption { + type = lib.types.int; + default = 10; + }; + }) + (lib.types.submodule { + options.str = lib.mkOption { + type = lib.types.str; + }; + }) + ]; + + config.theOption.str = "hello"; + +} diff --git a/tests/modules/raw.nix b/tests/modules/raw.nix new file mode 100644 index 000000000..418e671ed --- /dev/null +++ b/tests/modules/raw.nix @@ -0,0 +1,30 @@ +{ lib, ... }: { + + options = { + processedToplevel = lib.mkOption { + type = lib.types.raw; + }; + unprocessedNesting = lib.mkOption { + type = lib.types.raw; + }; + multiple = lib.mkOption { + type = lib.types.raw; + }; + priorities = lib.mkOption { + type = lib.types.raw; + }; + }; + + config = { + processedToplevel = lib.mkIf true 10; + unprocessedNesting.foo = throw "foo"; + multiple = lib.mkMerge [ + "foo" + "foo" + ]; + priorities = lib.mkMerge [ + "foo" + (lib.mkForce "bar") + ]; + }; +} diff --git a/tests/modules/shorthand-meta.nix b/tests/modules/shorthand-meta.nix new file mode 100644 index 000000000..8c9619e18 --- /dev/null +++ b/tests/modules/shorthand-meta.nix @@ -0,0 +1,19 @@ +{ lib, ... }: +let + inherit (lib) types mkOption; +in +{ + imports = [ + ({ config, ... }: { + options = { + meta.foo = mkOption { + type = types.listOf types.str; + }; + result = mkOption { default = lib.concatStringsSep " " config.meta.foo; }; + }; + }) + { + meta.foo = [ "one" "two" ]; + } + ]; +} diff --git a/tests/modules/submoduleFiles.nix b/tests/modules/submoduleFiles.nix new file mode 100644 index 000000000..c0d9b2cef --- /dev/null +++ b/tests/modules/submoduleFiles.nix @@ -0,0 +1,21 @@ +{ lib, ... }: { + options.submodule = lib.mkOption { + default = {}; + type = lib.types.submoduleWith { + modules = [ ({ options, ... }: { + options.value = lib.mkOption {}; + + options.internalFiles = lib.mkOption { + default = options.value.files; + }; + })]; + }; + }; + + imports = [ + { + _file = "the-file.nix"; + submodule.value = 10; + } + ]; +} diff --git a/tests/modules/types-anything/attrs-coercible.nix b/tests/modules/types-anything/attrs-coercible.nix new file mode 100644 index 000000000..085cbd638 --- /dev/null +++ b/tests/modules/types-anything/attrs-coercible.nix @@ -0,0 +1,12 @@ +{ lib, ... }: { + + options.value = lib.mkOption { + type = lib.types.anything; + }; + + config.value = { + outPath = "foo"; + err = throw "err"; + }; + +} diff --git a/tests/modules/types-anything/equal-atoms.nix b/tests/modules/types-anything/equal-atoms.nix new file mode 100644 index 000000000..972711201 --- /dev/null +++ b/tests/modules/types-anything/equal-atoms.nix @@ -0,0 +1,26 @@ +{ lib, ... }: { + + options.value = lib.mkOption { + type = lib.types.anything; + }; + + config = lib.mkMerge [ + { + value.int = 0; + value.bool = false; + value.string = ""; + value.path = /.; + value.null = null; + value.float = 0.1; + } + { + value.int = 0; + value.bool = false; + value.string = ""; + value.path = /.; + value.null = null; + value.float = 0.1; + } + ]; + +} diff --git a/tests/modules/types-anything/functions.nix b/tests/modules/types-anything/functions.nix new file mode 100644 index 000000000..21edd4aff --- /dev/null +++ b/tests/modules/types-anything/functions.nix @@ -0,0 +1,23 @@ +{ lib, config, ... }: { + + options.value = lib.mkOption { + type = lib.types.anything; + }; + + options.applied = lib.mkOption { + default = lib.mapAttrs (name: fun: fun null) config.value; + }; + + config = lib.mkMerge [ + { + value.single-lambda = x: x; + value.multiple-lambdas = x: { inherit x; }; + value.merging-lambdas = x: { inherit x; }; + } + { + value.multiple-lambdas = x: [ x ]; + value.merging-lambdas = y: { inherit y; }; + } + ]; + +} diff --git a/tests/modules/types-anything/lists.nix b/tests/modules/types-anything/lists.nix new file mode 100644 index 000000000..bd846afd3 --- /dev/null +++ b/tests/modules/types-anything/lists.nix @@ -0,0 +1,16 @@ +{ lib, ... }: { + + options.value = lib.mkOption { + type = lib.types.anything; + }; + + config = lib.mkMerge [ + { + value = [ null ]; + } + { + value = [ null ]; + } + ]; + +} diff --git a/tests/modules/types-anything/mk-mods.nix b/tests/modules/types-anything/mk-mods.nix new file mode 100644 index 000000000..f84ad01df --- /dev/null +++ b/tests/modules/types-anything/mk-mods.nix @@ -0,0 +1,44 @@ +{ lib, ... }: { + + options.value = lib.mkOption { + type = lib.types.anything; + }; + + config = lib.mkMerge [ + { + value.mkiffalse = lib.mkIf false {}; + } + { + value.mkiftrue = lib.mkIf true {}; + } + { + value.mkdefault = lib.mkDefault 0; + } + { + value.mkdefault = 1; + } + { + value.mkmerge = lib.mkMerge [ + {} + ]; + } + { + value.mkbefore = lib.mkBefore true; + } + { + value.nested = lib.mkMerge [ + { + foo = lib.mkDefault 0; + bar = lib.mkIf false 0; + } + (lib.mkIf true { + foo = lib.mkIf true (lib.mkForce 1); + bar = { + baz = lib.mkDefault "baz"; + }; + }) + ]; + } + ]; + +} diff --git a/tests/modules/types-anything/nested-attrs.nix b/tests/modules/types-anything/nested-attrs.nix new file mode 100644 index 000000000..e57d33ef8 --- /dev/null +++ b/tests/modules/types-anything/nested-attrs.nix @@ -0,0 +1,22 @@ +{ lib, ... }: { + + options.value = lib.mkOption { + type = lib.types.anything; + }; + + config = lib.mkMerge [ + { + value.foo = null; + } + { + value.l1.foo = null; + } + { + value.l1.l2.foo = null; + } + { + value.l1.l2.l3.foo = null; + } + ]; + +} diff --git a/tests/release.nix b/tests/release.nix new file mode 100644 index 000000000..c3bf58db2 --- /dev/null +++ b/tests/release.nix @@ -0,0 +1,64 @@ +{ # The pkgs used for dependencies for the testing itself + # Don't test properties of pkgs.lib, but rather the lib in the parent directory + pkgs ? import ../.. {} // { lib = throw "pkgs.lib accessed, but the lib tests should use nixpkgs' lib path directly!"; }, + nix ? pkgs.nix, + nixVersions ? [ pkgs.nixVersions.minimum nix pkgs.nixVersions.unstable ], +}: + +let + testWithNix = nix: + pkgs.runCommand "nixpkgs-lib-tests-nix-${nix.version}" { + buildInputs = [ + (import ./check-eval.nix) + (import ./maintainers.nix { + inherit pkgs; + lib = import ../.; + }) + (import ./teams.nix { + inherit pkgs; + lib = import ../.; + }) + (import ../path/tests { + inherit pkgs; + }) + ]; + nativeBuildInputs = [ + nix + ]; + strictDeps = true; + } '' + datadir="${nix}/share" + export TEST_ROOT=$(pwd)/test-tmp + export NIX_BUILD_HOOK= + export NIX_CONF_DIR=$TEST_ROOT/etc + export NIX_LOCALSTATE_DIR=$TEST_ROOT/var + export NIX_LOG_DIR=$TEST_ROOT/var/log/nix + export NIX_STATE_DIR=$TEST_ROOT/var/nix + export NIX_STORE_DIR=$TEST_ROOT/store + export PAGER=cat + cacheDir=$TEST_ROOT/binary-cache + + mkdir -p $NIX_CONF_DIR + echo "experimental-features = nix-command" >> $NIX_CONF_DIR/nix.conf + + nix-store --init + + cp -r ${../.} lib + echo "Running lib/tests/modules.sh" + bash lib/tests/modules.sh + + echo "Running lib/tests/filesystem.sh" + TEST_LIB=$PWD/lib bash lib/tests/filesystem.sh + + echo "Running lib/tests/sources.sh" + TEST_LIB=$PWD/lib bash lib/tests/sources.sh + + mkdir $out + echo success > $out/${nix.version} + ''; + +in + pkgs.symlinkJoin { + name = "nixpkgs-lib-tests"; + paths = map testWithNix nixVersions; + } diff --git a/tests/sources.sh b/tests/sources.sh new file mode 100755 index 000000000..cda77aa96 --- /dev/null +++ b/tests/sources.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -euo pipefail +shopt -s inherit_errexit + +# Use +# || die +die() { + echo >&2 "test case failed: " "$@" + exit 1 +} + +if test -n "${TEST_LIB:-}"; then + NIX_PATH=nixpkgs="$(dirname "$TEST_LIB")" +else + NIX_PATH=nixpkgs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.."; pwd)" +fi +export NIX_PATH + +work="$(mktemp -d)" +clean_up() { + rm -rf "$work" +} +trap clean_up EXIT +cd "$work" + +# Crudely unquotes a JSON string by just taking everything between the first and the second quote. +# We're only using this for resulting /nix/store paths, which can't contain " anyways, +# nor can they contain any other characters that would need to be escaped specially in JSON +# This way we don't need to add a dependency on e.g. jq +crudeUnquoteJSON() { + cut -d \" -f2 +} + +touch {README.md,module.o,foo.bar} + +dir="$(nix-instantiate --eval --strict --read-write-mode --json --expr '(with import ; "${ + cleanSource ./. +}")' | crudeUnquoteJSON)" +(cd "$dir"; find) | sort -f | diff -U10 - <(cat <&2 tests ok diff --git a/tests/systems.nix b/tests/systems.nix new file mode 100644 index 000000000..2afe128c4 --- /dev/null +++ b/tests/systems.nix @@ -0,0 +1,42 @@ +# We assert that the new algorithmic way of generating these lists matches the +# way they were hard-coded before. +# +# One might think "if we exhaustively test, what's the point of procedurally +# calculating the lists anyway?". The answer is one can mindlessly update these +# tests as new platforms become supported, and then just give the diff a quick +# sanity check before committing :). +let + lib = import ../default.nix; + mseteq = x: y: { + expr = lib.sort lib.lessThan x; + expected = lib.sort lib.lessThan y; + }; +in +with lib.systems.doubles; lib.runTests { + testall = mseteq all (linux ++ darwin ++ freebsd ++ openbsd ++ netbsd ++ illumos ++ wasi ++ windows ++ embedded ++ mmix ++ js ++ genode ++ redox); + + testarm = mseteq arm [ "armv5tel-linux" "armv6l-linux" "armv6l-netbsd" "armv6l-none" "armv7a-linux" "armv7a-netbsd" "armv7l-linux" "armv7l-netbsd" "arm-none" "armv7a-darwin" ]; + testarmv7 = mseteq armv7 [ "armv7a-darwin" "armv7a-linux" "armv7l-linux" "armv7a-netbsd" "armv7l-netbsd" ]; + testi686 = mseteq i686 [ "i686-linux" "i686-freebsd13" "i686-genode" "i686-netbsd" "i686-openbsd" "i686-cygwin" "i686-windows" "i686-none" "i686-darwin" ]; + testmips = mseteq mips [ "mips-linux" "mips64-linux" "mips64el-linux" "mipsel-linux" "mipsel-netbsd" ]; + testmmix = mseteq mmix [ "mmix-mmixware" ]; + testpower = mseteq power [ "powerpc-netbsd" "powerpc-none" "powerpc64-linux" "powerpc64le-linux" "powerpcle-none" ]; + testriscv = mseteq riscv [ "riscv32-linux" "riscv64-linux" "riscv32-netbsd" "riscv64-netbsd" "riscv32-none" "riscv64-none" ]; + testriscv32 = mseteq riscv32 [ "riscv32-linux" "riscv32-netbsd" "riscv32-none" ]; + testriscv64 = mseteq riscv64 [ "riscv64-linux" "riscv64-netbsd" "riscv64-none" ]; + tests390x = mseteq s390x [ "s390x-linux" "s390x-none" ]; + testx86_64 = mseteq x86_64 [ "x86_64-linux" "x86_64-darwin" "x86_64-freebsd13" "x86_64-genode" "x86_64-redox" "x86_64-openbsd" "x86_64-netbsd" "x86_64-cygwin" "x86_64-solaris" "x86_64-windows" "x86_64-none" ]; + + testcygwin = mseteq cygwin [ "i686-cygwin" "x86_64-cygwin" ]; + testdarwin = mseteq darwin [ "x86_64-darwin" "i686-darwin" "aarch64-darwin" "armv7a-darwin" ]; + testfreebsd = mseteq freebsd [ "i686-freebsd13" "x86_64-freebsd13" ]; + testgenode = mseteq genode [ "aarch64-genode" "i686-genode" "x86_64-genode" ]; + testredox = mseteq redox [ "x86_64-redox" ]; + testgnu = mseteq gnu (linux /* ++ kfreebsd ++ ... */); + testillumos = mseteq illumos [ "x86_64-solaris" ]; + testlinux = mseteq linux [ "aarch64-linux" "armv5tel-linux" "armv6l-linux" "armv7a-linux" "armv7l-linux" "i686-linux" "loongarch64-linux" "m68k-linux" "microblaze-linux" "microblazeel-linux" "mips-linux" "mips64-linux" "mips64el-linux" "mipsel-linux" "powerpc64-linux" "powerpc64le-linux" "riscv32-linux" "riscv64-linux" "s390-linux" "s390x-linux" "x86_64-linux" ]; + testnetbsd = mseteq netbsd [ "aarch64-netbsd" "armv6l-netbsd" "armv7a-netbsd" "armv7l-netbsd" "i686-netbsd" "m68k-netbsd" "mipsel-netbsd" "powerpc-netbsd" "riscv32-netbsd" "riscv64-netbsd" "x86_64-netbsd" ]; + testopenbsd = mseteq openbsd [ "i686-openbsd" "x86_64-openbsd" ]; + testwindows = mseteq windows [ "i686-cygwin" "x86_64-cygwin" "i686-windows" "x86_64-windows" ]; + testunix = mseteq unix (linux ++ darwin ++ freebsd ++ openbsd ++ netbsd ++ illumos ++ cygwin ++ redox); +} diff --git a/tests/teams.nix b/tests/teams.nix new file mode 100644 index 000000000..8a0a5d272 --- /dev/null +++ b/tests/teams.nix @@ -0,0 +1,50 @@ +# to run these tests: +# nix-build nixpkgs/lib/tests/teams.nix +# If it builds, all tests passed +{ pkgs ? import ../.. {}, lib ? pkgs.lib }: + +let + inherit (lib) types; + + teamModule = { config, ... }: { + options = { + shortName = lib.mkOption { + type = types.str; + }; + scope = lib.mkOption { + type = types.str; + }; + enableFeatureFreezePing = lib.mkOption { + type = types.bool; + default = false; + }; + members = lib.mkOption { + type = types.listOf (types.submodule + (import ./maintainer-module.nix { inherit lib; }) + ); + default = []; + }; + githubTeams = lib.mkOption { + type = types.listOf types.str; + default = []; + }; + }; + }; + + checkTeam = team: uncheckedAttrs: + let + prefix = [ "lib" "maintainer-team" team ]; + checkedAttrs = (lib.modules.evalModules { + inherit prefix; + modules = [ + teamModule + { + _file = toString ../../maintainers/team-list.nix; + config = uncheckedAttrs; + } + ]; + }).config; + in checkedAttrs; + + checkedTeams = lib.mapAttrs checkTeam lib.teams; +in pkgs.writeTextDir "maintainer-teams.json" (builtins.toJSON checkedTeams) diff --git a/tests/test-to-plist-expected.plist b/tests/test-to-plist-expected.plist new file mode 100644 index 000000000..df0528a60 --- /dev/null +++ b/tests/test-to-plist-expected.plist @@ -0,0 +1,46 @@ + + + + + nested + + values + + attrs + + foo b/ar + baz + + bool + + emptyattrs + + + + emptylist + + + + emptystring + + float + 0.133700 + int + 42 + list + + 3 + 4 + test + + newlinestring + + + path + /foo + string + fn${o}"r\d + + + + \ No newline at end of file diff --git a/trivial.nix b/trivial.nix new file mode 100644 index 000000000..26e4b3240 --- /dev/null +++ b/trivial.nix @@ -0,0 +1,522 @@ +{ lib }: + +rec { + + ## Simple (higher order) functions + + /* The identity function + For when you need a function that does “nothing”. + + Type: id :: a -> a + */ + id = + # The value to return + x: x; + + /* The constant function + + Ignores the second argument. If called with only one argument, + constructs a function that always returns a static value. + + Type: const :: a -> b -> a + Example: + let f = const 5; in f 10 + => 5 + */ + const = + # Value to return + x: + # Value to ignore + y: x; + + /* Pipes a value through a list of functions, left to right. + + Type: pipe :: a -> [] -> + Example: + pipe 2 [ + (x: x + 2) # 2 + 2 = 4 + (x: x * 2) # 4 * 2 = 8 + ] + => 8 + + # ideal to do text transformations + pipe [ "a/b" "a/c" ] [ + + # create the cp command + (map (file: ''cp "${src}/${file}" $out\n'')) + + # concatenate all commands into one string + lib.concatStrings + + # make that string into a nix derivation + (pkgs.runCommand "copy-to-out" {}) + + ] + => + + The output type of each function has to be the input type + of the next function, and the last function returns the + final value. + */ + pipe = val: functions: + let reverseApply = x: f: f x; + in builtins.foldl' reverseApply val functions; + + # note please don’t add a function like `compose = flip pipe`. + # This would confuse users, because the order of the functions + # in the list is not clear. With pipe, it’s obvious that it + # goes first-to-last. With `compose`, not so much. + + ## Named versions corresponding to some builtin operators. + + /* Concatenate two lists + + Type: concat :: [a] -> [a] -> [a] + + Example: + concat [ 1 2 ] [ 3 4 ] + => [ 1 2 3 4 ] + */ + concat = x: y: x ++ y; + + /* boolean “or” */ + or = x: y: x || y; + + /* boolean “and” */ + and = x: y: x && y; + + /* bitwise “and” */ + bitAnd = builtins.bitAnd + or (import ./zip-int-bits.nix + (a: b: if a==1 && b==1 then 1 else 0)); + + /* bitwise “or” */ + bitOr = builtins.bitOr + or (import ./zip-int-bits.nix + (a: b: if a==1 || b==1 then 1 else 0)); + + /* bitwise “xor” */ + bitXor = builtins.bitXor + or (import ./zip-int-bits.nix + (a: b: if a!=b then 1 else 0)); + + /* bitwise “not” */ + bitNot = builtins.sub (-1); + + /* Convert a boolean to a string. + + This function uses the strings "true" and "false" to represent + boolean values. Calling `toString` on a bool instead returns "1" + and "" (sic!). + + Type: boolToString :: bool -> string + */ + boolToString = b: if b then "true" else "false"; + + /* Merge two attribute sets shallowly, right side trumps left + + mergeAttrs :: attrs -> attrs -> attrs + + Example: + mergeAttrs { a = 1; b = 2; } { b = 3; c = 4; } + => { a = 1; b = 3; c = 4; } + */ + mergeAttrs = + # Left attribute set + x: + # Right attribute set (higher precedence for equal keys) + y: x // y; + + /* Flip the order of the arguments of a binary function. + + Type: flip :: (a -> b -> c) -> (b -> a -> c) + + Example: + flip concat [1] [2] + => [ 2 1 ] + */ + flip = f: a: b: f b a; + + /* Apply function if the supplied argument is non-null. + + Example: + mapNullable (x: x+1) null + => null + mapNullable (x: x+1) 22 + => 23 + */ + mapNullable = + # Function to call + f: + # Argument to check for null before passing it to `f` + a: if a == null then a else f a; + + # Pull in some builtins not included elsewhere. + inherit (builtins) + pathExists readFile isBool + isInt isFloat add sub lessThan + seq deepSeq genericClosure; + + + ## nixpkgs version strings + + /* Returns the current full nixpkgs version number. */ + version = release + versionSuffix; + + /* Returns the current nixpkgs release number as string. */ + release = lib.strings.fileContents ../.version; + + /* The latest release that is supported, at the time of release branch-off, + if applicable. + + Ideally, out-of-tree modules should be able to evaluate cleanly with all + supported Nixpkgs versions (master, release and old release until EOL). + So if possible, deprecation warnings should take effect only when all + out-of-tree expressions/libs/modules can upgrade to the new way without + losing support for supported Nixpkgs versions. + + This release number allows deprecation warnings to be implemented such that + they take effect as soon as the oldest release reaches end of life. */ + oldestSupportedRelease = + # Update on master only. Do not backport. + 2211; + + /* Whether a feature is supported in all supported releases (at the time of + release branch-off, if applicable). See `oldestSupportedRelease`. */ + isInOldestRelease = + /* Release number of feature introduction as an integer, e.g. 2111 for 21.11. + Set it to the upcoming release, matching the nixpkgs/.version file. + */ + release: + release <= lib.trivial.oldestSupportedRelease; + + /* Returns the current nixpkgs release code name. + + On each release the first letter is bumped and a new animal is chosen + starting with that new letter. + */ + codeName = "Tapir"; + + /* Returns the current nixpkgs version suffix as string. */ + versionSuffix = + let suffixFile = ../.version-suffix; + in if pathExists suffixFile + then lib.strings.fileContents suffixFile + else "pre-git"; + + /* Attempts to return the the current revision of nixpkgs and + returns the supplied default value otherwise. + + Type: revisionWithDefault :: string -> string + */ + revisionWithDefault = + # Default value to return if revision can not be determined + default: + let + revisionFile = "${toString ./..}/.git-revision"; + gitRepo = "${toString ./..}/.git"; + in if lib.pathIsGitRepo gitRepo + then lib.commitIdFromGitRepo gitRepo + else if lib.pathExists revisionFile then lib.fileContents revisionFile + else default; + + nixpkgsVersion = builtins.trace "`lib.nixpkgsVersion` is deprecated, use `lib.version` instead!" version; + + /* Determine whether the function is being called from inside a Nix + shell. + + Type: inNixShell :: bool + */ + inNixShell = builtins.getEnv "IN_NIX_SHELL" != ""; + + /* Determine whether the function is being called from inside pure-eval mode + by seeing whether `builtins` contains `currentSystem`. If not, we must be in + pure-eval mode. + + Type: inPureEvalMode :: bool + */ + inPureEvalMode = ! builtins ? currentSystem; + + ## Integer operations + + /* Return minimum of two numbers. */ + min = x: y: if x < y then x else y; + + /* Return maximum of two numbers. */ + max = x: y: if x > y then x else y; + + /* Integer modulus + + Example: + mod 11 10 + => 1 + mod 1 10 + => 1 + */ + mod = base: int: base - (int * (builtins.div base int)); + + + ## Comparisons + + /* C-style comparisons + + a < b, compare a b => -1 + a == b, compare a b => 0 + a > b, compare a b => 1 + */ + compare = a: b: + if a < b + then -1 + else if a > b + then 1 + else 0; + + /* Split type into two subtypes by predicate `p`, take all elements + of the first subtype to be less than all the elements of the + second subtype, compare elements of a single subtype with `yes` + and `no` respectively. + + Type: (a -> bool) -> (a -> a -> int) -> (a -> a -> int) -> (a -> a -> int) + + Example: + let cmp = splitByAndCompare (hasPrefix "foo") compare compare; in + + cmp "a" "z" => -1 + cmp "fooa" "fooz" => -1 + + cmp "f" "a" => 1 + cmp "fooa" "a" => -1 + # while + compare "fooa" "a" => 1 + */ + splitByAndCompare = + # Predicate + p: + # Comparison function if predicate holds for both values + yes: + # Comparison function if predicate holds for neither value + no: + # First value to compare + a: + # Second value to compare + b: + if p a + then if p b then yes a b else -1 + else if p b then 1 else no a b; + + + /* Reads a JSON file. + + Type :: path -> any + */ + importJSON = path: + builtins.fromJSON (builtins.readFile path); + + /* Reads a TOML file. + + Type :: path -> any + */ + importTOML = path: + builtins.fromTOML (builtins.readFile path); + + ## Warnings + + # See https://github.com/NixOS/nix/issues/749. Eventually we'd like these + # to expand to Nix builtins that carry metadata so that Nix can filter out + # the INFO messages without parsing the message string. + # + # Usage: + # { + # foo = lib.warn "foo is deprecated" oldFoo; + # bar = lib.warnIf (bar == "") "Empty bar is deprecated" bar; + # } + # + # TODO: figure out a clever way to integrate location information from + # something like __unsafeGetAttrPos. + + /* + Print a warning before returning the second argument. This function behaves + like `builtins.trace`, but requires a string message and formats it as a + warning, including the `warning: ` prefix. + + To get a call stack trace and abort evaluation, set the environment variable + `NIX_ABORT_ON_WARN=true` and set the Nix options `--option pure-eval false --show-trace` + + Type: string -> a -> a + */ + warn = + if lib.elem (builtins.getEnv "NIX_ABORT_ON_WARN") ["1" "true" "yes"] + then msg: builtins.trace "warning: ${msg}" (abort "NIX_ABORT_ON_WARN=true; warnings are treated as unrecoverable errors.") + else msg: builtins.trace "warning: ${msg}"; + + /* + Like warn, but only warn when the first argument is `true`. + + Type: bool -> string -> a -> a + */ + warnIf = cond: msg: if cond then warn msg else x: x; + + /* + Like warnIf, but negated (warn if the first argument is `false`). + + Type: bool -> string -> a -> a + */ + warnIfNot = cond: msg: if cond then x: x else warn msg; + + /* + Like the `assert b; e` expression, but with a custom error message and + without the semicolon. + + If true, return the identity function, `r: r`. + + If false, throw the error message. + + Calls can be juxtaposed using function application, as `(r: r) a = a`, so + `(r: r) (r: r) a = a`, and so forth. + + Type: bool -> string -> a -> a + + Example: + + throwIfNot (lib.isList overlays) "The overlays argument to nixpkgs must be a list." + lib.foldr (x: throwIfNot (lib.isFunction x) "All overlays passed to nixpkgs must be functions.") (r: r) overlays + pkgs + + */ + throwIfNot = cond: msg: if cond then x: x else throw msg; + + /* + Like throwIfNot, but negated (throw if the first argument is `true`). + + Type: bool -> string -> a -> a + */ + throwIf = cond: msg: if cond then throw msg else x: x; + + /* Check if the elements in a list are valid values from a enum, returning the identity function, or throwing an error message otherwise. + + Example: + let colorVariants = ["bright" "dark" "black"] + in checkListOfEnum "color variants" [ "standard" "light" "dark" ] colorVariants; + => + error: color variants: bright, black unexpected; valid ones: standard, light, dark + + Type: String -> List ComparableVal -> List ComparableVal -> a -> a + */ + checkListOfEnum = msg: valid: given: + let + unexpected = lib.subtractLists valid given; + in + lib.throwIfNot (unexpected == []) + "${msg}: ${builtins.concatStringsSep ", " (builtins.map builtins.toString unexpected)} unexpected; valid ones: ${builtins.concatStringsSep ", " (builtins.map builtins.toString valid)}"; + + info = msg: builtins.trace "INFO: ${msg}"; + + showWarnings = warnings: res: lib.foldr (w: x: warn w x) res warnings; + + ## Function annotations + + /* Add metadata about expected function arguments to a function. + The metadata should match the format given by + builtins.functionArgs, i.e. a set from expected argument to a bool + representing whether that argument has a default or not. + setFunctionArgs : (a → b) → Map String Bool → (a → b) + + This function is necessary because you can't dynamically create a + function of the { a, b ? foo, ... }: format, but some facilities + like callPackage expect to be able to query expected arguments. + */ + setFunctionArgs = f: args: + { # TODO: Should we add call-time "type" checking like built in? + __functor = self: f; + __functionArgs = args; + }; + + /* Extract the expected function arguments from a function. + This works both with nix-native { a, b ? foo, ... }: style + functions and functions with args set with 'setFunctionArgs'. It + has the same return type and semantics as builtins.functionArgs. + setFunctionArgs : (a → b) → Map String Bool. + */ + functionArgs = f: + if f ? __functor + then f.__functionArgs or (lib.functionArgs (f.__functor f)) + else builtins.functionArgs f; + + /* Check whether something is a function or something + annotated with function args. + */ + isFunction = f: builtins.isFunction f || + (f ? __functor && isFunction (f.__functor f)); + + /* + Turns any non-callable values into constant functions. + Returns callable values as is. + + Example: + + nix-repl> lib.toFunction 1 2 + 1 + + nix-repl> lib.toFunction (x: x + 1) 2 + 3 + */ + toFunction = + # Any value + v: + if isFunction v + then v + else k: v; + + /* Convert the given positive integer to a string of its hexadecimal + representation. For example: + + toHexString 0 => "0" + + toHexString 16 => "10" + + toHexString 250 => "FA" + */ + toHexString = i: + let + toHexDigit = d: + if d < 10 + then toString d + else + { + "10" = "A"; + "11" = "B"; + "12" = "C"; + "13" = "D"; + "14" = "E"; + "15" = "F"; + }.${toString d}; + in + lib.concatMapStrings toHexDigit (toBaseDigits 16 i); + + /* `toBaseDigits base i` converts the positive integer i to a list of its + digits in the given base. For example: + + toBaseDigits 10 123 => [ 1 2 3 ] + + toBaseDigits 2 6 => [ 1 1 0 ] + + toBaseDigits 16 250 => [ 15 10 ] + */ + toBaseDigits = base: i: + let + go = i: + if i < base + then [i] + else + let + r = i - ((i / base) * base); + q = (i - r) / base; + in + [r] ++ go q; + in + assert (isInt base); + assert (isInt i); + assert (base >= 2); + assert (i >= 0); + lib.reverseList (go i); +} diff --git a/types.nix b/types.nix new file mode 100644 index 000000000..9360d42f5 --- /dev/null +++ b/types.nix @@ -0,0 +1,908 @@ +# Definitions related to run-time type checking. Used in particular +# to type-check NixOS configurations. +{ lib }: + +let + inherit (lib) + elem + flip + isAttrs + isBool + isDerivation + isFloat + isFunction + isInt + isList + isString + isStorePath + toDerivation + toList + ; + inherit (lib.lists) + all + concatLists + count + elemAt + filter + foldl' + head + imap1 + last + length + tail + ; + inherit (lib.attrsets) + attrNames + filterAttrs + hasAttr + mapAttrs + optionalAttrs + zipAttrsWith + ; + inherit (lib.options) + getFiles + getValues + mergeDefaultOption + mergeEqualOption + mergeOneOption + mergeUniqueOption + showFiles + showOption + ; + inherit (lib.strings) + concatMapStringsSep + concatStringsSep + escapeNixString + hasInfix + isStringLike + ; + inherit (lib.trivial) + boolToString + ; + + inherit (lib.modules) + mergeDefinitions + fixupOptionType + mergeOptionDecls + ; + outer_types = +rec { + isType = type: x: (x._type or "") == type; + + setType = typeName: value: value // { + _type = typeName; + }; + + + # Default type merging function + # takes two type functors and return the merged type + defaultTypeMerge = f: f': + let wrapped = f.wrapped.typeMerge f'.wrapped.functor; + payload = f.binOp f.payload f'.payload; + in + # cannot merge different types + if f.name != f'.name + then null + # simple types + else if (f.wrapped == null && f'.wrapped == null) + && (f.payload == null && f'.payload == null) + then f.type + # composed types + else if (f.wrapped != null && f'.wrapped != null) && (wrapped != null) + then f.type wrapped + # value types + else if (f.payload != null && f'.payload != null) && (payload != null) + then f.type payload + else null; + + # Default type functor + defaultFunctor = name: { + inherit name; + type = types.${name} or null; + wrapped = null; + payload = null; + binOp = a: b: null; + }; + + isOptionType = isType "option-type"; + mkOptionType = + { # Human-readable representation of the type, should be equivalent to + # the type function name. + name + , # Description of the type, defined recursively by embedding the wrapped type if any. + description ? null + # A hint for whether or not this description needs parentheses. Possible values: + # - "noun": a simple noun phrase such as "positive integer" + # - "conjunction": a phrase with a potentially ambiguous "or" connective. + # - "composite": a phrase with an "of" connective + # See the `optionDescriptionPhrase` function. + , descriptionClass ? null + , # DO NOT USE WITHOUT KNOWING WHAT YOU ARE DOING! + # Function applied to each definition that must return false when a definition + # does not match the type. It should not check more than the root of the value, + # because checking nested values reduces laziness, leading to unnecessary + # infinite recursions in the module system. + # Further checks of nested values should be performed by throwing in + # the merge function. + # Strict and deep type checking can be performed by calling lib.deepSeq on + # the merged value. + # + # See https://github.com/NixOS/nixpkgs/pull/6794 that introduced this change, + # https://github.com/NixOS/nixpkgs/pull/173568 and + # https://github.com/NixOS/nixpkgs/pull/168295 that attempted to revert this, + # https://github.com/NixOS/nixpkgs/issues/191124 and + # https://github.com/NixOS/nixos-search/issues/391 for what happens if you ignore + # this disclaimer. + check ? (x: true) + , # Merge a list of definitions together into a single value. + # This function is called with two arguments: the location of + # the option in the configuration as a list of strings + # (e.g. ["boot" "loader "grub" "enable"]), and a list of + # definition values and locations (e.g. [ { file = "/foo.nix"; + # value = 1; } { file = "/bar.nix"; value = 2 } ]). + merge ? mergeDefaultOption + , # Whether this type has a value representing nothingness. If it does, + # this should be a value of the form { value = ; } + # If it doesn't, this should be {} + # This may be used when a value is required for `mkIf false`. This allows the extra laziness in e.g. `lazyAttrsOf`. + emptyValue ? {} + , # Return a flat list of sub-options. Used to generate + # documentation. + getSubOptions ? prefix: {} + , # List of modules if any, or null if none. + getSubModules ? null + , # Function for building the same option type with a different list of + # modules. + substSubModules ? m: null + , # Function that merge type declarations. + # internal, takes a functor as argument and returns the merged type. + # returning null means the type is not mergeable + typeMerge ? defaultTypeMerge functor + , # The type functor. + # internal, representation of the type as an attribute set. + # name: name of the type + # type: type function. + # wrapped: the type wrapped in case of compound types. + # payload: values of the type, two payloads of the same type must be + # combinable with the binOp binary operation. + # binOp: binary operation that merge two payloads of the same type. + functor ? defaultFunctor name + , # The deprecation message to display when this type is used by an option + # If null, the type isn't deprecated + deprecationMessage ? null + , # The types that occur in the definition of this type. This is used to + # issue deprecation warnings recursively. Can also be used to reuse + # nested types + nestedTypes ? {} + }: + { _type = "option-type"; + inherit + name check merge emptyValue getSubOptions getSubModules substSubModules + typeMerge functor deprecationMessage nestedTypes descriptionClass; + description = if description == null then name else description; + }; + + # optionDescriptionPhrase :: (str -> bool) -> optionType -> str + # + # Helper function for producing unambiguous but readable natural language + # descriptions of types. + # + # Parameters + # + # optionDescriptionPhase unparenthesize optionType + # + # `unparenthesize`: A function from descriptionClass string to boolean. + # It must return true when the class of phrase will fit unambiguously into + # the description of the caller. + # + # `optionType`: The option type to parenthesize or not. + # The option whose description we're returning. + # + # Return value + # + # The description of the `optionType`, with parentheses if there may be an + # ambiguity. + optionDescriptionPhrase = unparenthesize: t: + if unparenthesize (t.descriptionClass or null) + then t.description + else "(${t.description})"; + + # When adding new types don't forget to document them in + # nixos/doc/manual/development/option-types.xml! + types = rec { + + raw = mkOptionType rec { + name = "raw"; + description = "raw value"; + descriptionClass = "noun"; + check = value: true; + merge = mergeOneOption; + }; + + anything = mkOptionType { + name = "anything"; + description = "anything"; + descriptionClass = "noun"; + check = value: true; + merge = loc: defs: + let + getType = value: + if isAttrs value && isStringLike value + then "stringCoercibleSet" + else builtins.typeOf value; + + # Returns the common type of all definitions, throws an error if they + # don't have the same type + commonType = foldl' (type: def: + if getType def.value == type + then type + else throw "The option `${showOption loc}' has conflicting option types in ${showFiles (getFiles defs)}" + ) (getType (head defs).value) defs; + + mergeFunction = { + # Recursively merge attribute sets + set = (attrsOf anything).merge; + # Safe and deterministic behavior for lists is to only accept one definition + # listOf only used to apply mkIf and co. + list = + if length defs > 1 + then throw "The option `${showOption loc}' has conflicting definitions, in ${showFiles (getFiles defs)}." + else (listOf anything).merge; + # This is the type of packages, only accept a single definition + stringCoercibleSet = mergeOneOption; + lambda = loc: defs: arg: anything.merge + (loc ++ [ "" ]) + (map (def: { + file = def.file; + value = def.value arg; + }) defs); + # Otherwise fall back to only allowing all equal definitions + }.${commonType} or mergeEqualOption; + in mergeFunction loc defs; + }; + + unspecified = mkOptionType { + name = "unspecified"; + description = "unspecified value"; + descriptionClass = "noun"; + }; + + bool = mkOptionType { + name = "bool"; + description = "boolean"; + descriptionClass = "noun"; + check = isBool; + merge = mergeEqualOption; + }; + + int = mkOptionType { + name = "int"; + description = "signed integer"; + descriptionClass = "noun"; + check = isInt; + merge = mergeEqualOption; + }; + + # Specialized subdomains of int + ints = + let + betweenDesc = lowest: highest: + "${toString lowest} and ${toString highest} (both inclusive)"; + between = lowest: highest: + assert lib.assertMsg (lowest <= highest) + "ints.between: lowest must be smaller than highest"; + addCheck int (x: x >= lowest && x <= highest) // { + name = "intBetween"; + description = "integer between ${betweenDesc lowest highest}"; + }; + ign = lowest: highest: name: docStart: + between lowest highest // { + inherit name; + description = docStart + "; between ${betweenDesc lowest highest}"; + }; + unsign = bit: range: ign 0 (range - 1) + "unsignedInt${toString bit}" "${toString bit} bit unsigned integer"; + sign = bit: range: ign (0 - (range / 2)) (range / 2 - 1) + "signedInt${toString bit}" "${toString bit} bit signed integer"; + + in { + /* An int with a fixed range. + * + * Example: + * (ints.between 0 100).check (-1) + * => false + * (ints.between 0 100).check (101) + * => false + * (ints.between 0 0).check 0 + * => true + */ + inherit between; + + unsigned = addCheck types.int (x: x >= 0) // { + name = "unsignedInt"; + description = "unsigned integer, meaning >=0"; + }; + positive = addCheck types.int (x: x > 0) // { + name = "positiveInt"; + description = "positive integer, meaning >0"; + }; + u8 = unsign 8 256; + u16 = unsign 16 65536; + # the biggest int Nix accepts is 2^63 - 1 (9223372036854775808) + # the smallest int Nix accepts is -2^63 (-9223372036854775807) + u32 = unsign 32 4294967296; + # u64 = unsign 64 18446744073709551616; + + s8 = sign 8 256; + s16 = sign 16 65536; + s32 = sign 32 4294967296; + }; + + # Alias of u16 for a port number + port = ints.u16; + + float = mkOptionType { + name = "float"; + description = "floating point number"; + descriptionClass = "noun"; + check = isFloat; + merge = mergeEqualOption; + }; + + number = either int float; + + numbers = let + betweenDesc = lowest: highest: + "${builtins.toJSON lowest} and ${builtins.toJSON highest} (both inclusive)"; + in { + between = lowest: highest: + assert lib.assertMsg (lowest <= highest) + "numbers.between: lowest must be smaller than highest"; + addCheck number (x: x >= lowest && x <= highest) // { + name = "numberBetween"; + description = "integer or floating point number between ${betweenDesc lowest highest}"; + }; + + nonnegative = addCheck number (x: x >= 0) // { + name = "numberNonnegative"; + description = "nonnegative integer or floating point number, meaning >=0"; + }; + positive = addCheck number (x: x > 0) // { + name = "numberPositive"; + description = "positive integer or floating point number, meaning >0"; + }; + }; + + str = mkOptionType { + name = "str"; + description = "string"; + descriptionClass = "noun"; + check = isString; + merge = mergeEqualOption; + }; + + nonEmptyStr = mkOptionType { + name = "nonEmptyStr"; + description = "non-empty string"; + descriptionClass = "noun"; + check = x: str.check x && builtins.match "[ \t\n]*" x == null; + inherit (str) merge; + }; + + # Allow a newline character at the end and trim it in the merge function. + singleLineStr = + let + inherit (strMatching "[^\n\r]*\n?") check merge; + in + mkOptionType { + name = "singleLineStr"; + description = "(optionally newline-terminated) single-line string"; + descriptionClass = "noun"; + inherit check; + merge = loc: defs: + lib.removeSuffix "\n" (merge loc defs); + }; + + strMatching = pattern: mkOptionType { + name = "strMatching ${escapeNixString pattern}"; + description = "string matching the pattern ${pattern}"; + descriptionClass = "noun"; + check = x: str.check x && builtins.match pattern x != null; + inherit (str) merge; + }; + + # Merge multiple definitions by concatenating them (with the given + # separator between the values). + separatedString = sep: mkOptionType rec { + name = "separatedString"; + description = if sep == "" + then "Concatenated string" # for types.string. + else "strings concatenated with ${builtins.toJSON sep}" + ; + descriptionClass = "noun"; + check = isString; + merge = loc: defs: concatStringsSep sep (getValues defs); + functor = (defaultFunctor name) // { + payload = sep; + binOp = sepLhs: sepRhs: + if sepLhs == sepRhs then sepLhs + else null; + }; + }; + + lines = separatedString "\n"; + commas = separatedString ","; + envVar = separatedString ":"; + + # Deprecated; should not be used because it quietly concatenates + # strings, which is usually not what you want. + string = separatedString "" // { + name = "string"; + deprecationMessage = "See https://github.com/NixOS/nixpkgs/pull/66346 for better alternative types."; + }; + + passwdEntry = entryType: addCheck entryType (str: !(hasInfix ":" str || hasInfix "\n" str)) // { + name = "passwdEntry ${entryType.name}"; + description = "${optionDescriptionPhrase (class: class == "noun") entryType}, not containing newlines or colons"; + }; + + attrs = mkOptionType { + name = "attrs"; + description = "attribute set"; + check = isAttrs; + merge = loc: foldl' (res: def: res // def.value) {}; + emptyValue = { value = {}; }; + }; + + # A package is a top-level store path (/nix/store/hash-name). This includes: + # - derivations + # - more generally, attribute sets with an `outPath` or `__toString` attribute + # pointing to a store path, e.g. flake inputs + # - strings with context, e.g. "${pkgs.foo}" or (toString pkgs.foo) + # - hardcoded store path literals (/nix/store/hash-foo) or strings without context + # ("/nix/store/hash-foo"). These get a context added to them using builtins.storePath. + package = mkOptionType { + name = "package"; + descriptionClass = "noun"; + check = x: isDerivation x || isStorePath x; + merge = loc: defs: + let res = mergeOneOption loc defs; + in if builtins.isPath res || (builtins.isString res && ! builtins.hasContext res) + then toDerivation res + else res; + }; + + shellPackage = package // { + check = x: isDerivation x && hasAttr "shellPath" x; + }; + + pkgs = addCheck + (unique { message = "A Nixpkgs pkgs set can not be merged with another pkgs set."; } attrs // { + name = "pkgs"; + descriptionClass = "noun"; + description = "Nixpkgs package set"; + }) + (x: (x._type or null) == "pkgs"); + + path = mkOptionType { + name = "path"; + descriptionClass = "noun"; + check = x: isStringLike x && builtins.substring 0 1 (toString x) == "/"; + merge = mergeEqualOption; + }; + + listOf = elemType: mkOptionType rec { + name = "listOf"; + description = "list of ${optionDescriptionPhrase (class: class == "noun" || class == "composite") elemType}"; + descriptionClass = "composite"; + check = isList; + merge = loc: defs: + map (x: x.value) (filter (x: x ? value) (concatLists (imap1 (n: def: + imap1 (m: def': + (mergeDefinitions + (loc ++ ["[definition ${toString n}-entry ${toString m}]"]) + elemType + [{ inherit (def) file; value = def'; }] + ).optionalValue + ) def.value + ) defs))); + emptyValue = { value = []; }; + getSubOptions = prefix: elemType.getSubOptions (prefix ++ ["*"]); + getSubModules = elemType.getSubModules; + substSubModules = m: listOf (elemType.substSubModules m); + functor = (defaultFunctor name) // { wrapped = elemType; }; + nestedTypes.elemType = elemType; + }; + + nonEmptyListOf = elemType: + let list = addCheck (types.listOf elemType) (l: l != []); + in list // { + description = "non-empty ${optionDescriptionPhrase (class: class == "noun") list}"; + emptyValue = { }; # no .value attr, meaning unset + }; + + attrsOf = elemType: mkOptionType rec { + name = "attrsOf"; + description = "attribute set of ${optionDescriptionPhrase (class: class == "noun" || class == "composite") elemType}"; + descriptionClass = "composite"; + check = isAttrs; + merge = loc: defs: + mapAttrs (n: v: v.value) (filterAttrs (n: v: v ? value) (zipAttrsWith (name: defs: + (mergeDefinitions (loc ++ [name]) elemType defs).optionalValue + ) + # Push down position info. + (map (def: mapAttrs (n: v: { inherit (def) file; value = v; }) def.value) defs))); + emptyValue = { value = {}; }; + getSubOptions = prefix: elemType.getSubOptions (prefix ++ [""]); + getSubModules = elemType.getSubModules; + substSubModules = m: attrsOf (elemType.substSubModules m); + functor = (defaultFunctor name) // { wrapped = elemType; }; + nestedTypes.elemType = elemType; + }; + + # A version of attrsOf that's lazy in its values at the expense of + # conditional definitions not working properly. E.g. defining a value with + # `foo.attr = mkIf false 10`, then `foo ? attr == true`, whereas with + # attrsOf it would correctly be `false`. Accessing `foo.attr` would throw an + # error that it's not defined. Use only if conditional definitions don't make sense. + lazyAttrsOf = elemType: mkOptionType rec { + name = "lazyAttrsOf"; + description = "lazy attribute set of ${optionDescriptionPhrase (class: class == "noun" || class == "composite") elemType}"; + descriptionClass = "composite"; + check = isAttrs; + merge = loc: defs: + zipAttrsWith (name: defs: + let merged = mergeDefinitions (loc ++ [name]) elemType defs; + # mergedValue will trigger an appropriate error when accessed + in merged.optionalValue.value or elemType.emptyValue.value or merged.mergedValue + ) + # Push down position info. + (map (def: mapAttrs (n: v: { inherit (def) file; value = v; }) def.value) defs); + emptyValue = { value = {}; }; + getSubOptions = prefix: elemType.getSubOptions (prefix ++ [""]); + getSubModules = elemType.getSubModules; + substSubModules = m: lazyAttrsOf (elemType.substSubModules m); + functor = (defaultFunctor name) // { wrapped = elemType; }; + nestedTypes.elemType = elemType; + }; + + # TODO: deprecate this in the future: + loaOf = elemType: types.attrsOf elemType // { + name = "loaOf"; + deprecationMessage = "Mixing lists with attribute values is no longer" + + " possible; please use `types.attrsOf` instead. See" + + " https://github.com/NixOS/nixpkgs/issues/1800 for the motivation."; + nestedTypes.elemType = elemType; + }; + + # Value of given type but with no merging (i.e. `uniq list`s are not concatenated). + uniq = elemType: mkOptionType rec { + name = "uniq"; + inherit (elemType) description descriptionClass check; + merge = mergeOneOption; + emptyValue = elemType.emptyValue; + getSubOptions = elemType.getSubOptions; + getSubModules = elemType.getSubModules; + substSubModules = m: uniq (elemType.substSubModules m); + functor = (defaultFunctor name) // { wrapped = elemType; }; + nestedTypes.elemType = elemType; + }; + + unique = { message }: type: mkOptionType rec { + name = "unique"; + inherit (type) description descriptionClass check; + merge = mergeUniqueOption { inherit message; }; + emptyValue = type.emptyValue; + getSubOptions = type.getSubOptions; + getSubModules = type.getSubModules; + substSubModules = m: uniq (type.substSubModules m); + functor = (defaultFunctor name) // { wrapped = type; }; + nestedTypes.elemType = type; + }; + + # Null or value of ... + nullOr = elemType: mkOptionType rec { + name = "nullOr"; + description = "null or ${optionDescriptionPhrase (class: class == "noun" || class == "conjunction") elemType}"; + descriptionClass = "conjunction"; + check = x: x == null || elemType.check x; + merge = loc: defs: + let nrNulls = count (def: def.value == null) defs; in + if nrNulls == length defs then null + else if nrNulls != 0 then + throw "The option `${showOption loc}` is defined both null and not null, in ${showFiles (getFiles defs)}." + else elemType.merge loc defs; + emptyValue = { value = null; }; + getSubOptions = elemType.getSubOptions; + getSubModules = elemType.getSubModules; + substSubModules = m: nullOr (elemType.substSubModules m); + functor = (defaultFunctor name) // { wrapped = elemType; }; + nestedTypes.elemType = elemType; + }; + + functionTo = elemType: mkOptionType { + name = "functionTo"; + description = "function that evaluates to a(n) ${optionDescriptionPhrase (class: class == "noun" || class == "composite") elemType}"; + descriptionClass = "composite"; + check = isFunction; + merge = loc: defs: + fnArgs: (mergeDefinitions (loc ++ [ "" ]) elemType (map (fn: { inherit (fn) file; value = fn.value fnArgs; }) defs)).mergedValue; + getSubOptions = prefix: elemType.getSubOptions (prefix ++ [ "" ]); + getSubModules = elemType.getSubModules; + substSubModules = m: functionTo (elemType.substSubModules m); + functor = (defaultFunctor "functionTo") // { wrapped = elemType; }; + nestedTypes.elemType = elemType; + }; + + # A submodule (like typed attribute set). See NixOS manual. + submodule = modules: submoduleWith { + shorthandOnlyDefinesConfig = true; + modules = toList modules; + }; + + # A module to be imported in some other part of the configuration. + deferredModule = deferredModuleWith { }; + + # A module to be imported in some other part of the configuration. + # `staticModules`' options will be added to the documentation, unlike + # options declared via `config`. + deferredModuleWith = attrs@{ staticModules ? [] }: mkOptionType { + name = "deferredModule"; + description = "module"; + descriptionClass = "noun"; + check = x: isAttrs x || isFunction x || path.check x; + merge = loc: defs: { + imports = staticModules ++ map (def: lib.setDefaultModuleLocation "${def.file}, via option ${showOption loc}" def.value) defs; + }; + inherit (submoduleWith { modules = staticModules; }) + getSubOptions + getSubModules; + substSubModules = m: deferredModuleWith (attrs // { + staticModules = m; + }); + functor = defaultFunctor "deferredModuleWith" // { + type = types.deferredModuleWith; + payload = { + inherit staticModules; + }; + binOp = lhs: rhs: { + staticModules = lhs.staticModules ++ rhs.staticModules; + }; + }; + }; + + # The type of a type! + optionType = mkOptionType { + name = "optionType"; + description = "optionType"; + descriptionClass = "noun"; + check = value: value._type or null == "option-type"; + merge = loc: defs: + if length defs == 1 + then (head defs).value + else let + # Prepares the type definitions for mergeOptionDecls, which + # annotates submodules types with file locations + optionModules = map ({ value, file }: + { + _file = file; + # There's no way to merge types directly from the module system, + # but we can cheat a bit by just declaring an option with the type + options = lib.mkOption { + type = value; + }; + } + ) defs; + # Merges all the types into a single one, including submodule merging. + # This also propagates file information to all submodules + mergedOption = fixupOptionType loc (mergeOptionDecls loc optionModules); + in mergedOption.type; + }; + + submoduleWith = + { modules + , specialArgs ? {} + , shorthandOnlyDefinesConfig ? false + , description ? null + , class ? null + }@attrs: + let + inherit (lib.modules) evalModules; + + allModules = defs: map ({ value, file }: + if isAttrs value && shorthandOnlyDefinesConfig + then { _file = file; config = value; } + else { _file = file; imports = [ value ]; } + ) defs; + + base = evalModules { + inherit class specialArgs; + modules = [{ + # This is a work-around for the fact that some sub-modules, + # such as the one included in an attribute set, expects an "args" + # attribute to be given to the sub-module. As the option + # evaluation does not have any specific attribute name yet, we + # provide a default for the documentation and the freeform type. + # + # This is necessary as some option declaration might use the + # "name" attribute given as argument of the submodule and use it + # as the default of option declarations. + # + # We use lookalike unicode single angle quotation marks because + # of the docbook transformation the options receive. In all uses + # > and < wouldn't be encoded correctly so the encoded values + # would be used, and use of `<` and `>` would break the XML document. + # It shouldn't cause an issue since this is cosmetic for the manual. + _module.args.name = lib.mkOptionDefault "‹name›"; + }] ++ modules; + }; + + freeformType = base._module.freeformType; + + name = "submodule"; + + in + mkOptionType { + inherit name; + description = + if description != null then description + else freeformType.description or name; + check = x: isAttrs x || isFunction x || path.check x; + merge = loc: defs: + (base.extendModules { + modules = [ { _module.args.name = last loc; } ] ++ allModules defs; + prefix = loc; + }).config; + emptyValue = { value = {}; }; + getSubOptions = prefix: (base.extendModules + { inherit prefix; }).options // optionalAttrs (freeformType != null) { + # Expose the sub options of the freeform type. Note that the option + # discovery doesn't care about the attribute name used here, so this + # is just to avoid conflicts with potential options from the submodule + _freeformOptions = freeformType.getSubOptions prefix; + }; + getSubModules = modules; + substSubModules = m: submoduleWith (attrs // { + modules = m; + }); + nestedTypes = lib.optionalAttrs (freeformType != null) { + freeformType = freeformType; + }; + functor = defaultFunctor name // { + type = types.submoduleWith; + payload = { + inherit modules class specialArgs shorthandOnlyDefinesConfig description; + }; + binOp = lhs: rhs: { + class = + # `or null` was added for backwards compatibility only. `class` is + # always set in the current version of the module system. + if lhs.class or null == null then rhs.class or null + else if rhs.class or null == null then lhs.class or null + else if lhs.class or null == rhs.class then lhs.class or null + else throw "A submoduleWith option is declared multiple times with conflicting class values \"${toString lhs.class}\" and \"${toString rhs.class}\"."; + modules = lhs.modules ++ rhs.modules; + specialArgs = + let intersecting = builtins.intersectAttrs lhs.specialArgs rhs.specialArgs; + in if intersecting == {} + then lhs.specialArgs // rhs.specialArgs + else throw "A submoduleWith option is declared multiple times with the same specialArgs \"${toString (attrNames intersecting)}\""; + shorthandOnlyDefinesConfig = + if lhs.shorthandOnlyDefinesConfig == null + then rhs.shorthandOnlyDefinesConfig + else if rhs.shorthandOnlyDefinesConfig == null + then lhs.shorthandOnlyDefinesConfig + else if lhs.shorthandOnlyDefinesConfig == rhs.shorthandOnlyDefinesConfig + then lhs.shorthandOnlyDefinesConfig + else throw "A submoduleWith option is declared multiple times with conflicting shorthandOnlyDefinesConfig values"; + description = + if lhs.description == null + then rhs.description + else if rhs.description == null + then lhs.description + else if lhs.description == rhs.description + then lhs.description + else throw "A submoduleWith option is declared multiple times with conflicting descriptions"; + }; + }; + }; + + # A value from a set of allowed ones. + enum = values: + let + inherit (lib.lists) unique; + show = v: + if builtins.isString v then ''"${v}"'' + else if builtins.isInt v then builtins.toString v + else if builtins.isBool v then boolToString v + else ''<${builtins.typeOf v}>''; + in + mkOptionType rec { + name = "enum"; + description = + # Length 0 or 1 enums may occur in a design pattern with type merging + # where an "interface" module declares an empty enum and other modules + # provide implementations, each extending the enum with their own + # identifier. + if values == [] then + "impossible (empty enum)" + else if builtins.length values == 1 then + "value ${show (builtins.head values)} (singular enum)" + else + "one of ${concatMapStringsSep ", " show values}"; + descriptionClass = + if builtins.length values < 2 + then "noun" + else "conjunction"; + check = flip elem values; + merge = mergeEqualOption; + functor = (defaultFunctor name) // { payload = values; binOp = a: b: unique (a ++ b); }; + }; + + # Either value of type `t1` or `t2`. + either = t1: t2: mkOptionType rec { + name = "either"; + description = "${optionDescriptionPhrase (class: class == "noun" || class == "conjunction") t1} or ${optionDescriptionPhrase (class: class == "noun" || class == "conjunction" || class == "composite") t2}"; + descriptionClass = "conjunction"; + check = x: t1.check x || t2.check x; + merge = loc: defs: + let + defList = map (d: d.value) defs; + in + if all (x: t1.check x) defList + then t1.merge loc defs + else if all (x: t2.check x) defList + then t2.merge loc defs + else mergeOneOption loc defs; + typeMerge = f': + let mt1 = t1.typeMerge (elemAt f'.wrapped 0).functor; + mt2 = t2.typeMerge (elemAt f'.wrapped 1).functor; + in + if (name == f'.name) && (mt1 != null) && (mt2 != null) + then functor.type mt1 mt2 + else null; + functor = (defaultFunctor name) // { wrapped = [ t1 t2 ]; }; + nestedTypes.left = t1; + nestedTypes.right = t2; + }; + + # Any of the types in the given list + oneOf = ts: + let + head' = if ts == [] then throw "types.oneOf needs to get at least one type in its argument" else head ts; + tail' = tail ts; + in foldl' either head' tail'; + + # Either value of type `coercedType` or `finalType`, the former is + # converted to `finalType` using `coerceFunc`. + coercedTo = coercedType: coerceFunc: finalType: + assert lib.assertMsg (coercedType.getSubModules == null) + "coercedTo: coercedType must not have submodules (it’s a ${ + coercedType.description})"; + mkOptionType rec { + name = "coercedTo"; + description = "${optionDescriptionPhrase (class: class == "noun") finalType} or ${optionDescriptionPhrase (class: class == "noun") coercedType} convertible to it"; + check = x: (coercedType.check x && finalType.check (coerceFunc x)) || finalType.check x; + merge = loc: defs: + let + coerceVal = val: + if coercedType.check val then coerceFunc val + else val; + in finalType.merge loc (map (def: def // { value = coerceVal def.value; }) defs); + emptyValue = finalType.emptyValue; + getSubOptions = finalType.getSubOptions; + getSubModules = finalType.getSubModules; + substSubModules = m: coercedTo coercedType coerceFunc (finalType.substSubModules m); + typeMerge = t1: t2: null; + functor = (defaultFunctor name) // { wrapped = finalType; }; + nestedTypes.coercedType = coercedType; + nestedTypes.finalType = finalType; + }; + + # Augment the given type with an additional type check function. + addCheck = elemType: check: elemType // { check = x: elemType.check x && check x; }; + + }; +}; + +in outer_types // outer_types.types diff --git a/versions.nix b/versions.nix new file mode 100644 index 000000000..986e7e5f9 --- /dev/null +++ b/versions.nix @@ -0,0 +1,64 @@ +/* Version string functions. */ +{ lib }: + +rec { + + /* Break a version string into its component parts. + + Example: + splitVersion "1.2.3" + => ["1" "2" "3"] + */ + splitVersion = builtins.splitVersion or (lib.splitString "."); + + /* Get the major version string from a string. + + Example: + major "1.2.3" + => "1" + */ + major = v: builtins.elemAt (splitVersion v) 0; + + /* Get the minor version string from a string. + + Example: + minor "1.2.3" + => "2" + */ + minor = v: builtins.elemAt (splitVersion v) 1; + + /* Get the patch version string from a string. + + Example: + patch "1.2.3" + => "3" + */ + patch = v: builtins.elemAt (splitVersion v) 2; + + /* Get string of the first two parts (major and minor) + of a version string. + + Example: + majorMinor "1.2.3" + => "1.2" + */ + majorMinor = v: + builtins.concatStringsSep "." + (lib.take 2 (splitVersion v)); + + /* Pad a version string with zeros to match the given number of components. + + Example: + pad 3 "1.2" + => "1.2.0" + pad 3 "1.3-rc1" + => "1.3.0-rc1" + pad 3 "1.2.3.4" + => "1.2.3" + */ + pad = n: version: let + numericVersion = lib.head (lib.splitString "-" version); + versionSuffix = lib.removePrefix numericVersion version; + in lib.concatStringsSep "." (lib.take n (lib.splitVersion numericVersion ++ lib.genList (_: "0") n)) + versionSuffix; + +} diff --git a/zip-int-bits.nix b/zip-int-bits.nix new file mode 100644 index 000000000..53efd2bb0 --- /dev/null +++ b/zip-int-bits.nix @@ -0,0 +1,39 @@ +/* Helper function to implement a fallback for the bit operators + `bitAnd`, `bitOr` and `bitXor` on older nix version. + See ./trivial.nix +*/ +f: x: y: + let + # (intToBits 6) -> [ 0 1 1 ] + intToBits = x: + if x == 0 || x == -1 then + [] + else + let + headbit = if (x / 2) * 2 != x then 1 else 0; # x & 1 + tailbits = if x < 0 then ((x + 1) / 2) - 1 else x / 2; # x >> 1 + in + [headbit] ++ (intToBits tailbits); + + # (bitsToInt [ 0 1 1 ] 0) -> 6 + # (bitsToInt [ 0 1 0 ] 1) -> -6 + bitsToInt = l: signum: + if l == [] then + (if signum == 0 then 0 else -1) + else + (builtins.head l) + (2 * (bitsToInt (builtins.tail l) signum)); + + xsignum = if x < 0 then 1 else 0; + ysignum = if y < 0 then 1 else 0; + zipListsWith' = fst: snd: + if fst==[] && snd==[] then + [] + else if fst==[] then + [(f xsignum (builtins.head snd))] ++ (zipListsWith' [] (builtins.tail snd)) + else if snd==[] then + [(f (builtins.head fst) ysignum )] ++ (zipListsWith' (builtins.tail fst) [] ) + else + [(f (builtins.head fst) (builtins.head snd))] ++ (zipListsWith' (builtins.tail fst) (builtins.tail snd)); + in + assert (builtins.isInt x) && (builtins.isInt y); + bitsToInt (zipListsWith' (intToBits x) (intToBits y)) (f xsignum ysignum) From d8079ee3503b487af0769ad53398171e617869da Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 16 Aug 2023 16:16:58 +0200 Subject: [PATCH 183/909] Document jobCategory() --- src/libstore/build/goal.hh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index a313bf22c..d3127caea 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -162,6 +162,10 @@ public: virtual void cleanup() { } + /** + * @brief Hint for the scheduler, which concurrency limit applies. + * @see JobCategory + */ virtual JobCategory jobCategory() = 0; }; From 17ceec3a91a57c77c8df51f380f3a8a0c88460c4 Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Thu, 17 Aug 2023 13:03:43 +0200 Subject: [PATCH 184/909] Test repl formatting with and without :p --- tests/repl.sh | 58 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/tests/repl.sh b/tests/repl.sh index 2b3789521..bb8b60e50 100644 --- a/tests/repl.sh +++ b/tests/repl.sh @@ -54,11 +54,17 @@ testRepl # Same thing (kind-of), but with a remote store. testRepl --store "$TEST_ROOT/store?real=$NIX_STORE_DIR" -testReplResponse () { +# Remove ANSI escape sequences. They can prevent grep from finding a match. +stripColors () { + sed -E 's/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' +} + +testReplResponseGeneral () { + local grepMode="$1"; shift local commands="$1"; shift local expectedResponse="$1"; shift - local response="$(nix repl "$@" <<< "$commands")" - echo "$response" | grepQuiet -s "$expectedResponse" \ + local response="$(nix repl "$@" <<< "$commands" | stripColors)" + echo "$response" | grepQuiet "$grepMode" -s "$expectedResponse" \ || fail "repl command set: $commands @@ -69,7 +75,16 @@ $expectedResponse but with: -$response" +$response +" +} + +testReplResponse () { + testReplResponseGeneral --basic-regexp "$@" +} + +testReplResponseNoRegex () { + testReplResponseGeneral --fixed-strings "$@" } # :a uses the newest version of a symbol @@ -83,9 +98,9 @@ testReplResponse ' # note the escaped \, # \\ # because the second argument is a regex -testReplResponse ' +testReplResponseNoRegex ' "$" + "{hi}" -' '"\\${hi}"' +' '"\${hi}"' testReplResponse ' drvPath @@ -131,3 +146,34 @@ echo "changingThing" ) | nix repl ./flake --experimental-features 'flakes repl-flake') echo "$replResult" | grepQuiet -s beforeChange echo "$replResult" | grepQuiet -s afterChange + +# Test recursive printing and formatting +# Normal output should print attributes in lexicographical order non-recursively +testReplResponseNoRegex ' +{ a = { b = 2; }; l = [ 1 2 3 ]; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } +' '{ a = { ... }; l = [ ... ]; n = 1234; s = "string"; x = { ... }; }' + +# Same for lists, but order is preserved +testReplResponseNoRegex ' +[ 42 1 "thingy" ({ a = 1; }) ([ 1 2 3 ]) ] +' '[ 42 1 "thingy" { ... } [ ... ] ]' + +# Same for let expressions +testReplResponseNoRegex ' +let x = { y = { a = 1; }; inherit x; }; in x +' '{ x = { ... }; y = { ... }; }' + +# The :p command should recursively print sets, but prevent infinite recursion +testReplResponseNoRegex ' +:p { a = { b = 2; }; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } +' '{ a = { b = 2; }; n = 1234; s = "string"; x = { y = { z = { y = «repeated»; }; }; }; }' + +# Same for lists +testReplResponseNoRegex ' +:p [ 42 1 "thingy" (rec { a = 1; b = { inherit a; inherit b; }; }) ([ 1 2 3 ]) ] +' '[ 42 1 "thingy" { a = 1; b = { a = 1; b = «repeated»; }; } [ 1 2 3 ] ]' + +# Same for let expressions +testReplResponseNoRegex ' +:p let x = { y = { a = 1; }; inherit x; }; in x +' '{ x = { x = «repeated»; y = { a = 1; }; }; y = «repeated»; }' From 1d7a57cfd9bf40658e2f35c13146d746a1011020 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Thu, 17 Aug 2023 13:40:57 -0700 Subject: [PATCH 185/909] libexpr/tests: test that parseFlakeRef doesn't percent-encode twice --- src/libexpr/tests/flakeref.cc | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 src/libexpr/tests/flakeref.cc diff --git a/src/libexpr/tests/flakeref.cc b/src/libexpr/tests/flakeref.cc new file mode 100644 index 000000000..2b7809b93 --- /dev/null +++ b/src/libexpr/tests/flakeref.cc @@ -0,0 +1,22 @@ +#include + +#include "flake/flakeref.hh" + +namespace nix { + +/* ----------- tests for flake/flakeref.hh --------------------------------------------------*/ + + /* ---------------------------------------------------------------------------- + * to_string + * --------------------------------------------------------------------------*/ + + TEST(to_string, doesntReencodeUrl) { + auto s = "http://localhost:8181/test/+3d.tar.gz"; + auto flakeref = parseFlakeRef(s); + auto parsed = flakeref.to_string(); + auto expected = "http://localhost:8181/test/%2B3d.tar.gz"; + + ASSERT_EQ(parsed, expected); + } + +} From 73696ec71678433dac87863bab36b66701d4b6ed Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Thu, 17 Aug 2023 13:21:38 -0700 Subject: [PATCH 186/909] libutil: fix double-encoding of URLs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If you have a URL that needs to be percent-encoded, such as `http://localhost:8181/test/+3d.tar.gz`, and try to lock that in a Nix flake such as the following: { inputs.test = { url = "http://localhost:8181/test/+3d.tar.gz"; flake = false; }; outputs = { test, ... }: { t = builtins.readFile test; }; } running `nix flake metadata` shows that the input URL has been incorrectly double-encoded (despite the flake.lock being correctly encoded only once): [...snip...] Inputs: └───test: http://localhost:8181/test/%252B3d.tar.gz?narHash=sha256-EFUdrtf6Rn0LWIJufrmg8q99aT3jGfLvd1//zaJEufY%3D (Notice the `%252B`? That's just `%2B` but percent-encoded again) With this patch, the double-encoding is gone; running `nix flake metadata` will show the proper URL: [...snip...] Inputs: └───test: http://localhost:8181/test/%2B3d.tar.gz?narHash=sha256-EFUdrtf6Rn0LWIJufrmg8q99aT3jGfLvd1//zaJEufY%3D --- As far as I can tell, this happens because Nix already percent-encodes the URL and stores this as the value of `inputs.asdf.url`. However, when Nix later tries to read this out of the eval state as a string (via `getStrAttr`), it has to run it through `parseURL` again to get the `ParsedURL` structure. Now, this itself isn't a problem -- the true problem arises when using `ParsedURL::to_string` later, which then _re-escapes the path_. It is at this point that what would have been `%2B` (`+`) becomes `%252B` (`%2B`). --- src/libutil/url.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 9e44241ac..a8f7d39fd 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -44,7 +44,7 @@ ParsedURL parseURL(const std::string & url) .base = base, .scheme = scheme, .authority = authority, - .path = path, + .path = percentDecode(path), .query = decodeQuery(query), .fragment = percentDecode(std::string(fragment)) }; From 75243c96937b472665b94729df81f8296b262085 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 18 Aug 2023 14:46:13 +0200 Subject: [PATCH 187/909] test/flakes/follow-paths.sh: Quote Co-authored-by: Alex Ameen --- tests/flakes/follow-paths.sh | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/flakes/follow-paths.sh b/tests/flakes/follow-paths.sh index f9d5c8c80..dc97027ac 100644 --- a/tests/flakes/follow-paths.sh +++ b/tests/flakes/follow-paths.sh @@ -146,8 +146,8 @@ EOF git -C $flakeFollowsA add flake.nix -nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid'" -nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid2'" +nix flake lock "$flakeFollowsA" 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid'" +nix flake lock "$flakeFollowsA" 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid2'" # Now test follow path overloading # This tests a lockfile checking regression https://github.com/NixOS/nix/pull/8819 @@ -169,18 +169,18 @@ nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override fo # error: input 'B/D' follows a non-existent input 'B/C/D' # # Note that for `B` to resolve its follow for `D`, it needs `C/D`, for which it needs to resolve the follow on `C` first. -flakeFollowsOverloadA=$TEST_ROOT/follows/overload/flakeA -flakeFollowsOverloadB=$TEST_ROOT/follows/overload/flakeA/flakeB -flakeFollowsOverloadC=$TEST_ROOT/follows/overload/flakeA/flakeB/flakeC -flakeFollowsOverloadD=$TEST_ROOT/follows/overload/flakeA/flakeB/flakeC/flakeD +flakeFollowsOverloadA="$TEST_ROOT/follows/overload/flakeA" +flakeFollowsOverloadB="$TEST_ROOT/follows/overload/flakeA/flakeB" +flakeFollowsOverloadC="$TEST_ROOT/follows/overload/flakeA/flakeB/flakeC" +flakeFollowsOverloadD="$TEST_ROOT/follows/overload/flakeA/flakeB/flakeC/flakeD" # Test following path flakerefs. -createGitRepo $flakeFollowsOverloadA -mkdir -p $flakeFollowsOverloadB -mkdir -p $flakeFollowsOverloadC -mkdir -p $flakeFollowsOverloadD +createGitRepo "$flakeFollowsOverloadA" +mkdir -p "$flakeFollowsOverloadB" +mkdir -p "$flakeFollowsOverloadC" +mkdir -p "$flakeFollowsOverloadD" -cat > $flakeFollowsOverloadD/flake.nix < "$flakeFollowsOverloadD/flake.nix" < $flakeFollowsOverloadD/flake.nix < $flakeFollowsOverloadC/flake.nix < "$flakeFollowsOverloadC/flake.nix" < $flakeFollowsOverloadC/flake.nix < $flakeFollowsOverloadB/flake.nix < "$flakeFollowsOverloadB/flake.nix" < $flakeFollowsOverloadB/flake.nix < $flakeFollowsOverloadA/flake.nix < "$flakeFollowsOverloadA/flake.nix" < $flakeFollowsOverloadA/flake.nix < Date: Wed, 16 Aug 2023 12:29:23 -0400 Subject: [PATCH 188/909] Fixing #7479 Types converted: - `NixStringContextElem` - `OutputsSpec` - `ExtendedOutputsSpec` - `DerivationOutput` - `DerivationType` Existing ones mostly conforming the pattern cleaned up: - `ContentAddressMethod` - `ContentAddressWithReferences` The `DerivationGoal::derivationType` field had a bogus initialization, now caught, so I made it `std::optional`. I think #8829 can make it non-optional again because it will ensure we always have the derivation when we construct a `DerivationGoal`. See that issue (#7479) for details on the general goal. `git grep 'Raw::Raw'` indicates the two types I didn't yet convert `DerivedPath` and `BuiltPath` (and their `Single` variants) . This is because @roberth and I (can't find issue right now...) plan on reworking them somewhat, so I didn't want to churn them more just yet. Co-authored-by: Eelco Dolstra --- src/libcmd/installable-attr-path.cc | 5 +- src/libcmd/installable-derived-path.cc | 2 +- src/libcmd/installable-flake.cc | 2 +- src/libcmd/installables.cc | 6 +- src/libexpr/eval-cache.cc | 2 +- src/libexpr/eval.cc | 2 +- src/libexpr/flake/flakeref.cc | 2 +- src/libexpr/primops.cc | 14 +- src/libexpr/primops/context.cc | 6 +- src/libexpr/tests/value/context.cc | 8 +- src/libexpr/value/context.cc | 2 +- src/libexpr/value/context.hh | 82 +++--- src/libstore/build/derivation-goal.cc | 7 +- src/libstore/build/derivation-goal.hh | 2 +- src/libstore/build/local-derivation-goal.cc | 18 +- src/libstore/build/worker.cc | 5 +- src/libstore/content-address.cc | 2 +- src/libstore/content-address.hh | 12 +- src/libstore/derivations.cc | 36 +-- src/libstore/derivations.hh | 311 ++++++++++---------- src/libstore/misc.cc | 8 +- src/libstore/outputs-spec.cc | 20 +- src/libstore/outputs-spec.hh | 95 +++--- src/libstore/path-with-outputs.cc | 2 +- src/libutil/variant-wrapper.hh | 30 ++ src/nix/app.cc | 2 +- src/nix/bundle.cc | 2 +- src/nix/develop.cc | 2 +- src/nix/flake.cc | 2 +- 29 files changed, 355 insertions(+), 334 deletions(-) create mode 100644 src/libutil/variant-wrapper.hh diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index 2f89eee02..06e507872 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -80,7 +80,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { return e; }, - }, extendedOutputsSpec.raw()); + }, extendedOutputsSpec.raw); auto [iter, didInsert] = byDrvPath.emplace(*drvPath, newOutputs); @@ -96,6 +96,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() .outputs = outputs, }, .info = make_ref(ExtraPathInfoValue::Value { + .extendedOutputsSpec = outputs, /* FIXME: reconsider backwards compatibility above so we can fill in this info. */ }), @@ -114,7 +115,7 @@ InstallableAttrPath InstallableAttrPath::parse( return { state, cmd, v, prefix == "." ? "" : std::string { prefix }, - extendedOutputsSpec + std::move(extendedOutputsSpec), }; } diff --git a/src/libcmd/installable-derived-path.cc b/src/libcmd/installable-derived-path.cc index b45641e8a..4d1f83a1c 100644 --- a/src/libcmd/installable-derived-path.cc +++ b/src/libcmd/installable-derived-path.cc @@ -55,7 +55,7 @@ InstallableDerivedPath InstallableDerivedPath::parse( .outputs = outputSpec, }; }, - }, extendedOutputsSpec.raw()); + }, extendedOutputsSpec.raw); return InstallableDerivedPath { store, std::move(derivedPath), diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 1b169c3bd..4074da06d 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -141,7 +141,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { return e; }, - }, extendedOutputsSpec.raw()), + }, extendedOutputsSpec.raw), }, .info = make_ref( ExtraPathInfoValue::Value { diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 4c7d134ec..eb1903084 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -459,7 +459,7 @@ Installables SourceExprCommand::parseInstallables( result.push_back( make_ref( InstallableAttrPath::parse( - state, *this, vFile, prefix, extendedOutputsSpec))); + state, *this, vFile, std::move(prefix), std::move(extendedOutputsSpec)))); } } else { @@ -475,7 +475,7 @@ Installables SourceExprCommand::parseInstallables( if (prefix.find('/') != std::string::npos) { try { result.push_back(make_ref( - InstallableDerivedPath::parse(store, prefix, extendedOutputsSpec))); + InstallableDerivedPath::parse(store, prefix, extendedOutputsSpec.raw))); continue; } catch (BadStorePath &) { } catch (...) { @@ -491,7 +491,7 @@ Installables SourceExprCommand::parseInstallables( getEvalState(), std::move(flakeRef), fragment, - extendedOutputsSpec, + std::move(extendedOutputsSpec), getDefaultFlakeAttrPaths(), getDefaultFlakeAttrPathPrefixes(), lockFlags)); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 6a60ba87b..2c9aa5532 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -604,7 +604,7 @@ string_t AttrCursor::getStringWithContext() [&](const NixStringContextElem::Opaque & o) -> const StorePath & { return o.path; }, - }, c.raw()); + }, c.raw); if (!root->state.store->isValidPath(path)) { valid = false; break; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 79e2c4727..7e839dbe7 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2364,7 +2364,7 @@ std::pair EvalState::coerceToSingleDerivedP [&](NixStringContextElem::Built && b) -> SingleDerivedPath { return std::move(b); }, - }, ((NixStringContextElem &&) *context.begin()).raw()); + }, ((NixStringContextElem &&) *context.begin()).raw); return { std::move(derivedPath), std::move(s), diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index d3fa1d557..e1bce90bc 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -246,7 +246,7 @@ std::tuple parseFlakeRefWithFragment { auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(url); auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, baseDir, allowMissing, isFlake); - return {std::move(flakeRef), fragment, extendedOutputsSpec}; + return {std::move(flakeRef), fragment, std::move(extendedOutputsSpec)}; } } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 283f99a48..5067da449 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -69,7 +69,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context) res.insert_or_assign(ctxS, ctxS); ensureValid(d.drvPath); }, - }, c.raw()); + }, c.raw); } if (drvs.empty()) return {}; @@ -1265,7 +1265,7 @@ drvName, Bindings * attrs, Value & v) [&](const NixStringContextElem::Opaque & o) { drv.inputSrcs.insert(o.path); }, - }, c.raw()); + }, c.raw); } /* Do we have all required attributes? */ @@ -1334,13 +1334,13 @@ drvName, Bindings * attrs, Value & v) if (isImpure) drv.outputs.insert_or_assign(i, DerivationOutput::Impure { - .method = method.raw, + .method = method, .hashType = ht, }); else drv.outputs.insert_or_assign(i, DerivationOutput::CAFloating { - .method = method.raw, + .method = method, .hashType = ht, }); } @@ -1373,7 +1373,7 @@ drvName, Bindings * attrs, Value & v) drv.env[i] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign( i, - DerivationOutputInputAddressed { + DerivationOutput::InputAddressed { .path = std::move(outPath), }); } @@ -1381,7 +1381,7 @@ drvName, Bindings * attrs, Value & v) ; case DrvHash::Kind::Deferred: for (auto & i : outputs) { - drv.outputs.insert_or_assign(i, DerivationOutputDeferred {}); + drv.outputs.insert_or_assign(i, DerivationOutput::Deferred {}); } } } @@ -2054,7 +2054,7 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val StorePathSet refs; for (auto c : context) { - if (auto p = std::get_if(&c)) + if (auto p = std::get_if(&c.raw)) refs.insert(p->path); else state.debugThrowLastTrace(EvalError({ diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index bfc731744..e8542503a 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -51,13 +51,13 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p NixStringContext context2; for (auto && c : context) { - if (auto * ptr = std::get_if(&c)) { + if (auto * ptr = std::get_if(&c.raw)) { context2.emplace(NixStringContextElem::Opaque { .path = ptr->drvPath }); } else { /* Can reuse original item */ - context2.emplace(std::move(c)); + context2.emplace(std::move(c).raw); } } @@ -114,7 +114,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, [&](NixStringContextElem::Opaque && o) { contextInfos[std::move(o.path)].path = true; }, - }, ((NixStringContextElem &&) i).raw()); + }, ((NixStringContextElem &&) i).raw); } auto attrs = state.buildBindings(contextInfos.size()); diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index c56b50b59..19407d071 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -47,7 +47,7 @@ TEST(NixStringContextElemTest, slash_invalid) { TEST(NixStringContextElemTest, opaque) { std::string_view opaque = "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; auto elem = NixStringContextElem::parse(opaque); - auto * p = std::get_if(&elem); + auto * p = std::get_if(&elem.raw); ASSERT_TRUE(p); ASSERT_EQ(p->path, StorePath { opaque }); ASSERT_EQ(elem.to_string(), opaque); @@ -60,7 +60,7 @@ TEST(NixStringContextElemTest, opaque) { TEST(NixStringContextElemTest, drvDeep) { std::string_view drvDeep = "=g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; auto elem = NixStringContextElem::parse(drvDeep); - auto * p = std::get_if(&elem); + auto * p = std::get_if(&elem.raw); ASSERT_TRUE(p); ASSERT_EQ(p->drvPath, StorePath { drvDeep.substr(1) }); ASSERT_EQ(elem.to_string(), drvDeep); @@ -73,7 +73,7 @@ TEST(NixStringContextElemTest, drvDeep) { TEST(NixStringContextElemTest, built_opaque) { std::string_view built = "!foo!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; auto elem = NixStringContextElem::parse(built); - auto * p = std::get_if(&elem); + auto * p = std::get_if(&elem.raw); ASSERT_TRUE(p); ASSERT_EQ(p->output, "foo"); ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { @@ -96,7 +96,7 @@ TEST(NixStringContextElemTest, built_built) { std::string_view built = "!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; auto elem = NixStringContextElem::parse(built, mockXpSettings); - auto * p = std::get_if(&elem); + auto * p = std::get_if(&elem.raw); ASSERT_TRUE(p); ASSERT_EQ(p->output, "foo"); auto * drvPath = std::get_if(&*p->drvPath); diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index d8116011e..22361d8fa 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -99,7 +99,7 @@ std::string NixStringContextElem::to_string() const res += '='; res += d.drvPath.to_string(); }, - }, raw()); + }, raw); return res; } diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index a1b71695b..9f1d59317 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -4,8 +4,7 @@ #include "util.hh" #include "comparator.hh" #include "derived-path.hh" - -#include +#include "variant-wrapper.hh" #include @@ -26,58 +25,47 @@ public: } }; -/** - * Plain opaque path to some store object. - * - * Encoded as just the path: ‘’. - */ -typedef SingleDerivedPath::Opaque NixStringContextElem_Opaque; +struct NixStringContextElem { + /** + * Plain opaque path to some store object. + * + * Encoded as just the path: ‘’. + */ + using Opaque = SingleDerivedPath::Opaque; -/** - * Path to a derivation and its entire build closure. - * - * The path doesn't just refer to derivation itself and its closure, but - * also all outputs of all derivations in that closure (including the - * root derivation). - * - * Encoded in the form ‘=’. - */ -struct NixStringContextElem_DrvDeep { - StorePath drvPath; + /** + * Path to a derivation and its entire build closure. + * + * The path doesn't just refer to derivation itself and its closure, but + * also all outputs of all derivations in that closure (including the + * root derivation). + * + * Encoded in the form ‘=’. + */ + struct DrvDeep { + StorePath drvPath; - GENERATE_CMP(NixStringContextElem_DrvDeep, me->drvPath); -}; + GENERATE_CMP(DrvDeep, me->drvPath); + }; -/** - * Derivation output. - * - * Encoded in the form ‘!!’. - */ -typedef SingleDerivedPath::Built NixStringContextElem_Built; + /** + * Derivation output. + * + * Encoded in the form ‘!!’. + */ + using Built = SingleDerivedPath::Built; -using _NixStringContextElem_Raw = std::variant< - NixStringContextElem_Opaque, - NixStringContextElem_DrvDeep, - NixStringContextElem_Built ->; + using Raw = std::variant< + Opaque, + DrvDeep, + Built + >; -struct NixStringContextElem : _NixStringContextElem_Raw { - using Raw = _NixStringContextElem_Raw; - using Raw::Raw; + Raw raw; - using Opaque = NixStringContextElem_Opaque; - using DrvDeep = NixStringContextElem_DrvDeep; - using Built = NixStringContextElem_Built; + GENERATE_CMP(NixStringContextElem, me->raw); - inline const Raw & raw() const & { - return static_cast(*this); - } - inline Raw & raw() & { - return static_cast(*this); - } - inline Raw && raw() && { - return static_cast(*this); - } + MAKE_WRAPPER_CONSTRUCTOR(NixStringContextElem); /** * Decode a context string, one of: diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 8bdf2f367..84da7f2e1 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -521,7 +521,7 @@ void DerivationGoal::inputsRealised() [&](const DerivationType::Impure &) { return true; } - }, drvType.raw()); + }, drvType.raw); if (resolveDrv && !fullDrv.inputDrvs.empty()) { experimentalFeatureSettings.require(Xp::CaDerivations); @@ -996,10 +996,11 @@ void DerivationGoal::buildDone() } else { + assert(derivationType); st = dynamic_cast(&e) ? BuildResult::NotDeterministic : statusOk(status) ? BuildResult::OutputRejected : - !derivationType.isSandboxed() || diskFull ? BuildResult::TransientFailure : + !derivationType->isSandboxed() || diskFull ? BuildResult::TransientFailure : BuildResult::PermanentFailure; } @@ -1358,7 +1359,7 @@ std::pair DerivationGoal::checkPathValidity() [&](const OutputsSpec::Names & names) { return static_cast(names); }, - }, wantedOutputs.raw()); + }, wantedOutputs.raw); SingleDrvOutputs validOutputs; for (auto & i : queryPartialDerivationOutputMap()) { diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index ee8f06f25..9d6fe1c0f 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -184,7 +184,7 @@ struct DerivationGoal : public Goal /** * The sort of derivation we are building. */ - DerivationType derivationType; + std::optional derivationType; typedef void (DerivationGoal::*GoalState)(); GoalState state; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 920097680..78f943d1f 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -178,6 +178,8 @@ void LocalDerivationGoal::tryLocalBuild() return; } + assert(derivationType); + /* Are we doing a chroot build? */ { auto noChroot = parsedDrv->getBoolAttr("__noChroot"); @@ -195,7 +197,7 @@ void LocalDerivationGoal::tryLocalBuild() else if (settings.sandboxMode == smDisabled) useChroot = false; else if (settings.sandboxMode == smRelaxed) - useChroot = derivationType.isSandboxed() && !noChroot; + useChroot = derivationType->isSandboxed() && !noChroot; } auto & localStore = getLocalStore(); @@ -689,7 +691,7 @@ void LocalDerivationGoal::startBuilder() "nogroup:x:65534:\n", sandboxGid())); /* Create /etc/hosts with localhost entry. */ - if (derivationType.isSandboxed()) + if (derivationType->isSandboxed()) writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); /* Make the closure of the inputs available in the chroot, @@ -893,7 +895,7 @@ void LocalDerivationGoal::startBuilder() us. */ - if (derivationType.isSandboxed()) + if (derivationType->isSandboxed()) privateNetwork = true; userNamespaceSync.create(); @@ -1121,7 +1123,7 @@ void LocalDerivationGoal::initEnv() derivation, tell the builder, so that for instance `fetchurl' can skip checking the output. On older Nixes, this environment variable won't be set, so `fetchurl' will do the check. */ - if (derivationType.isFixed()) env["NIX_OUTPUT_CHECKED"] = "1"; + if (derivationType->isFixed()) env["NIX_OUTPUT_CHECKED"] = "1"; /* *Only* if this is a fixed-output derivation, propagate the values of the environment variables specified in the @@ -1132,7 +1134,7 @@ void LocalDerivationGoal::initEnv() to the builder is generally impure, but the output of fixed-output derivations is by definition pure (since we already know the cryptographic hash of the output). */ - if (!derivationType.isSandboxed()) { + if (!derivationType->isSandboxed()) { for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) env[i] = getEnv(i).value_or(""); } @@ -1797,7 +1799,7 @@ void LocalDerivationGoal::runChild() /* Fixed-output derivations typically need to access the network, so give them access to /etc/resolv.conf and so on. */ - if (!derivationType.isSandboxed()) { + if (!derivationType->isSandboxed()) { // Only use nss functions to resolve hosts and // services. Don’t use it for anything else that may // be configured for this system. This limits the @@ -2048,7 +2050,7 @@ void LocalDerivationGoal::runChild() #include "sandbox-defaults.sb" ; - if (!derivationType.isSandboxed()) + if (!derivationType->isSandboxed()) sandboxProfile += #include "sandbox-network.sb" ; @@ -2599,7 +2601,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() }); }, - }, output->raw()); + }, output->raw); /* FIXME: set proper permissions in restorePath() so we don't have to do another traversal. */ diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 6779dbcf3..b58fc5c1c 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -268,7 +268,10 @@ void Worker::run(const Goals & _topGoals) for (auto & i : _topGoals) { topGoals.insert(i); if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(DerivedPath::Built{makeConstantStorePathRef(goal->drvPath), goal->wantedOutputs}); + topPaths.push_back(DerivedPath::Built { + .drvPath = makeConstantStorePathRef(goal->drvPath), + .outputs = goal->wantedOutputs, + }); } else if (auto goal = dynamic_cast(i.get())) { topPaths.push_back(DerivedPath::Opaque{goal->storePath}); } diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 080456e18..e290a8d38 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -115,7 +115,7 @@ ContentAddress ContentAddress::parse(std::string_view rawCa) auto [caMethod, hashType] = parseContentAddressMethodPrefix(rest); return ContentAddress { - .method = std::move(caMethod).raw, + .method = std::move(caMethod), .hash = Hash::parseNonSRIUnprefixed(rest, hashType), }; } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 01b771e52..c4d619bdc 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -5,6 +5,7 @@ #include "hash.hh" #include "path.hh" #include "comparator.hh" +#include "variant-wrapper.hh" namespace nix { @@ -71,11 +72,7 @@ struct ContentAddressMethod GENERATE_CMP(ContentAddressMethod, me->raw); - /* The moral equivalent of `using Raw::Raw;` */ - ContentAddressMethod(auto &&... arg) - : raw(std::forward(arg)...) - { } - + MAKE_WRAPPER_CONSTRUCTOR(ContentAddressMethod); /** * Parse the prefix tag which indicates how the files @@ -252,10 +249,7 @@ struct ContentAddressWithReferences GENERATE_CMP(ContentAddressWithReferences, me->raw); - /* The moral equivalent of `using Raw::Raw;` */ - ContentAddressWithReferences(auto &&... arg) - : raw(std::forward(arg)...) - { } + MAKE_WRAPPER_CONSTRUCTOR(ContentAddressWithReferences); /** * Create a `ContentAddressWithReferences` from a mere diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index f4e4980c2..0b8bdaf1c 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -32,7 +32,7 @@ std::optional DerivationOutput::path(const Store & store, std::string [](const DerivationOutput::Impure &) -> std::optional { return std::nullopt; }, - }, raw()); + }, raw); } @@ -60,7 +60,7 @@ bool DerivationType::isCA() const [](const Impure &) { return true; }, - }, raw()); + }, raw); } bool DerivationType::isFixed() const @@ -75,7 +75,7 @@ bool DerivationType::isFixed() const [](const Impure &) { return false; }, - }, raw()); + }, raw); } bool DerivationType::hasKnownOutputPaths() const @@ -90,7 +90,7 @@ bool DerivationType::hasKnownOutputPaths() const [](const Impure &) { return false; }, - }, raw()); + }, raw); } @@ -106,7 +106,7 @@ bool DerivationType::isSandboxed() const [](const Impure &) { return false; }, - }, raw()); + }, raw); } @@ -122,7 +122,7 @@ bool DerivationType::isPure() const [](const Impure &) { return false; }, - }, raw()); + }, raw); } @@ -408,13 +408,13 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, ""); }, - [&](const DerivationOutputImpure & doi) { + [&](const DerivationOutput::Impure & doi) { // FIXME s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashType(doi.hashType)); s += ','; printUnquotedString(s, "impure"); } - }, i.second.raw()); + }, i.second.raw); s += ')'; } @@ -509,7 +509,7 @@ DerivationType BasicDerivation::type() const [&](const DerivationOutput::Impure &) { impureOutputs.insert(i.first); }, - }, i.second.raw()); + }, i.second.raw); } if (inputAddressedOutputs.empty() @@ -626,7 +626,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut if (type.isFixed()) { std::map outputHashes; for (const auto & i : drv.outputs) { - auto & dof = std::get(i.second.raw()); + auto & dof = std::get(i.second.raw); auto hash = hashString(htSHA256, "fixed:out:" + dof.ca.printMethodAlgo() + ":" + dof.ca.hash.to_string(Base16, false) + ":" @@ -663,7 +663,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut [](const DerivationType::Impure &) -> DrvHash::Kind { assert(false); } - }, drv.type().raw()); + }, drv.type().raw); std::map inputs2; for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) { @@ -720,10 +720,10 @@ StringSet BasicDerivation::outputNames() const DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const { DerivationOutputsAndOptPaths outsAndOptPaths; - for (auto output : outputs) + for (auto & [outputName, output] : outputs) outsAndOptPaths.insert(std::make_pair( - output.first, - std::make_pair(output.second, output.second.path(store, name, output.first)) + outputName, + std::make_pair(output, output.path(store, name, outputName)) ) ); return outsAndOptPaths; @@ -798,7 +798,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr << (doi.method.renderPrefix() + printHashType(doi.hashType)) << "impure"; }, - }, i.second.raw()); + }, i.second.raw); } WorkerProto::write(store, WorkerProto::WriteConn { .to = out }, @@ -840,7 +840,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); for (auto & [outputName, output] : drv.outputs) { - if (std::holds_alternative(output.raw())) { + if (std::holds_alternative(output.raw)) { auto h = get(hashModulo.hashes, outputName); if (!h) throw Error("derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)", @@ -955,7 +955,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const [&](const DerivationOutput::Impure &) { /* Nothing to check */ }, - }, i.second.raw()); + }, i.second.raw); } } @@ -984,7 +984,7 @@ nlohmann::json DerivationOutput::toJSON( res["hashAlgo"] = doi.method.renderPrefix() + printHashType(doi.hashType); res["impure"] = true; }, - }, raw()); + }, raw); return res; } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index fa79f77fd..a92082089 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -9,6 +9,7 @@ #include "derived-path.hh" #include "sync.hh" #include "comparator.hh" +#include "variant-wrapper.hh" #include #include @@ -20,108 +21,110 @@ class Store; /* Abstract syntax of derivations. */ -/** - * The traditional non-fixed-output derivation type. - */ -struct DerivationOutputInputAddressed -{ - StorePath path; - - GENERATE_CMP(DerivationOutputInputAddressed, me->path); -}; - -/** - * Fixed-output derivations, whose output paths are content - * addressed according to that fixed output. - */ -struct DerivationOutputCAFixed -{ - /** - * Method and hash used for expected hash computation. - * - * References are not allowed by fiat. - */ - ContentAddress ca; - - /** - * Return the \ref StorePath "store path" corresponding to this output - * - * @param drvName The name of the derivation this is an output of, without the `.drv`. - * @param outputName The name of this output. - */ - StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; - - GENERATE_CMP(DerivationOutputCAFixed, me->ca); -}; - -/** - * Floating-output derivations, whose output paths are content - * addressed, but not fixed, and so are dynamically calculated from - * whatever the output ends up being. - * */ -struct DerivationOutputCAFloating -{ - /** - * How the file system objects will be serialized for hashing - */ - ContentAddressMethod method; - - /** - * How the serialization will be hashed - */ - HashType hashType; - - GENERATE_CMP(DerivationOutputCAFloating, me->method, me->hashType); -}; - -/** - * Input-addressed output which depends on a (CA) derivation whose hash - * isn't known yet. - */ -struct DerivationOutputDeferred { - GENERATE_CMP(DerivationOutputDeferred); -}; - -/** - * Impure output which is moved to a content-addressed location (like - * CAFloating) but isn't registered as a realization. - */ -struct DerivationOutputImpure -{ - /** - * How the file system objects will be serialized for hashing - */ - ContentAddressMethod method; - - /** - * How the serialization will be hashed - */ - HashType hashType; - - GENERATE_CMP(DerivationOutputImpure, me->method, me->hashType); -}; - -typedef std::variant< - DerivationOutputInputAddressed, - DerivationOutputCAFixed, - DerivationOutputCAFloating, - DerivationOutputDeferred, - DerivationOutputImpure -> _DerivationOutputRaw; - /** * A single output of a BasicDerivation (and Derivation). */ -struct DerivationOutput : _DerivationOutputRaw +struct DerivationOutput { - using Raw = _DerivationOutputRaw; - using Raw::Raw; + /** + * The traditional non-fixed-output derivation type. + */ + struct InputAddressed + { + StorePath path; - using InputAddressed = DerivationOutputInputAddressed; - using CAFixed = DerivationOutputCAFixed; - using CAFloating = DerivationOutputCAFloating; - using Deferred = DerivationOutputDeferred; - using Impure = DerivationOutputImpure; + GENERATE_CMP(InputAddressed, me->path); + }; + + /** + * Fixed-output derivations, whose output paths are content + * addressed according to that fixed output. + */ + struct CAFixed + { + /** + * Method and hash used for expected hash computation. + * + * References are not allowed by fiat. + */ + ContentAddress ca; + + /** + * Return the \ref StorePath "store path" corresponding to this output + * + * @param drvName The name of the derivation this is an output of, without the `.drv`. + * @param outputName The name of this output. + */ + StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; + + GENERATE_CMP(CAFixed, me->ca); + }; + + /** + * Floating-output derivations, whose output paths are content + * addressed, but not fixed, and so are dynamically calculated from + * whatever the output ends up being. + * */ + struct CAFloating + { + /** + * How the file system objects will be serialized for hashing + */ + ContentAddressMethod method; + + /** + * How the serialization will be hashed + */ + HashType hashType; + + GENERATE_CMP(CAFloating, me->method, me->hashType); + }; + + /** + * Input-addressed output which depends on a (CA) derivation whose hash + * isn't known yet. + */ + struct Deferred { + GENERATE_CMP(Deferred); + }; + + /** + * Impure output which is moved to a content-addressed location (like + * CAFloating) but isn't registered as a realization. + */ + struct Impure + { + /** + * How the file system objects will be serialized for hashing + */ + ContentAddressMethod method; + + /** + * How the serialization will be hashed + */ + HashType hashType; + + GENERATE_CMP(Impure, me->method, me->hashType); + }; + + typedef std::variant< + InputAddressed, + CAFixed, + CAFloating, + Deferred, + Impure + > Raw; + + Raw raw; + + GENERATE_CMP(DerivationOutput, me->raw); + + MAKE_WRAPPER_CONSTRUCTOR(DerivationOutput); + + /** + * Force choosing a variant + */ + DerivationOutput() = delete; /** * \note when you use this function you should make sure that you're @@ -131,10 +134,6 @@ struct DerivationOutput : _DerivationOutputRaw */ std::optional path(const Store & store, std::string_view drvName, std::string_view outputName) const; - inline const Raw & raw() const { - return static_cast(*this); - } - nlohmann::json toJSON( const Store & store, std::string_view drvName, @@ -167,61 +166,71 @@ typedef std::map DerivationInputs; -/** - * Input-addressed derivation types - */ -struct DerivationType_InputAddressed { +struct DerivationType { /** - * True iff the derivation type can't be determined statically, - * for instance because it (transitively) depends on a content-addressed - * derivation. - */ - bool deferred; -}; - -/** - * Content-addressed derivation types - */ -struct DerivationType_ContentAddressed { - /** - * Whether the derivation should be built safely inside a sandbox. + * Input-addressed derivation types */ - bool sandboxed; + struct InputAddressed { + /** + * True iff the derivation type can't be determined statically, + * for instance because it (transitively) depends on a content-addressed + * derivation. + */ + bool deferred; + + GENERATE_CMP(InputAddressed, me->deferred); + }; + /** - * Whether the derivation's outputs' content-addresses are "fixed" - * or "floating. - * - * - Fixed: content-addresses are written down as part of the - * derivation itself. If the outputs don't end up matching the - * build fails. - * - * - Floating: content-addresses are not written down, we do not - * know them until we perform the build. + * Content-addressed derivation types */ - bool fixed; -}; + struct ContentAddressed { + /** + * Whether the derivation should be built safely inside a sandbox. + */ + bool sandboxed; + /** + * Whether the derivation's outputs' content-addresses are "fixed" + * or "floating". + * + * - Fixed: content-addresses are written down as part of the + * derivation itself. If the outputs don't end up matching the + * build fails. + * + * - Floating: content-addresses are not written down, we do not + * know them until we perform the build. + */ + bool fixed; -/** - * Impure derivation type - * - * This is similar at buil-time to the content addressed, not standboxed, not fixed - * type, but has some restrictions on its usage. - */ -struct DerivationType_Impure { -}; + GENERATE_CMP(ContentAddressed, me->sandboxed, me->fixed); + }; -typedef std::variant< - DerivationType_InputAddressed, - DerivationType_ContentAddressed, - DerivationType_Impure -> _DerivationTypeRaw; + /** + * Impure derivation type + * + * This is similar at buil-time to the content addressed, not standboxed, not fixed + * type, but has some restrictions on its usage. + */ + struct Impure { + GENERATE_CMP(Impure); + }; -struct DerivationType : _DerivationTypeRaw { - using Raw = _DerivationTypeRaw; - using Raw::Raw; - using InputAddressed = DerivationType_InputAddressed; - using ContentAddressed = DerivationType_ContentAddressed; - using Impure = DerivationType_Impure; + typedef std::variant< + InputAddressed, + ContentAddressed, + Impure + > Raw; + + Raw raw; + + GENERATE_CMP(DerivationType, me->raw); + + MAKE_WRAPPER_CONSTRUCTOR(DerivationType); + + /** + * Force choosing a variant + */ + DerivationType() = delete; /** * Do the outputs of the derivation have paths calculated from their @@ -257,10 +266,6 @@ struct DerivationType : _DerivationTypeRaw { * closure, or if fixed output. */ bool hasKnownOutputPaths() const; - - inline const Raw & raw() const { - return static_cast(*this); - } }; struct BasicDerivation diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 185d61c15..631213306 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -88,7 +88,7 @@ const ContentAddress * getDerivationCA(const BasicDerivation & drv) auto out = drv.outputs.find("out"); if (out == drv.outputs.end()) return nullptr; - if (auto dof = std::get_if(&out->second)) { + if (auto dof = std::get_if(&out->second.raw)) { return &dof->ca; } return nullptr; @@ -370,7 +370,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, } return outputsOpt; }, - }, bfd.outputs.raw()); + }, bfd.outputs.raw); OutputPathMap outputs; for (auto & [outputName, outputPathOpt] : outputsOpt) { @@ -418,7 +418,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd) [&](const OutputsSpec::Names & names) { return static_cast(names); }, - }, bfd.outputs.raw()); + }, bfd.outputs.raw); for (auto iter = outputMap.begin(); iter != outputMap.end();) { auto & outputName = iter->first; if (bfd.outputs.contains(outputName)) { @@ -431,7 +431,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd) if (!outputsLeft.empty()) throw Error("derivation '%s' does not have an outputs %s", store.printStorePath(drvPath), - concatStringsSep(", ", quoteStrings(std::get(bfd.outputs)))); + concatStringsSep(", ", quoteStrings(std::get(bfd.outputs.raw)))); return outputMap; } diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc index e26c38138..d943bc111 100644 --- a/src/libstore/outputs-spec.cc +++ b/src/libstore/outputs-spec.cc @@ -17,7 +17,7 @@ bool OutputsSpec::contains(const std::string & outputName) const [&](const OutputsSpec::Names & outputNames) { return outputNames.count(outputName) > 0; }, - }, raw()); + }, raw); } static std::string outputSpecRegexStr = @@ -49,7 +49,7 @@ OutputsSpec OutputsSpec::parse(std::string_view s) std::optional spec = parseOpt(s); if (!spec) throw Error("invalid outputs specifier '%s'", s); - return *spec; + return std::move(*spec); } @@ -85,7 +85,7 @@ std::string OutputsSpec::to_string() const [&](const OutputsSpec::Names & outputNames) -> std::string { return concatStringsSep(",", outputNames); }, - }, raw()); + }, raw); } @@ -98,7 +98,7 @@ std::string ExtendedOutputsSpec::to_string() const [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> std::string { return "^" + outputSpec.to_string(); }, - }, raw()); + }, raw); } @@ -118,9 +118,9 @@ OutputsSpec OutputsSpec::union_(const OutputsSpec & that) const ret.insert(thoseNames.begin(), thoseNames.end()); return ret; }, - }, that.raw()); + }, that.raw); }, - }, raw()); + }, raw); } @@ -142,9 +142,9 @@ bool OutputsSpec::isSubsetOf(const OutputsSpec & that) const ret = false; return ret; }, - }, raw()); + }, raw); }, - }, that.raw()); + }, that.raw); } } @@ -169,7 +169,7 @@ void adl_serializer::to_json(json & json, OutputsSpec t) { [&](const OutputsSpec::Names & names) { json = names; }, - }, t.raw()); + }, t.raw); } @@ -189,7 +189,7 @@ void adl_serializer::to_json(json & json, ExtendedOutputsSp [&](const ExtendedOutputsSpec::Explicit & e) { adl_serializer::to_json(json, e); }, - }, t.raw()); + }, t.raw); } } diff --git a/src/libstore/outputs-spec.hh b/src/libstore/outputs-spec.hh index 5a726fe90..ae19f1040 100644 --- a/src/libstore/outputs-spec.hh +++ b/src/libstore/outputs-spec.hh @@ -6,62 +6,57 @@ #include #include +#include "comparator.hh" #include "json-impls.hh" +#include "comparator.hh" +#include "variant-wrapper.hh" namespace nix { -/** - * A non-empty set of outputs, specified by name - */ -struct OutputNames : std::set { - using std::set::set; +struct OutputsSpec { + /** + * A non-empty set of outputs, specified by name + */ + struct Names : std::set { + using std::set::set; - /* These need to be "inherited manually" */ + /* These need to be "inherited manually" */ - OutputNames(const std::set & s) - : std::set(s) - { assert(!empty()); } + Names(const std::set & s) + : std::set(s) + { assert(!empty()); } + + /** + * Needs to be "inherited manually" + */ + Names(std::set && s) + : std::set(s) + { assert(!empty()); } + + /* This set should always be non-empty, so we delete this + constructor in order make creating empty ones by mistake harder. + */ + Names() = delete; + }; /** - * Needs to be "inherited manually" + * The set of all outputs, without needing to name them explicitly */ - OutputNames(std::set && s) - : std::set(s) - { assert(!empty()); } + struct All : std::monostate { }; - /* This set should always be non-empty, so we delete this - constructor in order make creating empty ones by mistake harder. - */ - OutputNames() = delete; -}; + typedef std::variant Raw; -/** - * The set of all outputs, without needing to name them explicitly - */ -struct AllOutputs : std::monostate { }; + Raw raw; -typedef std::variant _OutputsSpecRaw; + GENERATE_CMP(OutputsSpec, me->raw); -struct OutputsSpec : _OutputsSpecRaw { - using Raw = _OutputsSpecRaw; - using Raw::Raw; + MAKE_WRAPPER_CONSTRUCTOR(OutputsSpec); /** * Force choosing a variant */ OutputsSpec() = delete; - using Names = OutputNames; - using All = AllOutputs; - - inline const Raw & raw() const { - return static_cast(*this); - } - - inline Raw & raw() { - return static_cast(*this); - } - bool contains(const std::string & output) const; /** @@ -84,20 +79,22 @@ struct OutputsSpec : _OutputsSpecRaw { std::string to_string() const; }; -struct DefaultOutputs : std::monostate { }; - -typedef std::variant _ExtendedOutputsSpecRaw; - -struct ExtendedOutputsSpec : _ExtendedOutputsSpecRaw { - using Raw = _ExtendedOutputsSpecRaw; - using Raw::Raw; - - using Default = DefaultOutputs; +struct ExtendedOutputsSpec { + struct Default : std::monostate { }; using Explicit = OutputsSpec; - inline const Raw & raw() const { - return static_cast(*this); - } + typedef std::variant Raw; + + Raw raw; + + GENERATE_CMP(ExtendedOutputsSpec, me->raw); + + MAKE_WRAPPER_CONSTRUCTOR(ExtendedOutputsSpec); + + /** + * Force choosing a variant + */ + ExtendedOutputsSpec() = delete; /** * Parse a string of the form 'prefix^output1,...outputN' or diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 81f360f3a..af6837370 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -63,7 +63,7 @@ StorePathWithOutputs::ParseResult StorePathWithOutputs::tryFromDerivedPath(const [&](const OutputsSpec::Names & outputs) { return static_cast(outputs); }, - }, bfd.outputs.raw()), + }, bfd.outputs.raw), }; }, [&](const SingleDerivedPath::Built &) -> StorePathWithOutputs::ParseResult { diff --git a/src/libutil/variant-wrapper.hh b/src/libutil/variant-wrapper.hh new file mode 100644 index 000000000..cedcb999c --- /dev/null +++ b/src/libutil/variant-wrapper.hh @@ -0,0 +1,30 @@ +#pragma once +///@file + +// not used, but will be used by callers +#include + +/** + * Force the default versions of all constructors (copy, move, copy + * assignment). + */ +#define FORCE_DEFAULT_CONSTRUCTORS(CLASS_NAME) \ + CLASS_NAME(const CLASS_NAME &) = default; \ + CLASS_NAME(CLASS_NAME &) = default; \ + CLASS_NAME(CLASS_NAME &&) = default; \ + \ + CLASS_NAME & operator =(const CLASS_NAME &) = default; \ + CLASS_NAME & operator =(CLASS_NAME &) = default; + +/** + * Make a wrapper constructor. All args are forwarded to the + * construction of the "raw" field. (Which we assume is the only one.) + * + * The moral equivalent of `using Raw::Raw;` + */ +#define MAKE_WRAPPER_CONSTRUCTOR(CLASS_NAME) \ + FORCE_DEFAULT_CONSTRUCTORS(CLASS_NAME) \ + \ + CLASS_NAME(auto &&... arg) \ + : raw(std::forward(arg)...) \ + { } diff --git a/src/nix/app.cc b/src/nix/app.cc index 16a921194..67c5ef344 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -81,7 +81,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) .path = o.path, }; }, - }, c.raw())); + }, c.raw)); } return UnresolvedApp{App { diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 5a80f0308..fbc83b08e 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -80,7 +80,7 @@ struct CmdBundle : InstallableValueCommand auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; InstallableFlake bundler{this, - evalState, std::move(bundlerFlakeRef), bundlerName, extendedOutputsSpec, + evalState, std::move(bundlerFlakeRef), bundlerName, std::move(extendedOutputsSpec), {"bundlers." + settings.thisSystem.get() + ".default", "defaultBundler." + settings.thisSystem.get() }, diff --git a/src/nix/develop.cc b/src/nix/develop.cc index c033804e4..c01ef1a2a 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -547,7 +547,7 @@ struct CmdDevelop : Common, MixEnvironment state, std::move(nixpkgs), "bashInteractive", - DefaultOutputs(), + ExtendedOutputsSpec::Default(), Strings{}, Strings{"legacyPackages." + settings.thisSystem.get() + "."}, nixpkgsLockFlags); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 83b74c8ca..87dd4da1b 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -778,7 +778,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath(".")); auto installable = InstallableFlake(nullptr, - evalState, std::move(templateFlakeRef), templateName, DefaultOutputs(), + evalState, std::move(templateFlakeRef), templateName, ExtendedOutputsSpec::Default(), defaultTemplateAttrPaths, defaultTemplateAttrPathsPrefixes, lockFlags); From fe71faa920ef34b67f56232decc22cf5706a00dd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 16 Aug 2023 12:04:15 -0400 Subject: [PATCH 189/909] Delete `EvalState::addToSearchPath` This function is now trivial enough that it doesn't need to exist. `EvalState` can still be initialized with a custom search path, but we don't have a need to mutate the search path after it has been constructed, and I don't see why we would need to in the future. Fixes #8229 --- src/libexpr/eval.cc | 4 ++-- src/libexpr/eval.hh | 2 -- src/libexpr/parser.y | 6 ------ 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7e839dbe7..6d445fd96 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -527,9 +527,9 @@ EvalState::EvalState( /* Initialise the Nix expression search path. */ if (!evalSettings.pureEval) { for (auto & i : _searchPath.elements) - addToSearchPath(SearchPath::Elem {i}); + searchPath.elements.emplace_back(SearchPath::Elem {i}); for (auto & i : evalSettings.nixPath.get()) - addToSearchPath(SearchPath::Elem::parse(i)); + searchPath.elements.emplace_back(SearchPath::Elem::parse(i)); } if (evalSettings.restrictEval || evalSettings.pureEval) { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 0c3eb6505..fa8fa462b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -341,8 +341,6 @@ public: std::shared_ptr buildStore = nullptr); ~EvalState(); - void addToSearchPath(SearchPath::Elem && elem); - SearchPath getSearchPath() { return searchPath; } /** diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 201370b90..792f51fde 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -736,12 +736,6 @@ Expr * EvalState::parseStdin() } -void EvalState::addToSearchPath(SearchPath::Elem && elem) -{ - searchPath.elements.emplace_back(std::move(elem)); -} - - SourcePath EvalState::findFile(const std::string_view path) { return findFile(searchPath, path); From 52248b1c2786f7aa98109d47ee12f527bd202b12 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Sat, 19 Aug 2023 17:03:31 -0400 Subject: [PATCH 190/909] feat: notation to refer to no attribute search prefix An attrPath prefix of "." indicates no need to try default attrPath prefixes. For example 1nixpkgs#legacyPackages.x86_64-linux.ERROR` searches through ``` trying flake output attribute 'packages.x86_64-linux.legacyPackages.x86_64-linux.ERROR' using cached attrset attribute '' trying flake output attribute 'legacyPackages.x86_64-linux.legacyPackages.x86_64-linux.ERROR' using cached attrset attribute 'legacyPackages.x86_64-linux' trying flake output attribute 'legacyPackages.x86_64-linux.ERROR' using cached attrset attribute 'legacyPackages.x86_64-linux' ``` And there is no way to specify that one does not want the automatic search behavior. Now one can specify `nixpkgs#.legacyPackages.x86_64-linux.ERROR` to only refer to the rooted attribute path without any default injection of attribute search path or system. --- src/libcmd/installable-flake.cc | 5 +++++ src/libcmd/installables.cc | 12 ++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 4074da06d..2f428cb7e 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -28,6 +28,11 @@ namespace nix { std::vector InstallableFlake::getActualAttrPaths() { std::vector res; + if (attrPaths.size() == 1 && attrPaths.front().starts_with(".")){ + attrPaths.front().erase(0,1); + res.push_back(attrPaths.front()); + return res; + } for (auto & prefix : prefixes) res.push_back(prefix + *attrPaths.begin()); diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index eb1903084..01c01441c 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -301,6 +301,11 @@ void completeFlakeRefWithFragment( completionType = ctAttrs; auto fragment = prefix.substr(hash + 1); + std::string prefixRoot = ""; + if (fragment.starts_with(".")){ + fragment = fragment.substr(1); + prefixRoot = "."; + } auto flakeRefS = std::string(prefix.substr(0, hash)); auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); @@ -309,6 +314,9 @@ void completeFlakeRefWithFragment( auto root = evalCache->getRoot(); + if (prefixRoot == "."){ + attrPathPrefixes.clear(); + } /* Complete 'fragment' relative to all the attrpath prefixes as well as the root of the flake. */ @@ -333,7 +341,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); - completions->add(flakeRefS + "#" + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); + completions->add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } } @@ -344,7 +352,7 @@ void completeFlakeRefWithFragment( for (auto & attrPath : defaultFlakeAttrPaths) { auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath)); if (!attr) continue; - completions->add(flakeRefS + "#"); + completions->add(flakeRefS + "#" + prefixRoot); } } } From c609be4072b03dc3eabcb2ac9916593847fd3b19 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Sat, 19 Aug 2023 17:19:52 -0400 Subject: [PATCH 191/909] doc: explain the . attrPath prefix notation --- doc/manual/src/release-notes/rl-next.md | 2 ++ src/nix/nix.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 7ddb5ca00..56222a21b 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -22,3 +22,5 @@ - Introduce a new [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) builtin. It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + +- Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.` or `legacyPackages.`. diff --git a/src/nix/nix.md b/src/nix/nix.md index e0f459d6b..6e7e8a649 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -132,6 +132,8 @@ subcommands, these are `packages.`*system*, attributes `packages.x86_64-linux.hello`, `legacyPackages.x86_64-linux.hello` and `hello`. +If *attrpath* begins with `.` then no prefixes or defaults are attempted. This allows the form *flakeref*[`#.`*attrpath*], such as `github:NixOS/nixpkgs#.lib.fakeSha256` to avoid a search of `packages.*system*.lib.fakeSha256` + ### Store path Example: `/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10` From 8130373be9a10681a94b0546242843db158f30d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 22:22:21 +0000 Subject: [PATCH 192/909] Bump zeebe-io/backport-action from 1.3.1 to 1.4.0 Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 1.3.1 to 1.4.0. - [Release notes](https://github.com/zeebe-io/backport-action/releases) - [Commits](https://github.com/zeebe-io/backport-action/compare/v1.3.1...v1.4.0) --- updated-dependencies: - dependency-name: zeebe-io/backport-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 816474ed5..62bef6322 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v1.3.1 + uses: zeebe-io/backport-action@v1.4.0 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} From 81045f243f91adcd44da0080c9ac83d2c4176125 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Aug 2023 19:29:25 +0200 Subject: [PATCH 193/909] Tarball trees: Propagate lastModified This makes them behave consistently with GitHub/GitLab flakes. --- src/libfetchers/tarball.cc | 5 ++++- tests/tarball.sh | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 107d38e92..e3a41e75e 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -232,7 +232,7 @@ struct CurlInputScheme : InputScheme if (type != inputType()) return {}; // FIXME: some of these only apply to TarballInputScheme. - std::set allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount"}; + std::set allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount", "lastModified"}; for (auto & [name, value] : attrs) if (!allowedNames.count(name)) throw Error("unsupported %s input attribute '%s'", *type, name); @@ -310,6 +310,9 @@ struct TarballInputScheme : CurlInputScheme input = immutableInput; } + if (result.lastModified && !input.attrs.contains("lastModified")) + input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); + return {result.tree.storePath, std::move(input)}; } }; diff --git a/tests/tarball.sh b/tests/tarball.sh index 5f39658c9..6e621a28c 100644 --- a/tests/tarball.sh +++ b/tests/tarball.sh @@ -9,6 +9,7 @@ rm -rf $tarroot mkdir -p $tarroot cp dependencies.nix $tarroot/default.nix cp config.nix dependencies.builder*.sh $tarroot/ +touch -d '@1000000000' $tarroot $tarroot/* hash=$(nix hash path $tarroot) @@ -36,6 +37,8 @@ test_tarball() { nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file:///does-not-exist/must-remain-unused/$tarball; narHash = \"$hash\"; })" expectStderr 102 nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" | grep 'NAR hash mismatch in input' + [[ $(nix eval --impure --expr "(fetchTree file://$tarball).lastModified") = 1000000000 ]] + nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2 nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true' From 4a435ad2288bffd5c28d90758e4e26160b53fb8c Mon Sep 17 00:00:00 2001 From: Uri Zafrir Date: Wed, 23 Aug 2023 18:18:25 +0300 Subject: [PATCH 194/909] Add introductory sentence to advanced topics (#8861) --- doc/manual/src/advanced-topics/advanced-topics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/advanced-topics/advanced-topics.md b/doc/manual/src/advanced-topics/advanced-topics.md index 8b1378917..9a4d12a33 100644 --- a/doc/manual/src/advanced-topics/advanced-topics.md +++ b/doc/manual/src/advanced-topics/advanced-topics.md @@ -1 +1 @@ - +This section lists advanced topics related to builds and builds performance From 7d82341633d81f50f3e5e0b0896cfac5ea805d57 Mon Sep 17 00:00:00 2001 From: p01arst0rm Date: Wed, 23 Aug 2023 19:28:24 +0100 Subject: [PATCH 195/909] update system definitions --- flake.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 6e1f48cd7..cf7c1fa12 100644 --- a/flake.nix +++ b/flake.nix @@ -19,10 +19,12 @@ then "" else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; + linux32BitSystems = [ "i686-linux" ]; linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ]; - linuxSystems = linux64BitSystems ++ [ "i686-linux" ]; - systems = linuxSystems ++ [ "x86_64-darwin" "aarch64-darwin" ]; - + linuxSystems = linux32BitSystems ++ linux64BitSystems; + darwinSystems = [ "x86_64-darwin" "aarch64-darwin" ]; + systems = linuxSystems ++ darwinSystems; + crossSystems = [ "armv6l-linux" "armv7l-linux" ]; stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ]; From d5b130ef13bd9a77d803950ca814ac9a612f77b8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 24 Aug 2023 10:00:17 +0200 Subject: [PATCH 196/909] glossary: dedent list and do not use forced line breaks this makes it slightly easier to work with and consistent with all the other markdown lists in use --- doc/manual/src/glossary.md | 367 ++++++++++++++++++++----------------- 1 file changed, 200 insertions(+), 167 deletions(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index ac0bb3c2f..d950a4ca8 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -1,236 +1,269 @@ # Glossary - - [derivation]{#gloss-derivation}\ - A description of a build task. The result of a derivation is a - store object. Derivations are typically specified in Nix expressions - using the [`derivation` primitive](./language/derivations.md). These are - translated into low-level *store derivations* (implicitly by - `nix-env` and `nix-build`, or explicitly by `nix-instantiate`). +- [derivation]{#gloss-derivation} - [derivation]: #gloss-derivation + A description of a build task. The result of a derivation is a + store object. Derivations are typically specified in Nix expressions + using the [`derivation` primitive](./language/derivations.md). These are + translated into low-level *store derivations* (implicitly by + `nix-env` and `nix-build`, or explicitly by `nix-instantiate`). - - [store derivation]{#gloss-store-derivation}\ - A [derivation] represented as a `.drv` file in the [store]. - It has a [store path], like any [store object]. + [derivation]: #gloss-derivation - Example: `/nix/store/g946hcz4c8mdvq2g8vxx42z51qb71rvp-git-2.38.1.drv` +- [store derivation]{#gloss-store-derivation} - See [`nix derivation show`](./command-ref/new-cli/nix3-derivation-show.md) (experimental) for displaying the contents of store derivations. + A [derivation] represented as a `.drv` file in the [store]. + It has a [store path], like any [store object]. - [store derivation]: #gloss-store-derivation + Example: `/nix/store/g946hcz4c8mdvq2g8vxx42z51qb71rvp-git-2.38.1.drv` - - [instantiate]{#gloss-instantiate}, instantiation\ - Translate a [derivation] into a [store derivation]. + See [`nix derivation show`](./command-ref/new-cli/nix3-derivation-show.md) (experimental) for displaying the contents of store derivations. - See [`nix-instantiate`](./command-ref/nix-instantiate.md). + [store derivation]: #gloss-store-derivation - [instantiate]: #gloss-instantiate +- [instantiate]{#gloss-instantiate}, instantiation - - [realise]{#gloss-realise}, realisation\ - Ensure a [store path] is [valid][validity]. + Translate a [derivation] into a [store derivation]. - This means either running the `builder` executable as specified in the corresponding [derivation] or fetching a pre-built [store object] from a [substituter]. + See [`nix-instantiate`](./command-ref/nix-instantiate.md). - See [`nix-build`](./command-ref/nix-build.md) and [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md). + [instantiate]: #gloss-instantiate - See [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). +- [realise]{#gloss-realise}, realisation - [realise]: #gloss-realise + Ensure a [store path] is [valid][validity]. - - [content-addressed derivation]{#gloss-content-addressed-derivation}\ - A derivation which has the - [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) - attribute set to `true`. + This means either running the `builder` executable as specified in the corresponding [derivation] or fetching a pre-built [store object] from a [substituter]. - - [fixed-output derivation]{#gloss-fixed-output-derivation}\ - A derivation which includes the - [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute. + See [`nix-build`](./command-ref/nix-build.md) and [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md). - - [store]{#gloss-store}\ - The location in the file system where store objects live. Typically - `/nix/store`. + See [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). - From the perspective of the location where Nix is - invoked, the Nix store can be referred to - as a "_local_" or a "_remote_" one: + [realise]: #gloss-realise - + A [local store]{#gloss-local-store} exists on the filesystem of - the machine where Nix is invoked. You can use other - local stores by passing the `--store` flag to the - `nix` command. Local stores can be used for building derivations. +- [content-addressed derivation]{#gloss-content-addressed-derivation} - + A *remote store* exists anywhere other than the - local filesystem. One example is the `/nix/store` - directory on another machine, accessed via `ssh` or - served by the `nix-serve` Perl script. + A derivation which has the + [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) + attribute set to `true`. - [store]: #gloss-store - [local store]: #gloss-local-store +- [fixed-output derivation]{#gloss-fixed-output-derivation} - - [chroot store]{#gloss-chroot-store}\ - A [local store] whose canonical path is anything other than `/nix/store`. + A derivation which includes the + [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute. - - [binary cache]{#gloss-binary-cache}\ - A *binary cache* is a Nix store which uses a different format: its - metadata and signatures are kept in `.narinfo` files rather than in a - [Nix database]. This different format simplifies serving store objects - over the network, but cannot host builds. Examples of binary caches - include S3 buckets and the [NixOS binary cache](https://cache.nixos.org). +- [store]{#gloss-store} - - [store path]{#gloss-store-path}\ - The location of a [store object] in the file system, i.e., an - immediate child of the Nix store directory. + The location in the file system where store objects live. Typically + `/nix/store`. - Example: `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` + From the perspective of the location where Nix is + invoked, the Nix store can be referred to + as a "_local_" or a "_remote_" one: - [store path]: #gloss-store-path + + A [local store]{#gloss-local-store} exists on the filesystem of + the machine where Nix is invoked. You can use other + local stores by passing the `--store` flag to the + `nix` command. Local stores can be used for building derivations. - - [file system object]{#gloss-store-object}\ - The Nix data model for representing simplified file system data. + + A *remote store* exists anywhere other than the + local filesystem. One example is the `/nix/store` + directory on another machine, accessed via `ssh` or + served by the `nix-serve` Perl script. - See [File System Object](@docroot@/architecture/file-system-object.md) for details. + [store]: #gloss-store + [local store]: #gloss-local-store - [file system object]: #gloss-file-system-object +- [chroot store]{#gloss-chroot-store} - - [store object]{#gloss-store-object}\ + A [local store] whose canonical path is anything other than `/nix/store`. - A store object consists of a [file system object], [reference]s to other store objects, and other metadata. - It can be referred to by a [store path]. +- [binary cache]{#gloss-binary-cache} - [store object]: #gloss-store-object + A *binary cache* is a Nix store which uses a different format: its + metadata and signatures are kept in `.narinfo` files rather than in a + [Nix database]. This different format simplifies serving store objects + over the network, but cannot host builds. Examples of binary caches + include S3 buckets and the [NixOS binary cache](https://cache.nixos.org). - - [input-addressed store object]{#gloss-input-addressed-store-object}\ - A store object produced by building a - non-[content-addressed](#gloss-content-addressed-derivation), - non-[fixed-output](#gloss-fixed-output-derivation) - derivation. +- [store path]{#gloss-store-path} - - [output-addressed store object]{#gloss-output-addressed-store-object}\ - A [store object] whose [store path] is determined by its contents. - This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). + The location of a [store object] in the file system, i.e., an + immediate child of the Nix store directory. - - [substitute]{#gloss-substitute}\ - A substitute is a command invocation stored in the [Nix database] that - describes how to build a store object, bypassing the normal build - mechanism (i.e., derivations). Typically, the substitute builds the - store object by downloading a pre-built version of the store object - from some server. + Example: `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` - - [substituter]{#gloss-substituter}\ - An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them. - Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter. + [store path]: #gloss-store-path - See the [`substituters` configuration option](./command-ref/conf-file.md#conf-substituters) for details. +- [file system object]{#gloss-store-object} - [substituter]: #gloss-substituter + The Nix data model for representing simplified file system data. - - [purity]{#gloss-purity}\ - The assumption that equal Nix derivations when run always produce - the same output. This cannot be guaranteed in general (e.g., a - builder can rely on external inputs such as the network or the - system time) but the Nix model assumes it. + See [File System Object](@docroot@/architecture/file-system-object.md) for details. - - [Nix database]{#gloss-nix-database}\ - An SQlite database to track [reference]s between [store object]s. - This is an implementation detail of the [local store]. + [file system object]: #gloss-file-system-object - Default location: `/nix/var/nix/db`. +- [store object]{#gloss-store-object} - [Nix database]: #gloss-nix-database - - [Nix expression]{#gloss-nix-expression}\ - A high-level description of software packages and compositions - thereof. Deploying software using Nix entails writing Nix - expressions for your packages. Nix expressions are translated to - derivations that are stored in the Nix store. These derivations can - then be built. + A store object consists of a [file system object], [reference]s to other store objects, and other metadata. + It can be referred to by a [store path]. - - [reference]{#gloss-reference}\ - A [store object] `O` is said to have a *reference* to a store object `P` if a [store path] to `P` appears in the contents of `O`. + [store object]: #gloss-store-object - Store objects can refer to both other store objects and themselves. - References from a store object to itself are called *self-references*. - References other than a self-reference must not form a cycle. +- [input-addressed store object]{#gloss-input-addressed-store-object} - [reference]: #gloss-reference + A store object produced by building a + non-[content-addressed](#gloss-content-addressed-derivation), + non-[fixed-output](#gloss-fixed-output-derivation) + derivation. - - [reachable]{#gloss-reachable}\ - A store path `Q` is reachable from another store path `P` if `Q` - is in the *closure* of the *references* relation. +- [output-addressed store object]{#gloss-output-addressed-store-object} - - [closure]{#gloss-closure}\ - The closure of a store path is the set of store paths that are - directly or indirectly “reachable” from that store path; that is, - it’s the closure of the path under the *references* relation. For - a package, the closure of its derivation is equivalent to the - build-time dependencies, while the closure of its output path is - equivalent to its runtime dependencies. For correct deployment it - is necessary to deploy whole closures, since otherwise at runtime - files could be missing. The command `nix-store --query --requisites ` prints out - closures of store paths. + A [store object] whose [store path] is determined by its contents. + This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). - As an example, if the [store object] at path `P` contains a [reference] - to a store object at path `Q`, then `Q` is in the closure of `P`. Further, if `Q` - references `R` then `R` is also in the closure of `P`. +- [substitute]{#gloss-substitute} - [closure]: #gloss-closure + A substitute is a command invocation stored in the [Nix database] that + describes how to build a store object, bypassing the normal build + mechanism (i.e., derivations). Typically, the substitute builds the + store object by downloading a pre-built version of the store object + from some server. - - [output path]{#gloss-output-path}\ - A [store path] produced by a [derivation]. +- [substituter]{#gloss-substituter} - [output path]: #gloss-output-path + An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them. + Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter. - - [deriver]{#gloss-deriver}\ - The [store derivation] that produced an [output path]. + See the [`substituters` configuration option](./command-ref/conf-file.md#conf-substituters) for details. - - [validity]{#gloss-validity}\ - A store path is valid if all [store object]s in its [closure] can be read from the [store]. + [substituter]: #gloss-substituter - For a [local store], this means: - - The store path leads to an existing [store object] in that [store]. - - The store path is listed in the [Nix database] as being valid. - - All paths in the store path's [closure] are valid. +- [purity]{#gloss-purity} - [validity]: #gloss-validity + The assumption that equal Nix derivations when run always produce + the same output. This cannot be guaranteed in general (e.g., a + builder can rely on external inputs such as the network or the + system time) but the Nix model assumes it. - - [user environment]{#gloss-user-env}\ - An automatically generated store object that consists of a set of - symlinks to “active” applications, i.e., other store paths. These - are generated automatically by - [`nix-env`](./command-ref/nix-env.md). See *profiles*. +- [Nix database]{#gloss-nix-database} - - [profile]{#gloss-profile}\ - A symlink to the current *user environment* of a user, e.g., - `/nix/var/nix/profiles/default`. + An SQlite database to track [reference]s between [store object]s. + This is an implementation detail of the [local store]. - - [installable]{#gloss-installable}\ - Something that can be realised in the Nix store. + Default location: `/nix/var/nix/db`. - See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. + [Nix database]: #gloss-nix-database - - [NAR]{#gloss-nar}\ - A *N*ix *AR*chive. This is a serialisation of a path in the Nix - store. It can contain regular files, directories and symbolic - links. NARs are generated and unpacked using `nix-store --dump` - and `nix-store --restore`. +- [Nix expression]{#gloss-nix-expression} - - [`∅`]{#gloss-emtpy-set}\ - The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. + A high-level description of software packages and compositions + thereof. Deploying software using Nix entails writing Nix + expressions for your packages. Nix expressions are translated to + derivations that are stored in the Nix store. These derivations can + then be built. - - [`ε`]{#gloss-epsilon}\ - The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute. +- [reference]{#gloss-reference} - - [string interpolation]{#gloss-string-interpolation}\ - Expanding expressions enclosed in `${ }` within a [string], [path], or [attribute name]. + A [store object] `O` is said to have a *reference* to a store object `P` if a [store path] to `P` appears in the contents of `O`. - See [String interpolation](./language/string-interpolation.md) for details. + Store objects can refer to both other store objects and themselves. + References from a store object to itself are called *self-references*. + References other than a self-reference must not form a cycle. - [string]: ./language/values.md#type-string - [path]: ./language/values.md#type-path - [attribute name]: ./language/values.md#attribute-set + [reference]: #gloss-reference - - [experimental feature]{#gloss-experimental-feature}\ - Not yet stabilized functionality guarded by named experimental feature flags. - These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. +- [reachable]{#gloss-reachable} - See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). + A store path `Q` is reachable from another store path `P` if `Q` + is in the *closure* of the *references* relation. + +- [closure]{#gloss-closure} + + The closure of a store path is the set of store paths that are + directly or indirectly “reachable” from that store path; that is, + it’s the closure of the path under the *references* relation. For + a package, the closure of its derivation is equivalent to the + build-time dependencies, while the closure of its output path is + equivalent to its runtime dependencies. For correct deployment it + is necessary to deploy whole closures, since otherwise at runtime + files could be missing. The command `nix-store --query --requisites ` prints out + closures of store paths. + + As an example, if the [store object] at path `P` contains a [reference] + to a store object at path `Q`, then `Q` is in the closure of `P`. Further, if `Q` + references `R` then `R` is also in the closure of `P`. + + [closure]: #gloss-closure + +- [output path]{#gloss-output-path} + + A [store path] produced by a [derivation]. + + [output path]: #gloss-output-path + +- [deriver]{#gloss-deriver} + + The [store derivation] that produced an [output path]. + +- [validity]{#gloss-validity} + + A store path is valid if all [store object]s in its [closure] can be read from the [store]. + + For a [local store], this means: + - The store path leads to an existing [store object] in that [store]. + - The store path is listed in the [Nix database] as being valid. + - All paths in the store path's [closure] are valid. + + [validity]: #gloss-validity + +- [user environment]{#gloss-user-env} + + An automatically generated store object that consists of a set of + symlinks to “active” applications, i.e., other store paths. These + are generated automatically by + [`nix-env`](./command-ref/nix-env.md). See *profiles*. + +- [profile]{#gloss-profile} + + A symlink to the current *user environment* of a user, e.g., + `/nix/var/nix/profiles/default`. + +- [installable]{#gloss-installable} + + Something that can be realised in the Nix store. + + See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. + +- [NAR]{#gloss-nar} + + A *N*ix *AR*chive. This is a serialisation of a path in the Nix + store. It can contain regular files, directories and symbolic + links. NARs are generated and unpacked using `nix-store --dump` + and `nix-store --restore`. + +- [`∅`]{#gloss-emtpy-set} + + The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. + +- [`ε`]{#gloss-epsilon} + + The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute. + +- [string interpolation]{#gloss-string-interpolation} + + Expanding expressions enclosed in `${ }` within a [string], [path], or [attribute name]. + + See [String interpolation](./language/string-interpolation.md) for details. + + [string]: ./language/values.md#type-string + [path]: ./language/values.md#type-path + [attribute name]: ./language/values.md#attribute-set + +- [experimental feature]{#gloss-experimental-feature} + + Not yet stabilized functionality guarded by named experimental feature flags. + These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. + + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). From 925a444b925590df90e19d3c0071936f87d2b43d Mon Sep 17 00:00:00 2001 From: Guillaume Girol Date: Wed, 23 Aug 2023 12:00:00 +0000 Subject: [PATCH 197/909] add nix-store --query --valid-derivers command notably useful when nix-store --query --deriver returns a non-existing path. Co-authored-by: Felix Uhl --- doc/manual/src/command-ref/nix-store/query.md | 14 +++++++++++--- doc/manual/src/release-notes/rl-next.md | 4 ++++ src/nix-store/nix-store.cc | 18 +++++++++++++++++- tests/check-refs.nix | 2 +- tests/dependencies.nix | 13 +++++++++++++ tests/dependencies.sh | 17 +++++++++++++++++ tests/dyn-drv/eval-outputOf.sh | 4 ++-- tests/export-graph.nix | 4 ++-- tests/remote-store.sh | 2 +- 9 files changed, 68 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/command-ref/nix-store/query.md b/doc/manual/src/command-ref/nix-store/query.md index cd45a4932..a158c76aa 100644 --- a/doc/manual/src/command-ref/nix-store/query.md +++ b/doc/manual/src/command-ref/nix-store/query.md @@ -5,8 +5,8 @@ # Synopsis `nix-store` {`--query` | `-q`} - {`--outputs` | `--requisites` | `-R` | `--references` | - `--referrers` | `--referrers-closure` | `--deriver` | `-d` | + {`--outputs` | `--requisites` | `-R` | `--references` | `--referrers` | + `--referrers-closure` | `--deriver` | `-d` | `--valid-derivers` | `--graph` | `--tree` | `--binding` *name* | `-b` *name* | `--hash` | `--size` | `--roots`} [`--use-output`] [`-u`] [`--force-realise`] [`-f`] @@ -82,13 +82,21 @@ symlink. in the Nix store that are dependent on *paths*. - `--deriver`; `-d`\ - Prints the [deriver] of the store paths *paths*. If + Prints the [deriver] that was used to build the store paths *paths*. If the path has no deriver (e.g., if it is a source file), or if the deriver is not known (e.g., in the case of a binary-only deployment), the string `unknown-deriver` is printed. + The returned deriver is not guaranteed to exist in the local store, for + example when *paths* were substituted from a binary cache. + Use `--valid-derivers` instead to obtain valid paths only. [deriver]: ../../glossary.md#gloss-deriver + - `--valid-derivers`\ + Prints a set of derivation files (`.drv`) which are supposed produce + said paths when realized. Might print nothing, for example for source paths + or paths subsituted from a binary cache. + - `--graph`\ Prints the references graph of the store paths *paths* in the format of the `dot` tool of AT\&T's [Graphviz diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 7ddb5ca00..66fe0c427 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -22,3 +22,7 @@ - Introduce a new [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) builtin. It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + +- [`nix-store --query`](@docroot@/command-ref/nix-store/query.md) gained a new type of query: `--valid-derivers`. It returns all `.drv` files in the local store that *can be* used to build the output passed in argument. +This is in contrast to `--deriver`, which returns the single `.drv` file that *was actually* used to build the output passed in argument. In case the output was substituted from a binary cache, +this `.drv` file may only exist on said binary cache and not locally. diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 94956df66..96c3f7d7e 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -283,7 +283,7 @@ static void opQuery(Strings opFlags, Strings opArgs) { enum QueryType { qOutputs, qRequisites, qReferences, qReferrers - , qReferrersClosure, qDeriver, qBinding, qHash, qSize + , qReferrersClosure, qDeriver, qValidDerivers, qBinding, qHash, qSize , qTree, qGraph, qGraphML, qResolve, qRoots }; std::optional query; bool useOutput = false; @@ -299,6 +299,7 @@ static void opQuery(Strings opFlags, Strings opArgs) else if (i == "--referrers" || i == "--referers") query = qReferrers; else if (i == "--referrers-closure" || i == "--referers-closure") query = qReferrersClosure; else if (i == "--deriver" || i == "-d") query = qDeriver; + else if (i == "--valid-derivers") query = qValidDerivers; else if (i == "--binding" || i == "-b") { if (opArgs.size() == 0) throw UsageError("expected binding name"); @@ -372,6 +373,21 @@ static void opQuery(Strings opFlags, Strings opArgs) } break; + case qValidDerivers: { + StorePathSet result; + for (auto & i : opArgs) { + auto derivers = store->queryValidDerivers(store->followLinksToStorePath(i)); + for (const auto &i: derivers) { + result.insert(i); + } + } + auto sorted = store->topoSortPaths(result); + for (StorePaths::reverse_iterator i = sorted.rbegin(); + i != sorted.rend(); ++i) + cout << fmt("%s\n", store->printStorePath(*i)); + break; + } + case qBinding: for (auto & i : opArgs) { auto path = useDeriver(store->followLinksToStorePath(i)); diff --git a/tests/check-refs.nix b/tests/check-refs.nix index 99d69a226..89690e456 100644 --- a/tests/check-refs.nix +++ b/tests/check-refs.nix @@ -2,7 +2,7 @@ with import ./config.nix; rec { - dep = import ./dependencies.nix; + dep = import ./dependencies.nix {}; makeTest = nr: args: mkDerivation ({ name = "check-refs-" + toString nr; diff --git a/tests/dependencies.nix b/tests/dependencies.nix index 45aca1793..be1a7ae9a 100644 --- a/tests/dependencies.nix +++ b/tests/dependencies.nix @@ -1,3 +1,4 @@ +{ hashInvalidator ? "" }: with import ./config.nix; let { @@ -21,6 +22,17 @@ let { ''; }; + fod_input = mkDerivation { + name = "fod-input"; + buildCommand = '' + echo ${hashInvalidator} + echo FOD > $out + ''; + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = "1dq9p0hnm1y75q2x40fws5887bq1r840hzdxak0a9djbwvx0b16d"; + }; + body = mkDerivation { name = "dependencies-top"; builder = ./dependencies.builder0.sh + "/FOOBAR/../."; @@ -29,6 +41,7 @@ let { input1_drv = input1; input2_drv = input2; input0_drv = input0; + fod_input_drv = fod_input; meta.description = "Random test package"; }; diff --git a/tests/dependencies.sh b/tests/dependencies.sh index d5cd30396..b93dacac0 100644 --- a/tests/dependencies.sh +++ b/tests/dependencies.sh @@ -53,3 +53,20 @@ nix-store -q --referrers-closure "$input2OutPath" | grep "$outPath" # Check that the derivers are set properly. test $(nix-store -q --deriver "$outPath") = "$drvPath" nix-store -q --deriver "$input2OutPath" | grepQuiet -- "-input-2.drv" + +# --valid-derivers returns the currently single valid .drv file +test "$(nix-store -q --valid-derivers "$outPath")" = "$drvPath" + +# instantiate a different drv with the same output +drvPath2=$(nix-instantiate dependencies.nix --argstr hashInvalidator yay) + +# now --valid-derivers returns both +test "$(nix-store -q --valid-derivers "$outPath" | sort)" = "$(sort <<< "$drvPath"$'\n'"$drvPath2")" + +# check that nix-store --valid-derivers only returns existing drv +nix-store --delete "$drvPath" +test "$(nix-store -q --valid-derivers "$outPath")" = "$drvPath2" + +# check that --valid-derivers returns nothing when there are no valid derivers +nix-store --delete "$drvPath2" +test -z "$(nix-store -q --valid-derivers "$outPath")" diff --git a/tests/dyn-drv/eval-outputOf.sh b/tests/dyn-drv/eval-outputOf.sh index 99d917c06..9467feb8d 100644 --- a/tests/dyn-drv/eval-outputOf.sh +++ b/tests/dyn-drv/eval-outputOf.sh @@ -13,14 +13,14 @@ nix --experimental-features 'nix-command' eval --impure --expr \ # the future so it does work, but there are some design questions to # resolve first. Adding a test so we don't liberalise it by accident. expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \ - 'builtins.outputOf (import ../dependencies.nix) "out"' \ + 'builtins.outputOf (import ../dependencies.nix {}) "out"' \ | grepQuiet "value is a set while a string was expected" # Test that "DrvDeep" string contexts are not supported at this time # # Like the above, this is a restriction we could relax later. expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \ - 'builtins.outputOf (import ../dependencies.nix).drvPath "out"' \ + 'builtins.outputOf (import ../dependencies.nix {}).drvPath "out"' \ | grepQuiet "has a context which refers to a complete source and binary closure. This is not supported at this time" # Test using `builtins.outputOf` with static derivations diff --git a/tests/export-graph.nix b/tests/export-graph.nix index fdac9583d..64fe36bd1 100644 --- a/tests/export-graph.nix +++ b/tests/export-graph.nix @@ -17,13 +17,13 @@ rec { foo."bar.runtimeGraph" = mkDerivation { name = "dependencies"; builder = builtins.toFile "build-graph-builder" "${printRefs}"; - exportReferencesGraph = ["refs" (import ./dependencies.nix)]; + exportReferencesGraph = ["refs" (import ./dependencies.nix {})]; }; foo."bar.buildGraph" = mkDerivation { name = "dependencies"; builder = builtins.toFile "build-graph-builder" "${printRefs}"; - exportReferencesGraph = ["refs" (import ./dependencies.nix).drvPath]; + exportReferencesGraph = ["refs" (import ./dependencies.nix {}).drvPath]; }; } diff --git a/tests/remote-store.sh b/tests/remote-store.sh index ea32a20d3..50e6f24b9 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -24,7 +24,7 @@ fi import ( mkDerivation { name = "foo"; - bla = import ./dependencies.nix; + bla = import ./dependencies.nix {}; buildCommand = " echo \\\"hi\\\" > $out "; From 2f5d3da8062ae58242a8de2bad470a66478edea4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Aug 2023 09:53:12 -0400 Subject: [PATCH 198/909] Introduce `OutputName` and `OutputNameView` type aliases Hopefully they make the code easier to understand! --- src/libexpr/primops.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/build/local-derivation-goal.hh | 2 +- src/libstore/derivations.cc | 12 +++++----- src/libstore/derivations.hh | 12 +++++----- src/libstore/derived-path.cc | 4 ++-- src/libstore/derived-path.hh | 4 ++-- src/libstore/downstream-placeholder.cc | 4 ++-- src/libstore/downstream-placeholder.hh | 4 ++-- src/libstore/outputs-spec.hh | 26 +++++++++++++++------ src/libstore/realisation.hh | 6 ++--- 11 files changed, 45 insertions(+), 33 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5067da449..6b99b91e4 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1843,7 +1843,7 @@ static void prim_outputOf(EvalState & state, const PosIdx pos, Value * * args, V { SingleDerivedPath drvPath = state.coerceToSingleDerivedPath(pos, *args[0], "while evaluating the first argument to builtins.outputOf"); - std::string_view outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf"); + OutputNameView outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf"); state.mkSingleDerivedPathString( SingleDerivedPath::Built { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 78f943d1f..64b55ca6a 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2955,7 +2955,7 @@ bool LocalDerivationGoal::isReadDesc(int fd) } -StorePath LocalDerivationGoal::makeFallbackPath(std::string_view outputName) +StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName) { return worker.store.makeStorePath( "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 8827bfca3..0a05081c7 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -297,7 +297,7 @@ struct LocalDerivationGoal : public DerivationGoal * @todo Add option to randomize, so we can audit whether our * rewrites caught everything */ - StorePath makeFallbackPath(std::string_view outputName); + StorePath makeFallbackPath(OutputNameView outputName); }; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 0b8bdaf1c..dc32c3847 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -12,7 +12,7 @@ namespace nix { -std::optional DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const +std::optional DerivationOutput::path(const Store & store, std::string_view drvName, OutputNameView outputName) const { return std::visit(overloaded { [](const DerivationOutput::InputAddressed & doi) -> std::optional { @@ -36,7 +36,7 @@ std::optional DerivationOutput::path(const Store & store, std::string } -StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const +StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, OutputNameView outputName) const { return store.makeFixedOutputPathFromCA( outputPathName(drvName, outputName), @@ -466,7 +466,7 @@ bool isDerivation(std::string_view fileName) } -std::string outputPathName(std::string_view drvName, std::string_view outputName) { +std::string outputPathName(std::string_view drvName, OutputNameView outputName) { std::string res { drvName }; if (outputName != "out") { res += "-"; @@ -810,7 +810,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr } -std::string hashPlaceholder(const std::string_view outputName) +std::string hashPlaceholder(const OutputNameView outputName) { // FIXME: memoize? return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false); @@ -963,7 +963,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const const Hash impureOutputHash = hashString(htSHA256, "impure"); nlohmann::json DerivationOutput::toJSON( - const Store & store, std::string_view drvName, std::string_view outputName) const + const Store & store, std::string_view drvName, OutputNameView outputName) const { nlohmann::json res = nlohmann::json::object(); std::visit(overloaded { @@ -990,7 +990,7 @@ nlohmann::json DerivationOutput::toJSON( DerivationOutput DerivationOutput::fromJSON( - const Store & store, std::string_view drvName, std::string_view outputName, + const Store & store, std::string_view drvName, OutputNameView outputName, const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings) { diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index a92082089..106056f2d 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -55,7 +55,7 @@ struct DerivationOutput * @param drvName The name of the derivation this is an output of, without the `.drv`. * @param outputName The name of this output. */ - StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; + StorePath path(const Store & store, std::string_view drvName, OutputNameView outputName) const; GENERATE_CMP(CAFixed, me->ca); }; @@ -132,19 +132,19 @@ struct DerivationOutput * the safer interface provided by * BasicDerivation::outputsAndOptPaths */ - std::optional path(const Store & store, std::string_view drvName, std::string_view outputName) const; + std::optional path(const Store & store, std::string_view drvName, OutputNameView outputName) const; nlohmann::json toJSON( const Store & store, std::string_view drvName, - std::string_view outputName) const; + OutputNameView outputName) const; /** * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivationOutput fromJSON( const Store & store, std::string_view drvName, - std::string_view outputName, + OutputNameView outputName, const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); }; @@ -405,7 +405,7 @@ bool isDerivation(std::string_view fileName); * This is usually -, but is just when * the output name is "out". */ -std::string outputPathName(std::string_view drvName, std::string_view outputName); +std::string outputPathName(std::string_view drvName, OutputNameView outputName); /** @@ -499,7 +499,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr * own outputs without needing to use the hash of a derivation in * itself, making the hash near-impossible to calculate. */ -std::string hashPlaceholder(const std::string_view outputName); +std::string hashPlaceholder(const OutputNameView outputName); extern const Hash impureOutputHash; diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 3594b7570..47d784deb 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -167,7 +167,7 @@ void drvRequireExperiment( SingleDerivedPath::Built SingleDerivedPath::Built::parse( const Store & store, ref drv, - std::string_view output, + OutputNameView output, const ExperimentalFeatureSettings & xpSettings) { drvRequireExperiment(*drv, xpSettings); @@ -179,7 +179,7 @@ SingleDerivedPath::Built SingleDerivedPath::Built::parse( DerivedPath::Built DerivedPath::Built::parse( const Store & store, ref drv, - std::string_view outputsS, + OutputNameView outputsS, const ExperimentalFeatureSettings & xpSettings) { drvRequireExperiment(*drv, xpSettings); diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index ec30dac61..4d7033df2 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -42,7 +42,7 @@ struct SingleDerivedPath; */ struct SingleDerivedPathBuilt { ref drvPath; - std::string output; + OutputName output; /** * Get the store path this is ultimately derived from (by realising @@ -71,7 +71,7 @@ struct SingleDerivedPathBuilt { */ static SingleDerivedPathBuilt parse( const Store & store, ref drvPath, - std::string_view outputs, + OutputNameView outputs, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index d951b7b7d..7e3f7548d 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -11,7 +11,7 @@ std::string DownstreamPlaceholder::render() const DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput( const StorePath & drvPath, - std::string_view outputName, + OutputNameView outputName, const ExperimentalFeatureSettings & xpSettings) { xpSettings.require(Xp::CaDerivations); @@ -25,7 +25,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput( DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( const DownstreamPlaceholder & placeholder, - std::string_view outputName, + OutputNameView outputName, const ExperimentalFeatureSettings & xpSettings) { xpSettings.require(Xp::DynamicDerivations); diff --git a/src/libstore/downstream-placeholder.hh b/src/libstore/downstream-placeholder.hh index d58a2ac14..c911ecea2 100644 --- a/src/libstore/downstream-placeholder.hh +++ b/src/libstore/downstream-placeholder.hh @@ -58,7 +58,7 @@ public: */ static DownstreamPlaceholder unknownCaOutput( const StorePath & drvPath, - std::string_view outputName, + OutputNameView outputName, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** @@ -72,7 +72,7 @@ public: */ static DownstreamPlaceholder unknownDerivation( const DownstreamPlaceholder & drvPlaceholder, - std::string_view outputName, + OutputNameView outputName, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** diff --git a/src/libstore/outputs-spec.hh b/src/libstore/outputs-spec.hh index ae19f1040..1ef99a5fc 100644 --- a/src/libstore/outputs-spec.hh +++ b/src/libstore/outputs-spec.hh @@ -13,24 +13,36 @@ namespace nix { +/** + * An (owned) output name. Just a type alias used to make code more + * readible. + */ +typedef std::string OutputName; + +/** + * A borrowed output name. Just a type alias used to make code more + * readible. + */ +typedef std::string_view OutputNameView; + struct OutputsSpec { /** * A non-empty set of outputs, specified by name */ - struct Names : std::set { - using std::set::set; + struct Names : std::set { + using std::set::set; /* These need to be "inherited manually" */ - Names(const std::set & s) - : std::set(s) + Names(const std::set & s) + : std::set(s) { assert(!empty()); } /** * Needs to be "inherited manually" */ - Names(std::set && s) - : std::set(s) + Names(std::set && s) + : std::set(s) { assert(!empty()); } /* This set should always be non-empty, so we delete this @@ -57,7 +69,7 @@ struct OutputsSpec { */ OutputsSpec() = delete; - bool contains(const std::string & output) const; + bool contains(const OutputName & output) const; /** * Create a new OutputsSpec which is the union of this and that. diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 0548b30c1..559483ce3 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -34,7 +34,7 @@ struct DrvOutput { /** * The name of the output. */ - std::string outputName; + OutputName outputName; std::string to_string() const; @@ -84,7 +84,7 @@ struct Realisation { * Since these are the outputs of a single derivation, we know the * output names are unique so we can use them as the map key. */ -typedef std::map SingleDrvOutputs; +typedef std::map SingleDrvOutputs; /** * Collection type for multiple derivations' outputs' `Realisation`s. @@ -146,7 +146,7 @@ public: MissingRealisation(DrvOutput & outputId) : MissingRealisation(outputId.outputName, outputId.strHash()) {} - MissingRealisation(std::string_view drv, std::string outputName) + MissingRealisation(std::string_view drv, OutputName outputName) : Error( "cannot operate on output '%s' of the " "unbuilt derivation '%s'", outputName, From 1c4caef14b51dbb4f749c45311c8e2c9acb75a60 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 16 Aug 2023 00:05:35 -0400 Subject: [PATCH 199/909] Throw `MissingRealisation` not plain `Error` in both `resolveDerivedPath` Now we are consistent with the other `resolveDerivedPath`, and other such functions. --- src/libstore/misc.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 631213306..c043b9b93 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -399,8 +399,7 @@ StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store store.printStorePath(drvPath), bfd.output); auto & optPath = outputPaths.at(bfd.output); if (!optPath) - throw Error("'%s' does not yet map to a known concrete store path", - bfd.to_string(store)); + throw MissingRealisation(bfd.drvPath->to_string(store), bfd.output); return *optPath; }, }, req.raw()); From 692074f7142fcf8ede1266b6d8cbbd5feaf3221f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 15 Jan 2023 17:47:24 -0500 Subject: [PATCH 200/909] Use `Worker::makeDerivationGoal` less We're about to split up `DerivationGoal` a bit. At that point `makeDerivationGoal` will mean something more specific than it does today. (Perhaps a future rename will make this clearer.) On the other hand, the more public `Worker::makeGoal` function will continue to work exactly as before. So by moving some call sites to use that instead, we preemptively avoid issues in the next step. --- src/libstore/build/derivation-goal.cc | 14 ++++++++++++-- src/libstore/build/entry-points.cc | 7 +++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 84da7f2e1..ea56c0288 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -380,7 +380,12 @@ void DerivationGoal::gaveUpOnSubstitution() worker.store.printStorePath(i.first)); } - addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); + addWaitee(worker.makeGoal( + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(i.first), + .outputs = i.second, + }, + buildMode == bmRepair ? bmRepair : bmNormal)); } /* Copy the input sources from the eval store to the build @@ -452,7 +457,12 @@ void DerivationGoal::repairClosure() if (drvPath2 == outputsToDrv.end()) addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair))); else - addWaitee(worker.makeDerivationGoal(drvPath2->second, OutputsSpec::All(), bmRepair)); + addWaitee(worker.makeGoal( + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(drvPath2->second), + .outputs = OutputsSpec::All { }, + }, + bmRepair)); } if (waitees.empty()) { diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index e941b4e65..f71fb35a6 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -124,8 +124,11 @@ void Store::repairPath(const StorePath & path) auto info = queryPathInfo(path); if (info->deriver && isValidPath(*info->deriver)) { goals.clear(); - // FIXME: Should just build the specific output we need. - goals.insert(worker.makeDerivationGoal(*info->deriver, OutputsSpec::All { }, bmRepair)); + goals.insert(worker.makeGoal(DerivedPath::Built { + .drvPath = makeConstantStorePathRef(*info->deriver), + // FIXME: Should just build the specific output we need. + .outputs = OutputsSpec::All { }, + }, bmRepair)); worker.run(goals); } else throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path)); From 5e3986f59cb58f48186a49dcec7aa317b4787522 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 Mar 2021 16:24:49 -0500 Subject: [PATCH 201/909] Adapt scheduler to work with dynamic derivations To avoid dealing with an optional `drvPath` (because we might not know it yet) everywhere, make an `CreateDerivationAndRealiseGoal`. This goal just builds/substitutes the derivation file, and then kicks of a build for that obtained derivation; in other words it does the chaining of goals when the drv file is missing (as can already be the case) or computed (new case). This also means the `getDerivation` state can be removed from `DerivationGoal`, which makes the `BasicDerivation` / in memory case and `Derivation` / drv file file case closer together. The map type is factored out for clarity, and because we will soon hvae a second use for it (`Derivation` itself). Co-authored-by: Robert Hensing --- .../create-derivation-and-realise-goal.cc | 157 ++++++++++++++++++ .../create-derivation-and-realise-goal.hh | 96 +++++++++++ src/libstore/build/derivation-goal.cc | 22 +-- src/libstore/build/derivation-goal.hh | 15 +- .../build/drv-output-substitution-goal.hh | 4 +- src/libstore/build/entry-points.cc | 11 +- src/libstore/build/goal.cc | 2 +- src/libstore/build/goal.hh | 22 ++- src/libstore/build/substitution-goal.hh | 4 +- src/libstore/build/worker.cc | 118 ++++++++++--- src/libstore/build/worker.hh | 22 +++ src/libstore/derived-path-map.cc | 33 ++++ src/libstore/derived-path-map.hh | 73 ++++++++ tests/dyn-drv/build-built-drv.sh | 4 +- 14 files changed, 525 insertions(+), 58 deletions(-) create mode 100644 src/libstore/build/create-derivation-and-realise-goal.cc create mode 100644 src/libstore/build/create-derivation-and-realise-goal.hh create mode 100644 src/libstore/derived-path-map.cc create mode 100644 src/libstore/derived-path-map.hh diff --git a/src/libstore/build/create-derivation-and-realise-goal.cc b/src/libstore/build/create-derivation-and-realise-goal.cc new file mode 100644 index 000000000..b01042f00 --- /dev/null +++ b/src/libstore/build/create-derivation-and-realise-goal.cc @@ -0,0 +1,157 @@ +#include "create-derivation-and-realise-goal.hh" +#include "worker.hh" + +namespace nix { + +CreateDerivationAndRealiseGoal::CreateDerivationAndRealiseGoal(ref drvReq, + const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) + : Goal(worker, DerivedPath::Built { .drvPath = drvReq, .outputs = wantedOutputs }) + , drvReq(drvReq) + , wantedOutputs(wantedOutputs) + , buildMode(buildMode) +{ + state = &CreateDerivationAndRealiseGoal::getDerivation; + name = fmt( + "outer obtaining drv from '%s' and then building outputs %s", + drvReq->to_string(worker.store), + std::visit(overloaded { + [&](const OutputsSpec::All) -> std::string { + return "* (all of them)"; + }, + [&](const OutputsSpec::Names os) { + return concatStringsSep(", ", quoteStrings(os)); + }, + }, wantedOutputs.raw)); + trace("created outer"); + + worker.updateProgress(); +} + + +CreateDerivationAndRealiseGoal::~CreateDerivationAndRealiseGoal() +{ +} + + +static StorePath pathPartOfReq(const SingleDerivedPath & req) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + return bo.path; + }, + [&](const SingleDerivedPath::Built & bfd) { + return pathPartOfReq(*bfd.drvPath); + }, + }, req.raw()); +} + + +std::string CreateDerivationAndRealiseGoal::key() +{ + /* Ensure that derivations get built in order of their name, + i.e. a derivation named "aardvark" always comes before "baboon". And + substitution goals and inner derivation goals always happen before + derivation goals (due to "b$"). */ + return "c$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store); +} + + +void CreateDerivationAndRealiseGoal::timedOut(Error && ex) +{ +} + + +void CreateDerivationAndRealiseGoal::work() +{ + (this->*state)(); +} + + +void CreateDerivationAndRealiseGoal::addWantedOutputs(const OutputsSpec & outputs) +{ + /* If we already want all outputs, there is nothing to do. */ + auto newWanted = wantedOutputs.union_(outputs); + bool needRestart = !newWanted.isSubsetOf(wantedOutputs); + wantedOutputs = newWanted; + + if (!needRestart) return; + + if (!optDrvPath) + // haven't started steps where the outputs matter yet + return; + worker.makeDerivationGoal(*optDrvPath, outputs, buildMode); +} + + +void CreateDerivationAndRealiseGoal::getDerivation() +{ + trace("outer init"); + + /* The first thing to do is to make sure that the derivation + exists. If it doesn't, it may be created through a + substitute. */ + if (auto optDrvPath = [this]() -> std::optional { + if (buildMode != bmNormal) return std::nullopt; + + auto drvPath = StorePath::dummy; + try { + drvPath = resolveDerivedPath(worker.store, *drvReq); + } catch (MissingRealisation) { + return std::nullopt; + } + return worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath) + ? std::optional { drvPath } + : std::nullopt; + }()) { + trace(fmt("already have drv '%s' for '%s', can go straight to building", + worker.store.printStorePath(*optDrvPath), + drvReq->to_string(worker.store))); + + loadAndBuildDerivation(); + } else { + trace("need to obtain drv we want to build"); + + addWaitee(worker.makeGoal(DerivedPath::fromSingle(*drvReq))); + + state = &CreateDerivationAndRealiseGoal::loadAndBuildDerivation; + if (waitees.empty()) work(); + } +} + + +void CreateDerivationAndRealiseGoal::loadAndBuildDerivation() +{ + trace("outer load and build derivation"); + + if (nrFailed != 0) { + amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store))); + return; + } + + StorePath drvPath = resolveDerivedPath(worker.store, *drvReq); + /* Build this step! */ + concreteDrvGoal = worker.makeDerivationGoal(drvPath, wantedOutputs, buildMode); + addWaitee(upcast_goal(concreteDrvGoal)); + state = &CreateDerivationAndRealiseGoal::buildDone; + optDrvPath = std::move(drvPath); + if (waitees.empty()) work(); +} + + +void CreateDerivationAndRealiseGoal::buildDone() +{ + trace("outer build done"); + + buildResult = upcast_goal(concreteDrvGoal)->getBuildResult(DerivedPath::Built { + .drvPath = drvReq, + .outputs = wantedOutputs, + }); + + if (buildResult.success()) + amDone(ecSuccess); + else + amDone(ecFailed, Error("building '%s' failed", drvReq->to_string(worker.store))); +} + + +} diff --git a/src/libstore/build/create-derivation-and-realise-goal.hh b/src/libstore/build/create-derivation-and-realise-goal.hh new file mode 100644 index 000000000..ca936fc95 --- /dev/null +++ b/src/libstore/build/create-derivation-and-realise-goal.hh @@ -0,0 +1,96 @@ +#pragma once + +#include "parsed-derivations.hh" +#include "lock.hh" +#include "store-api.hh" +#include "pathlocks.hh" +#include "goal.hh" + +namespace nix { + +struct DerivationGoal; + +/** + * This goal type is essentially the serial composition (like function + * composition) of a goal for getting a derivation, and then a + * `DerivationGoal` using the newly-obtained derivation. + * + * In the (currently experimental) general inductive case of derivations + * that are themselves build outputs, that first goal will be *another* + * `CreateDerivationAndRealiseGoal`. In the (much more common) base-case + * where the derivation has no provence and is just referred to by + * (content-addressed) store path, that first goal is a + * `SubstitutionGoal`. + * + * If we already have the derivation (e.g. if the evalutator has created + * the derivation locally and then instructured the store to build it), + * we can skip the first goal entirely as a small optimization. + */ +struct CreateDerivationAndRealiseGoal : public Goal +{ + /** + * How to obtain a store path of the derivation to build. + */ + ref drvReq; + + /** + * The path of the derivation, once obtained. + **/ + std::optional optDrvPath; + + /** + * The goal for the corresponding concrete derivation. + **/ + std::shared_ptr concreteDrvGoal; + + /** + * The specific outputs that we need to build. + */ + OutputsSpec wantedOutputs; + + typedef void (CreateDerivationAndRealiseGoal::*GoalState)(); + GoalState state; + + /** + * The final output paths of the build. + * + * - For input-addressed derivations, always the precomputed paths + * + * - For content-addressed derivations, calcuated from whatever the + * hash ends up being. (Note that fixed outputs derivations that + * produce the "wrong" output still install that data under its + * true content-address.) + */ + OutputPathMap finalOutputs; + + BuildMode buildMode; + + CreateDerivationAndRealiseGoal(ref drvReq, + const OutputsSpec & wantedOutputs, Worker & worker, + BuildMode buildMode = bmNormal); + virtual ~CreateDerivationAndRealiseGoal(); + + void timedOut(Error && ex) override; + + std::string key() override; + + void work() override; + + /** + * Add wanted outputs to an already existing derivation goal. + */ + void addWantedOutputs(const OutputsSpec & outputs); + + /** + * The states. + */ + void getDerivation(); + void loadAndBuildDerivation(); + void buildDone(); + + JobCategory jobCategory() const override { + return JobCategory::Administration; + }; +}; + +} diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index ea56c0288..bec0bc538 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -71,7 +71,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, , wantedOutputs(wantedOutputs) , buildMode(buildMode) { - state = &DerivationGoal::getDerivation; + state = &DerivationGoal::loadDerivation; name = fmt( "building of '%s' from .drv file", DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); @@ -164,24 +164,6 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) } -void DerivationGoal::getDerivation() -{ - trace("init"); - - /* The first thing to do is to make sure that the derivation - exists. If it doesn't, it may be created through a - substitute. */ - if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) { - loadDerivation(); - return; - } - - addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath))); - - state = &DerivationGoal::loadDerivation; -} - - void DerivationGoal::loadDerivation() { trace("loading derivation"); @@ -1493,7 +1475,7 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) if (!useDerivation) return; auto & fullDrv = *dynamic_cast(drv.get()); - auto * dg = dynamic_cast(&*waitee); + auto * dg = tryGetConcreteDrvGoal(waitee); if (!dg) return; auto outputs = fullDrv.inputDrvs.find(dg->drvPath); diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 9d6fe1c0f..62b122c27 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -50,6 +50,13 @@ struct InitialOutput { std::optional known; }; +/** + * A goal for building some or all of the outputs of a derivation. + * + * The derivation must already be present, either in the store in a drv + * or in memory. If the derivation itself needs to be gotten first, a + * `CreateDerivationAndRealiseGoal` goal must be used instead. + */ struct DerivationGoal : public Goal { /** @@ -66,8 +73,7 @@ struct DerivationGoal : public Goal std::shared_ptr resolvedDrvGoal; /** - * The specific outputs that we need to build. Empty means all of - * them. + * The specific outputs that we need to build. */ OutputsSpec wantedOutputs; @@ -229,7 +235,6 @@ struct DerivationGoal : public Goal /** * The states. */ - void getDerivation(); void loadDerivation(); void haveDerivation(); void outputsSubstitutionTried(); @@ -334,7 +339,9 @@ struct DerivationGoal : public Goal StorePathSet exportReferences(const StorePathSet & storePaths); - JobCategory jobCategory() override { return JobCategory::Build; }; + JobCategory jobCategory() const override { + return JobCategory::Build; + }; }; MakeError(NotDeterministic, BuildError); diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 5d1253a71..da2426e5e 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -73,7 +73,9 @@ public: void work() override; void handleEOF(int fd) override; - JobCategory jobCategory() override { return JobCategory::Substitution; }; + JobCategory jobCategory() const override { + return JobCategory::Substitution; + }; }; } diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index f71fb35a6..f0f0e5519 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -1,5 +1,6 @@ #include "worker.hh" #include "substitution-goal.hh" +#include "create-derivation-and-realise-goal.hh" #include "derivation-goal.hh" #include "local-store.hh" @@ -15,7 +16,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod worker.run(goals); - StorePathSet failed; + StringSet failed; std::optional ex; for (auto & i : goals) { if (i->ex) { @@ -25,8 +26,10 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod ex = std::move(i->ex); } if (i->exitCode != Goal::ecSuccess) { - if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->drvPath); - else if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->storePath); + if (auto i2 = dynamic_cast(i.get())) + failed.insert(i2->drvReq->to_string(*this)); + else if (auto i2 = dynamic_cast(i.get())) + failed.insert(printStorePath(i2->storePath)); } } @@ -35,7 +38,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); - throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed)); + throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed))); } } diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index ca7097a68..f8db98280 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -11,7 +11,7 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { } -BuildResult Goal::getBuildResult(const DerivedPath & req) { +BuildResult Goal::getBuildResult(const DerivedPath & req) const { BuildResult res { buildResult }; if (auto pbp = std::get_if(&req)) { diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index d3127caea..01d3c3491 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -41,8 +41,24 @@ typedef std::map WeakGoalMap; * of each category in parallel. */ enum struct JobCategory { + /** + * A build of a derivation; it will use CPU and disk resources. + */ Build, + /** + * A substitution an arbitrary store object; it will use network resources. + */ Substitution, + /** + * A goal that does no "real" work by itself, and just exists to depend on + * other goals which *do* do real work. These goals therefore are not + * limited. + * + * These goals cannot infinitely create themselves, so there is no risk of + * a "fork bomb" type situation (which would be a problem even though the + * goal do no real work) either. + */ + Administration, }; struct Goal : public std::enable_shared_from_this @@ -110,7 +126,7 @@ public: * sake of both privacy and determinism, and this "safe accessor" * ensures we don't. */ - BuildResult getBuildResult(const DerivedPath &); + BuildResult getBuildResult(const DerivedPath &) const; /** * Exception containing an error message, if any. @@ -144,7 +160,7 @@ public: void trace(std::string_view s); - std::string getName() + std::string getName() const { return name; } @@ -166,7 +182,7 @@ public: * @brief Hint for the scheduler, which concurrency limit applies. * @see JobCategory */ - virtual JobCategory jobCategory() = 0; + virtual JobCategory jobCategory() const = 0; }; void addToWeakGoals(WeakGoals & goals, GoalPtr p); diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 1b693baa1..1d389d328 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -117,7 +117,9 @@ public: /* Called by destructor, can't be overridden */ void cleanup() override final; - JobCategory jobCategory() override { return JobCategory::Substitution; }; + JobCategory jobCategory() const override { + return JobCategory::Substitution; + }; }; } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b58fc5c1c..f65f63b99 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -2,6 +2,7 @@ #include "worker.hh" #include "substitution-goal.hh" #include "drv-output-substitution-goal.hh" +#include "create-derivation-and-realise-goal.hh" #include "local-derivation-goal.hh" #include "hook-instance.hh" @@ -41,6 +42,24 @@ Worker::~Worker() } +std::shared_ptr Worker::makeCreateDerivationAndRealiseGoal( + ref drvReq, + const OutputsSpec & wantedOutputs, + BuildMode buildMode) +{ + std::weak_ptr & goal_weak = outerDerivationGoals.ensureSlot(*drvReq).value; + std::shared_ptr goal = goal_weak.lock(); + if (!goal) { + goal = std::make_shared(drvReq, wantedOutputs, *this, buildMode); + goal_weak = goal; + wakeUp(goal); + } else { + goal->addWantedOutputs(wantedOutputs); + } + return goal; +} + + std::shared_ptr Worker::makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, @@ -111,10 +130,7 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { [&](const DerivedPath::Built & bfd) -> GoalPtr { - if (auto bop = std::get_if(&*bfd.drvPath)) - return makeDerivationGoal(bop->path, bfd.outputs, buildMode); - else - throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); + return makeCreateDerivationAndRealiseGoal(bfd.drvPath, bfd.outputs, buildMode); }, [&](const DerivedPath::Opaque & bo) -> GoalPtr { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); @@ -123,24 +139,46 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) } +template +static void cullMap(std::map & goalMap, F f) +{ + for (auto i = goalMap.begin(); i != goalMap.end();) + if (!f(i->second)) + i = goalMap.erase(i); + else ++i; +} + + template static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { /* !!! inefficient */ - for (auto i = goalMap.begin(); - i != goalMap.end(); ) - if (i->second.lock() == goal) { - auto j = i; ++j; - goalMap.erase(i); - i = j; - } - else ++i; + cullMap(goalMap, [&](const std::weak_ptr & gp) -> bool { + return gp.lock() != goal; + }); +} + +template +static void removeGoal(std::shared_ptr goal, std::map>::ChildNode> & goalMap); + +template +static void removeGoal(std::shared_ptr goal, std::map>::ChildNode> & goalMap) +{ + /* !!! inefficient */ + cullMap(goalMap, [&](DerivedPathMap>::ChildNode & node) -> bool { + if (node.value.lock() == goal) + node.value.reset(); + removeGoal(goal, node.childMap); + return !node.value.expired() || !node.childMap.empty(); + }); } void Worker::removeGoal(GoalPtr goal) { - if (auto drvGoal = std::dynamic_pointer_cast(goal)) + if (auto drvGoal = std::dynamic_pointer_cast(goal)) + nix::removeGoal(drvGoal, outerDerivationGoals.map); + else if (auto drvGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(drvGoal, derivationGoals); else if (auto subGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(subGoal, substitutionGoals); @@ -198,8 +236,19 @@ void Worker::childStarted(GoalPtr goal, const std::set & fds, child.respectTimeouts = respectTimeouts; children.emplace_back(child); if (inBuildSlot) { - if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++; - else nrLocalBuilds++; + switch (goal->jobCategory()) { + case JobCategory::Substitution: + nrSubstitutions++; + break; + case JobCategory::Build: + nrLocalBuilds++; + break; + case JobCategory::Administration: + /* Intentionally not limited, see docs */ + break; + default: + abort(); + } } } @@ -211,12 +260,20 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers) if (i == children.end()) return; if (i->inBuildSlot) { - if (goal->jobCategory() == JobCategory::Substitution) { + switch (goal->jobCategory()) { + case JobCategory::Substitution: assert(nrSubstitutions > 0); nrSubstitutions--; - } else { + break; + case JobCategory::Build: assert(nrLocalBuilds > 0); nrLocalBuilds--; + break; + case JobCategory::Administration: + /* Intentionally not limited, see docs */ + break; + default: + abort(); } } @@ -267,9 +324,9 @@ void Worker::run(const Goals & _topGoals) for (auto & i : _topGoals) { topGoals.insert(i); - if (auto goal = dynamic_cast(i.get())) { + if (auto goal = dynamic_cast(i.get())) { topPaths.push_back(DerivedPath::Built { - .drvPath = makeConstantStorePathRef(goal->drvPath), + .drvPath = goal->drvReq, .outputs = goal->wantedOutputs, }); } else if (auto goal = dynamic_cast(i.get())) { @@ -522,11 +579,26 @@ void Worker::markContentsGood(const StorePath & path) } -GoalPtr upcast_goal(std::shared_ptr subGoal) { - return subGoal; -} -GoalPtr upcast_goal(std::shared_ptr subGoal) { +GoalPtr upcast_goal(std::shared_ptr subGoal) +{ return subGoal; } +GoalPtr upcast_goal(std::shared_ptr subGoal) +{ + return subGoal; +} + +GoalPtr upcast_goal(std::shared_ptr subGoal) +{ + return subGoal; +} + +const DerivationGoal * tryGetConcreteDrvGoal(GoalPtr waitee) +{ + auto * odg = dynamic_cast(&*waitee); + if (!odg) return nullptr; + return &*odg->concreteDrvGoal; +} + } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 5abceca0d..a778e311c 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -4,6 +4,7 @@ #include "types.hh" #include "lock.hh" #include "store-api.hh" +#include "derived-path-map.hh" #include "goal.hh" #include "realisation.hh" @@ -13,6 +14,7 @@ namespace nix { /* Forward definition. */ +struct CreateDerivationAndRealiseGoal; struct DerivationGoal; struct PathSubstitutionGoal; class DrvOutputSubstitutionGoal; @@ -31,9 +33,23 @@ class DrvOutputSubstitutionGoal; */ GoalPtr upcast_goal(std::shared_ptr subGoal); GoalPtr upcast_goal(std::shared_ptr subGoal); +GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; +/** + * The current implementation of impure derivations has + * `DerivationGoal`s accumulate realisations from their waitees. + * Unfortunately, `DerivationGoal`s don't directly depend on other + * goals, but instead depend on `CreateDerivationAndRealiseGoal`s. + * + * We try not to share any of the details of any goal type with any + * other, for sake of modularity and quicker rebuilds. This means we + * cannot "just" downcast and fish out the field. So as an escape hatch, + * we have made the function, written in `worker.cc` where all the goal + * types are visible, and use it instead. + */ +const DerivationGoal * tryGetConcreteDrvGoal(GoalPtr waitee); /** * A mapping used to remember for each child process to what goal it @@ -102,6 +118,9 @@ private: * Maps used to prevent multiple instantiations of a goal for the * same derivation / path. */ + + DerivedPathMap> outerDerivationGoals; + std::map> derivationGoals; std::map> substitutionGoals; std::map> drvOutputSubstitutionGoals; @@ -189,6 +208,9 @@ public: * @ref DerivationGoal "derivation goal" */ private: + std::shared_ptr makeCreateDerivationAndRealiseGoal( + ref drvPath, + const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, std::function()> mkDrvGoal); diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc new file mode 100644 index 000000000..5c8c7a4f2 --- /dev/null +++ b/src/libstore/derived-path-map.cc @@ -0,0 +1,33 @@ +#include "derived-path-map.hh" + +namespace nix { + +template +typename DerivedPathMap::ChildNode & DerivedPathMap::ensureSlot(const SingleDerivedPath & k) +{ + std::function initIter; + initIter = [&](const auto & k) -> auto & { + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) -> auto & { + // will not overwrite if already there + return map[bo.path]; + }, + [&](const SingleDerivedPath::Built & bfd) -> auto & { + auto & n = initIter(*bfd.drvPath); + return n.childMap[bfd.output]; + }, + }, k.raw()); + }; + return initIter(k); +} + +} + +// instantiations + +#include "create-derivation-and-realise-goal.hh" +namespace nix { + +template struct DerivedPathMap>; + +} diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh new file mode 100644 index 000000000..9ce58206e --- /dev/null +++ b/src/libstore/derived-path-map.hh @@ -0,0 +1,73 @@ +#pragma once + +#include "types.hh" +#include "derived-path.hh" + +namespace nix { + +/** + * A simple Trie, of sorts. Conceptually a map of `SingleDerivedPath` to + * values. + * + * Concretely, an n-ary tree, as described below. A + * `SingleDerivedPath::Opaque` maps to the value of an immediate child + * of the root node. A `SingleDerivedPath::Built` maps to a deeper child + * node: the `SingleDerivedPath::Built::drvPath` is first mapped to a a + * child node (inductively), and then the + * `SingleDerivedPath::Built::output` is used to look up that child's + * child via its map. In this manner, every `SingleDerivedPath` is + * mapped to a child node. + * + * @param V A type to instantiate for each output. It should probably + * should be an "optional" type so not every interior node has to have a + * value. For example, the scheduler uses + * `DerivedPathMap>` to + * remember which goals correspond to which outputs. `* const Something` + * or `std::optional` would also be good choices for + * "optional" types. + */ +template +struct DerivedPathMap { + /** + * A child node (non-root node). + */ + struct ChildNode { + /** + * Value of this child node. + * + * @see DerivedPathMap for what `V` should be. + */ + V value; + + /** + * The map type for the root node. + */ + using Map = std::map; + + /** + * The map of the root node. + */ + Map childMap; + }; + + /** + * The map type for the root node. + */ + using Map = std::map; + + /** + * The map of root node. + */ + Map map; + + /** + * Find the node for `k`, creating it if needed. + * + * The node is referred to as a "slot" on the assumption that `V` is + * some sort of optional type, so the given key can be set or unset + * by changing this node. + */ + ChildNode & ensureSlot(const SingleDerivedPath & k); +}; + +} diff --git a/tests/dyn-drv/build-built-drv.sh b/tests/dyn-drv/build-built-drv.sh index 647be9457..94f3550bd 100644 --- a/tests/dyn-drv/build-built-drv.sh +++ b/tests/dyn-drv/build-built-drv.sh @@ -18,4 +18,6 @@ clearStore drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv) -expectStderr 1 nix build "${drvDep}^out^out" --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" +out2=$(nix build "${drvDep}^out^out" --no-link) + +test $out1 == $out2 From d2e6cfa0750cb38ceef7dc0b8bb31cf3b0387e9c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 25 Aug 2023 17:17:33 +0200 Subject: [PATCH 202/909] tests/lang/eval-okay-pathexists: Add cases --- tests/lang/eval-okay-pathexists.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/lang/eval-okay-pathexists.nix b/tests/lang/eval-okay-pathexists.nix index 50c28ee0c..8eae37e70 100644 --- a/tests/lang/eval-okay-pathexists.nix +++ b/tests/lang/eval-okay-pathexists.nix @@ -1,4 +1,6 @@ -builtins.pathExists (builtins.toPath ./lib.nix) +builtins.pathExists (./lib.nix) +&& builtins.pathExists (builtins.toPath ./lib.nix) +&& builtins.pathExists (builtins.toString ./lib.nix) && builtins.pathExists (builtins.toPath (builtins.toString ./lib.nix)) && !builtins.pathExists (builtins.toPath (builtins.toString ./bla.nix)) && builtins.pathExists ./lib.nix From 1e08e12d8138b09e6872cb498b723ade9ad71d68 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 25 Aug 2023 17:18:37 +0200 Subject: [PATCH 203/909] pathExists: isDir when endswith / Fixes https://github.com/NixOS/nix/issues/8838 --- src/libexpr/primops.cc | 16 +++++++++++++--- tests/lang/eval-okay-pathexists.nix | 1 + 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 283f99a48..915b872c8 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1520,15 +1520,25 @@ static RegisterPrimOp primop_storePath({ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, Value & v) { + auto & arg = *args[0]; + /* We don’t check the path right now, because we don’t want to throw if the path isn’t allowed, but just return false (and we can’t just catch the exception here because we still want to - throw if something in the evaluation of `*args[0]` tries to + throw if something in the evaluation of `arg` tries to access an unauthorized path). */ - auto path = realisePath(state, pos, *args[0], { .checkForPureEval = false }); + auto path = realisePath(state, pos, arg, { .checkForPureEval = false }); + + /* SourcePath doesn't know about trailing slash. */ + auto mustBeDir = arg.type() == nString && arg.str().ends_with("/"); try { - v.mkBool(state.checkSourcePath(path).pathExists()); + auto checked = state.checkSourcePath(path); + auto exists = checked.pathExists(); + if (exists && mustBeDir) { + exists = checked.lstat().type == InputAccessor::tDirectory; + } + v.mkBool(exists); } catch (SysError & e) { /* Don't give away info from errors while canonicalising ‘path’ in restricted mode. */ diff --git a/tests/lang/eval-okay-pathexists.nix b/tests/lang/eval-okay-pathexists.nix index 8eae37e70..e1246e370 100644 --- a/tests/lang/eval-okay-pathexists.nix +++ b/tests/lang/eval-okay-pathexists.nix @@ -1,6 +1,7 @@ builtins.pathExists (./lib.nix) && builtins.pathExists (builtins.toPath ./lib.nix) && builtins.pathExists (builtins.toString ./lib.nix) +&& !builtins.pathExists (builtins.toString ./lib.nix + "/") && builtins.pathExists (builtins.toPath (builtins.toString ./lib.nix)) && !builtins.pathExists (builtins.toPath (builtins.toString ./bla.nix)) && builtins.pathExists ./lib.nix From 696eb79b150011629d86addd10f79c943b2f16b2 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Sun, 27 Aug 2023 04:42:52 -0400 Subject: [PATCH 204/909] test: test behavior of .-prefixed attrPaths --- tests/flakes/absolute-attr-paths.sh | 17 +++++++++++++++++ tests/local.mk | 1 + 2 files changed, 18 insertions(+) create mode 100644 tests/flakes/absolute-attr-paths.sh diff --git a/tests/flakes/absolute-attr-paths.sh b/tests/flakes/absolute-attr-paths.sh new file mode 100644 index 000000000..491adceb7 --- /dev/null +++ b/tests/flakes/absolute-attr-paths.sh @@ -0,0 +1,17 @@ +source ./common.sh + +flake1Dir=$TEST_ROOT/flake1 + +mkdir -p $flake1Dir +cat > $flake1Dir/flake.nix < Date: Mon, 28 Aug 2023 15:43:34 +0200 Subject: [PATCH 205/909] Port the flags of nix-daemon to nix daemon (#8788) The new `nix daemon` command didn't accept the same flags that `nix-daemon` did. * docs(daemon): clarify the daemon trust override flags * fix: change declaration order * docs: add examples of nix daemon usage * Apply suggestions from code review --------- Co-authored-by: Eelco Dolstra Co-authored-by: John Ericson Co-authored-by: tomberek --- src/nix/daemon.cc | 41 ++++++++++++++++++++++++++++++++++++++++- src/nix/daemon.md | 30 +++++++++++++++++++++++++++--- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 1511f9e6e..af428018a 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -500,6 +500,45 @@ static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon); struct CmdDaemon : StoreCommand { + bool stdio = false; + std::optional isTrustedOpt = std::nullopt; + + CmdDaemon() + { + addFlag({ + .longName = "stdio", + .description = "Attach to standard I/O, instead of trying to bind to a UNIX socket.", + .handler = {&stdio, true}, + }); + + addFlag({ + .longName = "force-trusted", + .description = "Force the daemon to trust connecting clients.", + .handler = {[&]() { + isTrustedOpt = Trusted; + }}, + .experimentalFeature = Xp::DaemonTrustOverride, + }); + + addFlag({ + .longName = "force-untrusted", + .description = "Force the daemon to not trust connecting clients. The connection will be processed by the receiving daemon before forwarding commands.", + .handler = {[&]() { + isTrustedOpt = NotTrusted; + }}, + .experimentalFeature = Xp::DaemonTrustOverride, + }); + + addFlag({ + .longName = "default-trust", + .description = "Use Nix's default trust.", + .handler = {[&]() { + isTrustedOpt = std::nullopt; + }}, + .experimentalFeature = Xp::DaemonTrustOverride, + }); + } + std::string description() override { return "daemon to perform store operations on behalf of non-root clients"; @@ -516,7 +555,7 @@ struct CmdDaemon : StoreCommand void run(ref store) override { - runDaemon(false, std::nullopt); + runDaemon(stdio, isTrustedOpt); } }; diff --git a/src/nix/daemon.md b/src/nix/daemon.md index d5cdadf08..b1ea850ed 100644 --- a/src/nix/daemon.md +++ b/src/nix/daemon.md @@ -1,20 +1,44 @@ R""( -# Example +# Examples -* Run the daemon in the foreground: +* Run the daemon: ```console # nix daemon ``` +* Run the daemon and listen on standard I/O instead of binding to a UNIX socket: + + ```console + # nix daemon --stdio + ``` + +* Run the daemon and force all connections to be trusted: + + ```console + # nix daemon --force-trusted + ``` + +* Run the daemon and force all connections to be untrusted: + + ```console + # nix daemon --force-untrusted + ``` + +* Run the daemon, listen on standard I/O, and force all connections to use Nix's default trust: + + ```console + # nix daemon --stdio --default-trust + ``` + # Description This command runs the Nix daemon, which is a required component in multi-user Nix installations. It runs build tasks and other operations on the Nix store on behalf of non-root users. Usually you don't run the daemon directly; instead it's managed by a service -management framework such as `systemd`. +management framework such as `systemd` on Linux, or `launchctl` on Darwin. Note that this daemon does not fork into the background. From 880fef9cdf3b732fa76ec47710edaf572bf91e92 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 28 Aug 2023 20:51:44 +0200 Subject: [PATCH 206/909] do not change existing release notes --- doc/manual/src/release-notes/rl-2.12.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-2.12.md b/doc/manual/src/release-notes/rl-2.12.md index 57d092e01..e1e3efe1a 100644 --- a/doc/manual/src/release-notes/rl-2.12.md +++ b/doc/manual/src/release-notes/rl-2.12.md @@ -2,8 +2,19 @@ * On Linux, Nix can now run builds in a user namespace where they run as root (UID 0) and have 65,536 UIDs available. + This is primarily useful for running containers such as `systemd-nspawn` + inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn]. - This can be used by requiring `uid-range` [system feature] in derivations. + [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. + + A build can enable this by setting the derivation attribute: + + ``` + requiredSystemFeatures = [ "uid-range" ]; + ``` + + The `uid-range` [system feature] requires the [`auto-allocate-uids`] + setting to be enabled. [system feature]: ../command-ref/conf-file.md#conf-system-features From 151120a1ae082c6460f9a54cf795c57d154f6c27 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Mon, 28 Aug 2023 22:14:01 +0200 Subject: [PATCH 207/909] Document nix-prefetch-url defaults (#8878) --- doc/manual/src/command-ref/nix-prefetch-url.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 3bcd209e2..45ef01e02 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -31,15 +31,18 @@ store already contains a file with the same hash and base name. Otherwise, the file is downloaded, and an error is signaled if the actual hash of the file does not match the specified hash. -This command prints the hash on standard output. Additionally, if the -option `--print-path` is used, the path of the downloaded file in the -Nix store is also printed. +This command prints the hash on standard output. +The hash is printed using base-32 unless `--type md5` is specified, +in which case it's printed using base-16. +Additionally, if the option `--print-path` is used, +the path of the downloaded file in the Nix store is also printed. # Options - `--type` *hashAlgo*\ - Use the specified cryptographic hash algorithm, which can be one of - `md5`, `sha1`, `sha256`, and `sha512`. + Use the specified cryptographic hash algorithm, + which can be one of `md5`, `sha1`, `sha256`, and `sha512`. + The default is `sha256`. - `--print-path`\ Print the store path of the downloaded file on standard output. From 56763ff918eb308db23080e560ed2ea3e00c80a7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 29 Aug 2023 15:04:06 +0200 Subject: [PATCH 208/909] Document that redirected tarball flakerefs can specify lastModified --- doc/manual/src/protocols/tarball-fetcher.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/src/protocols/tarball-fetcher.md index 0d3212303..274fa6d63 100644 --- a/doc/manual/src/protocols/tarball-fetcher.md +++ b/doc/manual/src/protocols/tarball-fetcher.md @@ -20,8 +20,8 @@ Link: ; rel="immutable" (Note the required `<` and `>` characters around *flakeref*.) -*flakeref* must be a tarball flakeref. It can contain flake attributes -such as `narHash`, `rev` and `revCount`. If `narHash` is included, its +*flakeref* must be a tarball flakeref. It can contain the tarball flake attributes +`narHash`, `rev`, `revCount` and `lastModified`. If `narHash` is included, its value must be the NAR hash of the unpacked tarball (as computed via `nix hash path`). Nix checks the contents of the returned tarball against the `narHash` attribute. The `rev` and `revCount` attributes From 46478b44ffb477387235b3c597c24178845fb2a3 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Tue, 29 Aug 2023 15:13:35 -0700 Subject: [PATCH 209/909] docs/testing: point out the existence of `GTEST_FILTER` (#8883) --- doc/manual/src/contributing/testing.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index c3c82e3c0..cd94d5cfb 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -7,7 +7,8 @@ under `src/{library_name}/tests` using the [googletest](https://google.github.io/googletest/) and [rapidcheck](https://github.com/emil-e/rapidcheck) frameworks. -You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option. +You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. +Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable. ## Functional tests From 3384f70a3d5ee7bd1d1bf5fa48809c4eac5c8041 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 11 Aug 2023 15:58:43 +0200 Subject: [PATCH 210/909] nixpkgsLibTests: Only test our Nix Interface has changed upstream. It *should* be fine to test 23.05's other Nix versions as those *should* succeed, but that's not the case and it's obfuscating our terrible CI setup's log. --- flake.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index cf7c1fa12..36ba61484 100644 --- a/flake.nix +++ b/flake.nix @@ -656,7 +656,9 @@ tests.nixpkgsLibTests = forAllSystems (system: import (nixpkgs + "/lib/tests/release.nix") - { pkgs = nixpkgsFor.${system}.native; } + { pkgs = nixpkgsFor.${system}.native; + nixVersions = [ self.packages.${system}.nix ]; + } ); metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { From be3362e74782e47b9546de94be97f53540ddfe9e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 14 Aug 2023 14:28:02 +0200 Subject: [PATCH 211/909] Fix nix-copy test --- tests/nixos/nix-copy.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/nixos/nix-copy.nix b/tests/nixos/nix-copy.nix index ef053de03..2981cc2b8 100644 --- a/tests/nixos/nix-copy.nix +++ b/tests/nixos/nix-copy.nix @@ -1,4 +1,6 @@ # Test that ‘nix copy’ works over ssh. +# Run interactively with: +# rm key key.pub; nix run .#hydraJobs.tests.nix-copy.driverInteractive { lib, config, nixpkgs, hostPkgs, ... }: @@ -55,7 +57,9 @@ in { server.wait_for_unit("sshd") client.wait_for_unit("network.target") client.wait_for_unit("getty@tty1.service") - client.wait_for_text("]#") + # Either the prompt: ]# + # or an OCR misreading of it: 1# + client.wait_for_text("[]1]#") # Copy the closure of package A from the client to the server using password authentication, # and check that all prompts are visible From 1bc9257d7c5980fb048b519eeaa2a2b2e8925099 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 3 May 2023 15:36:13 +0200 Subject: [PATCH 212/909] reword description of how realisation works --- .../src/command-ref/nix-store/opt-common.md | 4 +- .../src/command-ref/nix-store/realise.md | 50 +++++++++---------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/doc/manual/src/command-ref/nix-store/opt-common.md b/doc/manual/src/command-ref/nix-store/opt-common.md index dd9a6bf21..104b20076 100644 --- a/doc/manual/src/command-ref/nix-store/opt-common.md +++ b/doc/manual/src/command-ref/nix-store/opt-common.md @@ -1,6 +1,6 @@ -# Options +# Command options -The following options are allowed for all `nix-store` operations, but may not always have an effect. +The following options are allowed for all operations in the `nix-store` command, but may not always have an effect. - [`--add-root`](#opt-add-root) *path* diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index c19aea75e..2881a69b3 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -8,33 +8,37 @@ # Description -The operation `--realise` essentially “builds” the specified store -paths. Realisation is a somewhat overloaded term: +Ensure that the given [store paths] are [valid]. - - If the store path is a *derivation*, realisation ensures that the - output paths of the derivation are [valid] (i.e., - the output path and its closure exist in the file system). This - can be done in several ways. First, it is possible that the - outputs are already valid, in which case we are done - immediately. Otherwise, there may be [substitutes] - that produce the outputs (e.g., by downloading them). Finally, the - outputs can be produced by running the build task described - by the derivation. +Realisation of a store path works as follows: - - If the store path is not a derivation, realisation ensures that the - specified path is valid (i.e., it and its closure exist in the file - system). If the path is already valid, we are done immediately. - Otherwise, the path and any missing paths in its closure may be - produced through substitutes. If there are no (successful) - substitutes, realisation fails. +- If the path is already valid, do nothing. +- If the path leads to a [store derivation]: + 1. Realise the store derivation file itself, as a regular store path. + 2. Realise its [output paths]: + - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure]. + - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying build certificates in the [Nix database]. + - For any store paths that cannot be substituted, produce the required store objects by first realising all outputs of the derivation's dependencies and then running the derivation's build instructions. +- Otherwise: Try to fetch the associated [store objects] in the paths' [closure] from the [substituters]. +If no substitutes are available and no store derivation is given, realisation fails. + +[store paths]: @docroot@/glossary.md#gloss-store-path [valid]: @docroot@/glossary.md#gloss-validity -[substitutes]: @docroot@/glossary.md#gloss-substitute +[store derivation]: @docroot@/glossary.md#gloss-store-derivation +[output paths]: @docroot@/glossary.md#gloss-output-path +[store objects]: @docroot@/glossary.md#gloss-store-object +[closure]: @docroot@/glossary.md#gloss-closure +[substituters]: @docroot@/command-ref/conf-file.md#conf-substituters +[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations +[Nix database]: @docroot@/glossary.md#gloss-nix-database -The output path of each derivation is printed on standard output. (For -non-derivations argument, the argument itself is printed.) +The resulting paths are printed on standard output. +For non-derivation arguments, the argument itself is printed. -The following flags are available: +{{#include ../status-build-failure.md}} + +# Options - `--dry-run`\ Print on standard error a description of what packages would be @@ -54,8 +58,6 @@ The following flags are available: previous build, the new output path is left in `/nix/store/name.check.` -{{#include ../status-build-failure.md}} - {{#include ./opt-common.md}} {{#include ../opt-common.md}} @@ -67,8 +69,6 @@ The following flags are available: This operation is typically used to build [store derivation]s produced by [`nix-instantiate`](@docroot@/command-ref/nix-instantiate.md): -[store derivation]: @docroot@/glossary.md#gloss-store-derivation - ```console $ nix-store --realise $(nix-instantiate ./test.nix) /nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1 From 315a11bcc9ce97cb25a41dcdfb4a859c7384f74b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 21 Jun 2023 10:54:46 +0200 Subject: [PATCH 213/909] remove superfluous word --- doc/manual/src/command-ref/nix-store/realise.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 2881a69b3..f304dde2e 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -19,7 +19,7 @@ Realisation of a store path works as follows: - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure]. - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying build certificates in the [Nix database]. - For any store paths that cannot be substituted, produce the required store objects by first realising all outputs of the derivation's dependencies and then running the derivation's build instructions. -- Otherwise: Try to fetch the associated [store objects] in the paths' [closure] from the [substituters]. +- Otherwise: Try to fetch the associated [store objects] in the paths' [closure] from [substituters]. If no substitutes are available and no store derivation is given, realisation fails. From a57e0e8c5c1be81e9352499ac31dcfce87ca44be Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 27 Jun 2023 09:16:36 +0200 Subject: [PATCH 214/909] reword introductory sentence --- doc/manual/src/command-ref/nix-store/realise.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index f304dde2e..00acee99c 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -10,7 +10,7 @@ Ensure that the given [store paths] are [valid]. -Realisation of a store path works as follows: +Each of *paths* is processed as follows: - If the path is already valid, do nothing. - If the path leads to a [store derivation]: From b7e9e2960566b8e21f4f69b6f222b945fababce7 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 27 Jun 2023 09:16:56 +0200 Subject: [PATCH 215/909] remove abstract description --- doc/manual/src/command-ref/nix-store/realise.md | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 00acee99c..72dfa618c 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -8,7 +8,6 @@ # Description -Ensure that the given [store paths] are [valid]. Each of *paths* is processed as follows: From d50f116421f7c999748887955b2d3c2e8b934b28 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 27 Jun 2023 09:17:12 +0200 Subject: [PATCH 216/909] add reference link --- doc/manual/src/command-ref/nix-store/realise.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 72dfa618c..2028631d0 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -11,7 +11,7 @@ Each of *paths* is processed as follows: -- If the path is already valid, do nothing. +- If the path is already [valid], do nothing. - If the path leads to a [store derivation]: 1. Realise the store derivation file itself, as a regular store path. 2. Realise its [output paths]: From 0cd8f36644042b41516e86e527ebfb19faebcbc9 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 13 Jul 2023 22:17:51 +0200 Subject: [PATCH 217/909] add anchor to `builder` --- doc/manual/src/command-ref/nix-store/realise.md | 2 +- doc/manual/src/glossary.md | 2 +- doc/manual/src/language/derivations.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 2028631d0..9dfe74277 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -17,7 +17,7 @@ Each of *paths* is processed as follows: 2. Realise its [output paths]: - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure]. - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying build certificates in the [Nix database]. - - For any store paths that cannot be substituted, produce the required store objects by first realising all outputs of the derivation's dependencies and then running the derivation's build instructions. + - For any store paths that cannot be substituted, produce the required store objects by first realising all outputs of the derivation's dependencies and then running the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable. - Otherwise: Try to fetch the associated [store objects] in the paths' [closure] from [substituters]. If no substitutes are available and no store derivation is given, realisation fails. diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index d950a4ca8..2fbb23311 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -33,7 +33,7 @@ Ensure a [store path] is [valid][validity]. - This means either running the `builder` executable as specified in the corresponding [derivation] or fetching a pre-built [store object] from a [substituter]. + This means either running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation] or fetching a pre-built [store object] from a [substituter]. See [`nix-build`](./command-ref/nix-build.md) and [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md). diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 043a38191..79a09122a 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -17,7 +17,7 @@ the attributes of which specify the inputs of the build. string. This is used as a symbolic name for the package by `nix-env`, and it is appended to the output paths of the derivation. - - There must be an attribute named `builder` that identifies the + - There must be an attribute named [`builder`]{#attr-builder} that identifies the program that is executed to perform the build. It can be either a derivation or a source (a local file reference, e.g., `./builder.sh`). From 6b3320ab05cefe73c21a469d077b277cff47733b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 09:27:36 +0200 Subject: [PATCH 218/909] mention remote builders Co-authored-by: Robert Hensing --- doc/manual/src/glossary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 2fbb23311..8f0b25c89 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -33,7 +33,7 @@ Ensure a [store path] is [valid][validity]. - This means either running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation] or fetching a pre-built [store object] from a [substituter]. + This means either running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation], or fetching a pre-built [store object] from a [substituter], or delegating to a [remote builder](@docroot@/advanced-topics/distributed-builds.html) and retrieving the outputs. See [`nix-build`](./command-ref/nix-build.md) and [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md). From d460dbdd30e3b518de962314bcf183961caf7f53 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 09:31:03 +0200 Subject: [PATCH 219/909] be more precise about substituting store derivations Co-authored-by: Robert Hensing --- doc/manual/src/command-ref/nix-store/realise.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 9dfe74277..750cbeaff 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -11,9 +11,8 @@ Each of *paths* is processed as follows: -- If the path is already [valid], do nothing. - If the path leads to a [store derivation]: - 1. Realise the store derivation file itself, as a regular store path. + 1. If it is not [valid], substitute the store derivation file itself. 2. Realise its [output paths]: - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure]. - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying build certificates in the [Nix database]. From cf4e14d58da04465c2afd05af6975d568f58ce1d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 13:28:44 +0200 Subject: [PATCH 220/909] accommodate "do nothing" branch --- doc/manual/src/command-ref/nix-store/realise.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 750cbeaff..6dc908c19 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -17,7 +17,7 @@ Each of *paths* is processed as follows: - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure]. - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying build certificates in the [Nix database]. - For any store paths that cannot be substituted, produce the required store objects by first realising all outputs of the derivation's dependencies and then running the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable. -- Otherwise: Try to fetch the associated [store objects] in the paths' [closure] from [substituters]. +- Otherwise, and if the path is not already valid: Try to fetch the associated [store objects] in the path's [closure] from [substituters]. If no substitutes are available and no store derivation is given, realisation fails. From b951e862d0bdb93a64e2c85af8762a9294d575c9 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 13:30:40 +0200 Subject: [PATCH 221/909] more meaningful tagline --- doc/manual/src/command-ref/nix-store/realise.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 6dc908c19..680e54ffa 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -1,6 +1,6 @@ # Name -`nix-store --realise` - realise specified store paths +`nix-store --realise` - build or fetch store objects # Synopsis From 894cbe43bc5caa339dab7f8af31bf065020c9f8d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 31 Aug 2023 20:50:22 +0200 Subject: [PATCH 222/909] don't invent terms yet --- doc/manual/src/command-ref/nix-store/realise.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 680e54ffa..b97c1f735 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -15,7 +15,7 @@ Each of *paths* is processed as follows: 1. If it is not [valid], substitute the store derivation file itself. 2. Realise its [output paths]: - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure]. - - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying build certificates in the [Nix database]. + - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying content-addressed realisation entries in the [Nix database]. - For any store paths that cannot be substituted, produce the required store objects by first realising all outputs of the derivation's dependencies and then running the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable. - Otherwise, and if the path is not already valid: Try to fetch the associated [store objects] in the path's [closure] from [substituters]. From d38a539437943f4ae6b1f72321c7dc29559547c5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 31 Aug 2023 20:50:50 +0200 Subject: [PATCH 223/909] make description open-ended, add TODO Co-authored-by: Robert Hensing --- doc/manual/src/command-ref/nix-store/realise.md | 2 +- doc/manual/src/glossary.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index b97c1f735..a421a4220 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -16,7 +16,7 @@ Each of *paths* is processed as follows: 2. Realise its [output paths]: - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure]. - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying content-addressed realisation entries in the [Nix database]. - - For any store paths that cannot be substituted, produce the required store objects by first realising all outputs of the derivation's dependencies and then running the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable. + - For any store paths that cannot be substituted, produce the required store objects. This involves first realising all outputs of the derivation's dependencies and then running the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable. - Otherwise, and if the path is not already valid: Try to fetch the associated [store objects] in the path's [closure] from [substituters]. If no substitutes are available and no store derivation is given, realisation fails. diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 8f0b25c89..57a3334dd 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -33,7 +33,7 @@ Ensure a [store path] is [valid][validity]. - This means either running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation], or fetching a pre-built [store object] from a [substituter], or delegating to a [remote builder](@docroot@/advanced-topics/distributed-builds.html) and retrieving the outputs. + This means either running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation], or fetching a pre-built [store object] from a [substituter], or delegating to a [remote builder](@docroot@/advanced-topics/distributed-builds.html) and retrieving the outputs. See [`nix-build`](./command-ref/nix-build.md) and [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md). From 1ac181759d975e4faeaf9083259287562009b4ec Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 31 Aug 2023 21:25:33 +0200 Subject: [PATCH 224/909] revert some random change --- doc/manual/src/command-ref/nix-store/opt-common.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/command-ref/nix-store/opt-common.md b/doc/manual/src/command-ref/nix-store/opt-common.md index 104b20076..dd9a6bf21 100644 --- a/doc/manual/src/command-ref/nix-store/opt-common.md +++ b/doc/manual/src/command-ref/nix-store/opt-common.md @@ -1,6 +1,6 @@ -# Command options +# Options -The following options are allowed for all operations in the `nix-store` command, but may not always have an effect. +The following options are allowed for all `nix-store` operations, but may not always have an effect. - [`--add-root`](#opt-add-root) *path* From 539cc5e5f00c0d524dec6e73b08ab8cb0f5a9630 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20=C4=8Cun=C3=A1t?= Date: Thu, 22 Jun 2023 11:08:56 +0200 Subject: [PATCH 225/909] flake: update nixpkgs: 22.11 -> 23.05 The lowdown input can't be updated; `nix build` would fail to find it. Co-authored-by: Robert Hensing --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 1d2aab5ed..2bc503258 100644 --- a/flake.lock +++ b/flake.lock @@ -34,16 +34,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1670461440, - "narHash": "sha256-jy1LB8HOMKGJEGXgzFRLDU1CBGL0/LlkolgnqIsF0D8=", + "lastModified": 1693494129, + "narHash": "sha256-YrHlSbniFmhcz0ORe8MMFttifKR4hTRzyX2OQUO9VxA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "04a75b2eecc0acf6239acf9dd04485ff8d14f425", + "rev": "22a584b861ab31a2dca52121999079832f0e0f73", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-22.11-small", + "ref": "nixos-23.05-small", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 36ba61484..49aa9c4ce 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.11-small"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; From 5c95b32c461da645826ff6bf248183f066c57b20 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 1 Sep 2023 14:49:49 +0200 Subject: [PATCH 226/909] Fix warning 'catching polymorphic type by value' --- src/libstore/build/create-derivation-and-realise-goal.cc | 2 +- src/libutil/util.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/create-derivation-and-realise-goal.cc b/src/libstore/build/create-derivation-and-realise-goal.cc index b01042f00..60f67956d 100644 --- a/src/libstore/build/create-derivation-and-realise-goal.cc +++ b/src/libstore/build/create-derivation-and-realise-goal.cc @@ -96,7 +96,7 @@ void CreateDerivationAndRealiseGoal::getDerivation() auto drvPath = StorePath::dummy; try { drvPath = resolveDerivedPath(worker.store, *drvReq); - } catch (MissingRealisation) { + } catch (MissingRealisation &) { return std::nullopt; } return worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index f24a6e165..5a10c69e2 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -60,7 +60,7 @@ void initLibUtil() { bool caught = false; try { throwExceptionSelfCheck(); - } catch (nix::Error _e) { + } catch (const nix::Error & _e) { caught = true; } // This is not actually the main point of this check, but let's make sure anyway: From 3ae1489847e3bde531833571d28a6e0915b4b766 Mon Sep 17 00:00:00 2001 From: Walter Franzini <5097668+wfranzini@users.noreply.github.com> Date: Tue, 31 Jan 2023 23:33:05 +0100 Subject: [PATCH 227/909] nix flakes metadata: Show lastModified timestamp for each input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before this commit, the output of `nix flake metadata` in nix repo looked like this: ... Last modified: 2023-07-09 16:00:16 Inputs: ├───flake-compat: github:edolstra/flake-compat/35bb57c0c8d8b62bbfd284272c928ceb64ddbde9 ├───lowdown-src: github:kristapsdz/lowdown/d2c2b44ff6c27b936ec27358a2653caaef8f73b8 ├───nixpkgs: github:NixOS/nixpkgs/04a75b2eecc0acf6239acf9dd04485ff8d14f425 └───nixpkgs-regression: github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2 This commit changes that to: ... Last modified: 2023-07-09 16:00:16 Inputs: ├───flake-compat: github:edolstra/flake-compat/35bb57c0c8d8b62bbfd284272c928ceb64ddbde9 (2023-01-17 11:47:33) ├───lowdown-src: github:kristapsdz/lowdown/d2c2b44ff6c27b936ec27358a2653caaef8f73b8 (2021-10-06 10:00:07) ├───nixpkgs: github:NixOS/nixpkgs/04a75b2eecc0acf6239acf9dd04485ff8d14f425 (2022-12-08 01:04:00) └───nixpkgs-regression: github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2 (2022-01-24 19:20:45) --- src/nix/flake.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 87dd4da1b..b62c45d7e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -233,9 +233,13 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON bool last = i + 1 == node.inputs.size(); if (auto lockedNode = std::get_if<0>(&input.second)) { - logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", + std::string lastModifiedStr = ""; + if (auto lastModified = (*lockedNode)->lockedRef.input.getLastModified()) + lastModifiedStr = fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%F %T")); + logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s%s", prefix + (last ? treeLast : treeConn), input.first, - (*lockedNode)->lockedRef); + (*lockedNode)->lockedRef, + lastModifiedStr); bool firstVisit = visited.insert(*lockedNode).second; From 4c50f5d130fc3adc3b048923e267d936df5b6250 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 3 Sep 2023 13:44:32 +0200 Subject: [PATCH 228/909] traces: Do not print unknown location Solves 1/3 of the infinite recursion at unknown location meme. See #8879 for ensuring we always have a trace (for stack overflows) We might want to re-add this for finding missing location info *while hacking on that problem only*. --- src/libutil/error.cc | 12 ++++-------- src/libutil/error.hh | 7 +++++++ tests/lang/eval-fail-fromTOML-timestamps.err.exp | 2 -- tests/lang/eval-fail-hashfile-missing.err.exp | 4 ---- tests/lang/eval-fail-set-override.err.exp | 2 -- tests/lang/eval-fail-substring.err.exp | 2 -- tests/lang/eval-fail-to-path.err.exp | 2 -- 7 files changed, 11 insertions(+), 20 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 4a1b346ef..3917fffc3 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -203,8 +203,6 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s std::ostringstream oss; - auto noSource = ANSI_ITALIC " (source not available)" ANSI_NORMAL "\n"; - /* * Traces * ------ @@ -320,7 +318,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s oss << "\n" << "… " << trace.hint.str() << "\n"; - if (trace.pos) { + if (trace.pos && *trace.pos) { count++; oss << "\n" << ellipsisIndent << ANSI_BLUE << "at " ANSI_WARNING << *trace.pos << ANSI_NORMAL << ":"; @@ -329,8 +327,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s oss << "\n"; printCodeLines(oss, "", *trace.pos, *loc); oss << "\n"; - } else - oss << noSource; + } } } oss << "\n" << prefix; @@ -338,15 +335,14 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s oss << einfo.msg << "\n"; - if (einfo.errPos) { + if (einfo.errPos && *einfo.errPos) { oss << "\n" << ANSI_BLUE << "at " ANSI_WARNING << *einfo.errPos << ANSI_NORMAL << ":"; if (auto loc = einfo.errPos->getCodeLines()) { oss << "\n"; printCodeLines(oss, "", *einfo.errPos, *loc); oss << "\n"; - } else - oss << noSource; + } } auto suggestions = einfo.suggestions.trim(); diff --git a/src/libutil/error.hh b/src/libutil/error.hh index be5c5e252..c04dcbd77 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -70,6 +70,13 @@ struct AbstractPos uint32_t line = 0; uint32_t column = 0; + /** + * An AbstractPos may be a "null object", representing an unknown position. + * + * Return true if this position is known. + */ + inline operator bool() const { return line != 0; }; + /** * Return the contents of the source file. */ diff --git a/tests/lang/eval-fail-fromTOML-timestamps.err.exp b/tests/lang/eval-fail-fromTOML-timestamps.err.exp index f6bd19f5a..5b60d253d 100644 --- a/tests/lang/eval-fail-fromTOML-timestamps.err.exp +++ b/tests/lang/eval-fail-fromTOML-timestamps.err.exp @@ -8,5 +8,3 @@ error: 2| key = "value" error: while parsing a TOML string: Dates and times are not supported - - at «none»:0: (source not available) diff --git a/tests/lang/eval-fail-hashfile-missing.err.exp b/tests/lang/eval-fail-hashfile-missing.err.exp index 8e77dec1e..6d38608c0 100644 --- a/tests/lang/eval-fail-hashfile-missing.err.exp +++ b/tests/lang/eval-fail-hashfile-missing.err.exp @@ -10,10 +10,6 @@ error: … while evaluating the first argument passed to builtins.toString - at «none»:0: (source not available) - … while calling the 'hashFile' builtin - at «none»:0: (source not available) - error: opening file '/pwd/lang/this-file-is-definitely-not-there-7392097': No such file or directory diff --git a/tests/lang/eval-fail-set-override.err.exp b/tests/lang/eval-fail-set-override.err.exp index beb29d678..71481683d 100644 --- a/tests/lang/eval-fail-set-override.err.exp +++ b/tests/lang/eval-fail-set-override.err.exp @@ -1,6 +1,4 @@ error: … while evaluating the `__overrides` attribute - at «none»:0: (source not available) - error: value is an integer while a set was expected diff --git a/tests/lang/eval-fail-substring.err.exp b/tests/lang/eval-fail-substring.err.exp index dc26a00bd..5c58be29a 100644 --- a/tests/lang/eval-fail-substring.err.exp +++ b/tests/lang/eval-fail-substring.err.exp @@ -8,5 +8,3 @@ error: 2| error: negative start position in 'substring' - - at «none»:0: (source not available) diff --git a/tests/lang/eval-fail-to-path.err.exp b/tests/lang/eval-fail-to-path.err.exp index 43ed2bdfc..4ffa2cf6d 100644 --- a/tests/lang/eval-fail-to-path.err.exp +++ b/tests/lang/eval-fail-to-path.err.exp @@ -9,6 +9,4 @@ error: … while evaluating the first argument passed to builtins.toPath - at «none»:0: (source not available) - error: string 'foo/bar' doesn't represent an absolute path From f1aeeea32b113ac5ae9518221015a945625fa112 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 3 Sep 2023 14:00:55 +0200 Subject: [PATCH 229/909] traces: DRY printPosMaybe --- src/libutil/error.cc | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 3917fffc3..5b604c328 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -155,6 +155,25 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR return res; } +/** + * Print a position, if it is known. + * + * @return true if a position was printed. + */ +static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std::shared_ptr & pos) { + bool hasPos = pos && *pos; + if (hasPos) { + oss << "\n" << indent << ANSI_BLUE << "at " ANSI_WARNING << *pos << ANSI_NORMAL << ":"; + + if (auto loc = pos->getCodeLines()) { + oss << "\n"; + printCodeLines(oss, "", *pos, *loc); + oss << "\n"; + } + } + return hasPos; +} + std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace) { std::string prefix; @@ -318,32 +337,15 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s oss << "\n" << "… " << trace.hint.str() << "\n"; - if (trace.pos && *trace.pos) { + if (printPosMaybe(oss, ellipsisIndent, trace.pos)) count++; - - oss << "\n" << ellipsisIndent << ANSI_BLUE << "at " ANSI_WARNING << *trace.pos << ANSI_NORMAL << ":"; - - if (auto loc = trace.pos->getCodeLines()) { - oss << "\n"; - printCodeLines(oss, "", *trace.pos, *loc); - oss << "\n"; - } - } } oss << "\n" << prefix; } oss << einfo.msg << "\n"; - if (einfo.errPos && *einfo.errPos) { - oss << "\n" << ANSI_BLUE << "at " ANSI_WARNING << *einfo.errPos << ANSI_NORMAL << ":"; - - if (auto loc = einfo.errPos->getCodeLines()) { - oss << "\n"; - printCodeLines(oss, "", *einfo.errPos, *loc); - oss << "\n"; - } - } + printPosMaybe(oss, "", einfo.errPos); auto suggestions = einfo.suggestions.trim(); if (!suggestions.suggestions.empty()) { From 477bc617bba5617d1d1da3ce039b5bd296a3e56e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 3 Sep 2023 14:08:03 +0200 Subject: [PATCH 230/909] traces: Add _NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS --- src/libutil/error.cc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 5b604c328..dd9612471 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -155,6 +155,15 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR return res; } +/** + * A development aid for finding missing positions, to improve error messages. Example use: + * + * NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test + * git diff -U20 tests + * + */ +static bool printUnknownLocations = getEnv("_NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS").has_value(); + /** * Print a position, if it is known. * @@ -170,6 +179,8 @@ static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std printCodeLines(oss, "", *pos, *loc); oss << "\n"; } + } else if (printUnknownLocations) { + oss << "\n" << indent << ANSI_BLUE << "at " ANSI_RED << "UNKNOWN LOCATION" << ANSI_NORMAL << "\n"; } return hasPos; } From 73f6407eeaf0018519be5d4f9dcd87537450d277 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 22:52:37 +0000 Subject: [PATCH 231/909] Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- .github/workflows/ci.yml | 8 ++++---- .github/workflows/hydra_status.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 62bef6322..12c60c649 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -14,7 +14,7 @@ jobs: if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name)) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} # required to find all branches diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3a17d106..bcda8a77a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 60 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: cachix/install-nix-action@v22 @@ -58,7 +58,7 @@ jobs: outputs: installerURL: ${{ steps.prepare-installer.outputs.installerURL }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV @@ -82,7 +82,7 @@ jobs: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/install-nix-action@v22 with: @@ -108,7 +108,7 @@ jobs: needs.check_secrets.outputs.docker == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: cachix/install-nix-action@v22 diff --git a/.github/workflows/hydra_status.yml b/.github/workflows/hydra_status.yml index 38a9c0877..2a7574747 100644 --- a/.github/workflows/hydra_status.yml +++ b/.github/workflows/hydra_status.yml @@ -13,7 +13,7 @@ jobs: if: github.repository_owner == 'NixOS' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - run: bash scripts/check-hydra-status.sh From 87508b1065c38d77870f7894a64f4edac162f3bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 22:52:40 +0000 Subject: [PATCH 232/909] Bump cachix/install-nix-action from 22 to 23 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 22 to 23. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v22...v23) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3a17d106..e024a851e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v22 + - uses: cachix/install-nix-action@v23 with: # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: "sandbox = true" @@ -62,7 +62,7 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v22 + - uses: cachix/install-nix-action@v23 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - uses: cachix/cachix-action@v12 @@ -84,7 +84,7 @@ jobs: steps: - uses: actions/checkout@v3 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v22 + - uses: cachix/install-nix-action@v23 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -111,7 +111,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v22 + - uses: cachix/install-nix-action@v23 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV From cc388fbc3a084a1e400c7c65df08b8dc6281056d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 30 May 2023 23:00:16 +0200 Subject: [PATCH 233/909] remove maintainers checklist in PR template maintainers are not really using it, and it produces a lot of noise when opening PRs. --- .github/PULL_REQUEST_TEMPLATE.md | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4488c7b7d..217b19108 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -10,24 +10,6 @@ -# Checklist for maintainers - - - -Maintainers: tick if completed or explain if not relevant - - - [ ] agreed on idea - - [ ] agreed on implementation strategy - - [ ] tests, as appropriate - - functional tests - `tests/**.sh` - - unit tests - `src/*/tests` - - integration tests - `tests/nixos/*` - - [ ] documentation in the manual - - [ ] documentation in the internal API docs - - [ ] code and comments are self-explanatory - - [ ] commit message explains why the change was made - - [ ] new feature or incompatible change: updated release notes - # Priorities Add :+1: to [pull requests you find important](https://github.com/NixOS/nix/pulls?q=is%3Aopen+sort%3Areactions-%2B1-desc). From 4f2b949ba808e2161f652cd914cba2517f1faea5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 13 Jul 2023 21:57:32 +0200 Subject: [PATCH 234/909] reorder list items --- CONTRIBUTING.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4a72a8eac..6c9d97c04 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,8 +30,8 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). 2. Search for related issues that cover what you're going to work on. It could help to mention there that you will work on the issue. - Issues labeled ["good first issue"](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly. - Pull requests addressing issues labeled ["idea approved"](https://github.com/NixOS/nix/labels/idea%20approved) are especially welcomed by maintainers and will receive prioritised review. + Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly. + Pull requests addressing issues labeled [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) are especially welcomed by maintainers and will receive prioritised review. 3. Check the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html) for information on building Nix and running its tests. @@ -40,10 +40,10 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). 4. Make your changes! 5. [Create a pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request) for your changes. - * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes. - * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request). * Link related issues in your pull request to inform interested parties and future contributors about your change. + * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request). If your pull request closes one or multiple issues, note that in the description using `Closes: #`, as it will then happen automatically when your change is merged. + * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes. 6. Do not expect your pull request to be reviewed immediately. Nix maintainers follow a [structured process for reviews and design decisions](https://github.com/NixOS/nix/tree/master/maintainers#project-board-protocol), which may or may not prioritise your work. From 3a9c1dc8a3b872adaf30562fb28697b58be4be2b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 13 Jul 2023 21:57:41 +0200 Subject: [PATCH 235/909] add checklist to contribution guide --- CONTRIBUTING.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6c9d97c04..facbf0eb0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,6 +48,19 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). 6. Do not expect your pull request to be reviewed immediately. Nix maintainers follow a [structured process for reviews and design decisions](https://github.com/NixOS/nix/tree/master/maintainers#project-board-protocol), which may or may not prioritise your work. + Following this checklist will make the process smoother for everyone: + + - [ ] Fixes an [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) issue + - [ ] Tests, as appropriate: + - Functional tests – [`tests/**.sh`](./tests) + - Unit tests – [`src/*/tests`](./src/) + - Integration tests – [`tests/nixos/*`](./tests/nixos) + - [ ] User documentation in the [manual](..doc/manual/src) + - [ ] API documentation in header files + - [ ] Code and comments are self-explanatory + - [ ] Commit message explains **why** the change was made + - [ ] New feature or incompatible change: updated [release notes](./doc/manual/src/release-notes/rl-next.md) + 7. If you need additional feedback or help to getting pull request into shape, ask other contributors using [@mentions](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#mentioning-people-and-teams). ## Making changes to the Nix manual From 7ff43435f9b23c9e0d445427cc37b147c84b056f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 4 Sep 2023 18:15:32 -0400 Subject: [PATCH 236/909] Unit test some worker protocol serializers Continue with the characterization testing idioms begun in c70484454f52a3bba994ac0c155b4a6f35a5013c, but this time for unit tests. Co-authored-by: Andreas Rammhold --- Makefile | 1 + doc/manual/src/contributing/testing.md | 73 ++++++++- flake.nix | 1 + mk/programs.mk | 2 +- src/libstore/tests/worker-protocol.cc | 139 ++++++++++++++++++ src/libstore/worker-protocol-impl.hh | 16 ++ src/libstore/worker-protocol.hh | 4 + .../worker-protocol/content-address.bin | Bin 0 -> 208 bytes .../libstore/worker-protocol/derived-path.bin | Bin 0 -> 120 bytes .../libstore/worker-protocol/store-path.bin | Bin 0 -> 120 bytes .../libstore/worker-protocol/string.bin | Bin 0 -> 88 bytes 11 files changed, 228 insertions(+), 8 deletions(-) create mode 100644 src/libstore/tests/worker-protocol.cc create mode 100644 unit-test-data/libstore/worker-protocol/content-address.bin create mode 100644 unit-test-data/libstore/worker-protocol/derived-path.bin create mode 100644 unit-test-data/libstore/worker-protocol/store-path.bin create mode 100644 unit-test-data/libstore/worker-protocol/string.bin diff --git a/Makefile b/Makefile index 31b54b93d..2eabeb077 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,7 @@ makefiles = \ -include Makefile.config ifeq ($(tests), yes) +UNIT_TEST_ENV = _NIX_TEST_UNIT_DATA=unit-test-data makefiles += \ src/libutil/tests/local.mk \ src/libstore/tests/local.mk \ diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index cd94d5cfb..329b34575 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -2,14 +2,70 @@ ## Unit-tests -The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined -under `src/{library_name}/tests` using the -[googletest](https://google.github.io/googletest/) and -[rapidcheck](https://github.com/emil-e/rapidcheck) frameworks. +The unit tests are defined using the [googletest] and [rapidcheck] frameworks. + +[googletest]: https://google.github.io/googletest/ +[rapidcheck]: https://github.com/emil-e/rapidcheck + +### Source and header layout + +> An example of some files, demonstrating much of what is described below +> +> ``` +> src +> ├── libexpr +> │ ├── value/context.hh +> │ ├── value/context.cc +> │ │ +> │ … +> └── tests +> │ ├── value/context.hh +> │ ├── value/context.cc +> │ │ +> │ … +> │ +> ├── unit-test-data +> │ ├── libstore +> │ │ ├── worker-protocol/content-address.bin +> │ │ … +> │ … +> … +> ``` + +The unit tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_shortname}/tests` within the directory for the library (`src/${library_shortname}`). + +The data is in `unit-test-data`, with one subdir per library, with the same name as where the code goes. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `unit-test-data/libstore`. +The path to the `unit-test-data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. + +> **Note** +> Due to the way googletest works, downstream unit test executables will actually include and re-run upstream library tests. +> Therefore it is important that the same value for `_NIX_TEST_UNIT_DATA` be used with the tests for each library. +> That is why we have the test data nested within a single `unit-test-data` directory. + +### Running tests You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable. +### Characterization testing + +See [below](#characterization-testing-1) for a broader discussion of characterization testing. + +Like with the functional characterization, `_NIX_TEST_ACCEPT=1` is also used. +For example: +```shell-session +$ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN +... +[ SKIPPED ] WorkerProtoTest.string_read +[ SKIPPED ] WorkerProtoTest.string_write +[ SKIPPED ] WorkerProtoTest.storePath_read +[ SKIPPED ] WorkerProtoTest.storePath_write +... +``` +will regenerate the "golden master" expected result for the `libnixstore` characterization tests. +The characterization tests will mark themselves "skipped" since they regenerated the expected result instead of actually testing anything. + ## Functional tests The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. @@ -124,9 +180,12 @@ This technique is to include the exact output/behavior of a former version of Ni For example, this technique is used for the language tests, to check both the printed final value if evaluation was successful, and any errors and warnings encountered. It is frequently useful to regenerate the expected output. -To do that, rerun the failed test with `_NIX_TEST_ACCEPT=1`. -(At least, this is the convention we've used for `tests/lang.sh`. -If we add more characterization testing we should always strive to be consistent.) +To do that, rerun the failed test(s) with `_NIX_TEST_ACCEPT=1`. +For example: +```bash +_NIX_TEST_ACCEPT=1 make tests/lang.sh.test +``` +This convention is shared with the [characterization unit tests](#characterization-testing-1) too. An interesting situation to document is the case when these tests are "overfitted". The language tests are, again, an example of this. diff --git a/flake.nix b/flake.nix index cf7c1fa12..ec6ab48bd 100644 --- a/flake.nix +++ b/flake.nix @@ -75,6 +75,7 @@ ./precompiled-headers.h ./src ./tests + ./unit-test-data ./COPYING ./scripts/local.mk (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) diff --git a/mk/programs.mk b/mk/programs.mk index 1ee1d3fa5..a88d9d949 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -87,6 +87,6 @@ define build-program # Phony target to run this program (typically as a dependency of 'check'). .PHONY: $(1)_RUN $(1)_RUN: $$($(1)_PATH) - $(trace-test) $$($(1)_PATH) + $(trace-test) $$(UNIT_TEST_ENV) $$($(1)_PATH) endef diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc new file mode 100644 index 000000000..4a6ccf7c0 --- /dev/null +++ b/src/libstore/tests/worker-protocol.cc @@ -0,0 +1,139 @@ +#include + +#include +#include + +#include "worker-protocol.hh" +#include "worker-protocol-impl.hh" +#include "derived-path.hh" +#include "tests/libstore.hh" + +namespace nix { + +class WorkerProtoTest : public LibStoreTest +{ +public: + Path unitTestData = getEnv("_NIX_TEST_UNIT_DATA").value() + "/libstore/worker-protocol"; + + bool testAccept() { + return getEnv("_NIX_TEST_ACCEPT") == "1"; + } + + Path goldenMaster(std::string_view testStem) { + return unitTestData + "/" + testStem + ".bin"; + } + + /** + * Golden test for `T` reading + */ + template + void readTest(PathView testStem, T value) + { + if (testAccept()) + { + GTEST_SKIP() << "Cannot read golden master because another test is also updating it"; + } + else + { + auto expected = readFile(goldenMaster(testStem)); + + T got = ({ + StringSource from { expected }; + WorkerProto::Serialise::read( + *store, + WorkerProto::ReadConn { .from = from }); + }); + + ASSERT_EQ(got, value); + } + } + + /** + * Golden test for `T` write + */ + template + void writeTest(PathView testStem, const T & value) + { + auto file = goldenMaster(testStem); + + StringSink to; + WorkerProto::write( + *store, + WorkerProto::WriteConn { .to = to }, + value); + + if (testAccept()) + { + createDirs(dirOf(file)); + writeFile(file, to.s); + GTEST_SKIP() << "Updating golden master"; + } + else + { + auto expected = readFile(file); + ASSERT_EQ(to.s, expected); + } + } +}; + +#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + TEST_F(WorkerProtoTest, NAME ## _read) { \ + readTest(STEM, VALUE); \ + } \ + TEST_F(WorkerProtoTest, NAME ## _write) { \ + writeTest(STEM, VALUE); \ + } + +CHARACTERIZATION_TEST( + string, + "string", + (std::tuple { + "", + "hi", + "white rabbit", + "大白兔", + "oh no \0\0\0 what was that!", + })) + +CHARACTERIZATION_TEST( + storePath, + "store-path", + (std::tuple { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + })) + +CHARACTERIZATION_TEST( + contentAddress, + "content-address", + (std::tuple { + ContentAddress { + .method = TextIngestionMethod {}, + .hash = hashString(HashType::htSHA256, "Derive(...)"), + }, + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + ContentAddress { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + }, + })) + +CHARACTERIZATION_TEST( + derivedPath, + "derived-path", + (std::tuple { + DerivedPath::Opaque { + .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::Names { "x", "y" }, + }, + })) + +} diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh index d3d2792ff..4f797f95a 100644 --- a/src/libstore/worker-protocol-impl.hh +++ b/src/libstore/worker-protocol-impl.hh @@ -75,4 +75,20 @@ void WorkerProto::Serialise>::write(const Store & store, WorkerPr } } +template +std::tuple WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) +{ + return std::tuple { + WorkerProto::Serialise::read(store, conn)..., + }; +} + +template +void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::tuple & res) +{ + std::apply([&](const Us &... args) { + (WorkerProto::Serialise::write(store, conn, args), ...); + }, res); +} + } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index ff762c924..70a5bddb9 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -28,6 +28,8 @@ class Store; struct Source; // items being serialised +class StorePath; +struct ContentAddress; struct DerivedPath; struct DrvOutput; struct Realisation; @@ -220,6 +222,8 @@ template MAKE_WORKER_PROTO(std::vector); template MAKE_WORKER_PROTO(std::set); +template +MAKE_WORKER_PROTO(std::tuple); template #define X_ std::map diff --git a/unit-test-data/libstore/worker-protocol/content-address.bin b/unit-test-data/libstore/worker-protocol/content-address.bin new file mode 100644 index 0000000000000000000000000000000000000000..8f14bcdb3e50cc72d87106ce159a6fd7b9f75713 GIT binary patch literal 208 zcmY+;K@P$o5QSmy;|7WYrYjRqQfMm$+LWPzDW_MfG4bucKks(>Y#V56lkFOiEzi24 zUMdC5VUCa86OJ&gL0BF^B{HBQEY%f|I<6v7#q+l_PBirI5N~-md%37WE|g33QTE2k zc~-WF&R}7Oxc@o)T?ojpD!+*5=xpO;%IoCKSV5FMh={>xePK|8<${>E)*huNHZ(pX literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/derived-path.bin b/unit-test-data/libstore/worker-protocol/derived-path.bin new file mode 100644 index 0000000000000000000000000000000000000000..bb1a81ac6d096cddf510a457e7506e6a81ee66a3 GIT binary patch literal 120 zcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7Eu*IT`wr5{vXwipmr#bSfDDBbg*z literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/store-path.bin b/unit-test-data/libstore/worker-protocol/store-path.bin new file mode 100644 index 0000000000000000000000000000000000000000..3fc05f2981d3398ab30ec34125c903cdcefa8729 GIT binary patch literal 120 tcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7Eu*F?srQlM;)-IsqP{BM|@q literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/string.bin b/unit-test-data/libstore/worker-protocol/string.bin new file mode 100644 index 0000000000000000000000000000000000000000..aa7b5a604745b735473ec34283177e69320776e4 GIT binary patch literal 88 zcmZQzfB+^aoskJ)@Id+H8JQ)i3Pp)YNtq=eAx^0H( Date: Tue, 5 Sep 2023 11:16:39 -0400 Subject: [PATCH 237/909] Test and begin documentation of the ATerm format for derivations Wanted to do this before the last dynamic derivations PR when I introduce a variation, to make sure I wasn't changing the old version by mistake. --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/protocols/derivation-aterm.md | 9 ++ src/libstore/tests/derivation.cc | 119 ++++++++++++------- 3 files changed, 83 insertions(+), 46 deletions(-) create mode 100644 doc/manual/src/protocols/derivation-aterm.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 6c599abcf..3de053a17 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -100,6 +100,7 @@ - [File System Object](architecture/file-system-object.md) - [Protocols](protocols/protocols.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) + - [Derivation on-disk "ATerm" format](protocols/derivation-aterm.md) - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md new file mode 100644 index 000000000..2377e51ae --- /dev/null +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -0,0 +1,9 @@ +# Derivation on-disk "ATerm" format + +For historical reasons, [derivations](@docroot@/glossary.md#gloss-store-derivation) are stored on-disk in [ATerm](https://homepages.cwi.nl/~daybuild/daily-books/technology/aterm-guide/aterm-guide.html) format. + +Derivations are serialised in the following format: + +``` +Derive(...) +``` diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 0e28c1f08..7fd7f9278 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -143,26 +143,76 @@ TEST_JSON(ImpureDerivationTest, impure, #undef TEST_JSON -#define TEST_JSON(NAME, STR, VAL, DRV_NAME) \ - TEST_F(DerivationTest, Derivation_ ## NAME ## _to_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - STR ## _json, \ - (Derivation { VAL }).toJSON(*store)); \ - } \ - \ - TEST_F(DerivationTest, Derivation_ ## NAME ## _from_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - Derivation { VAL }, \ - Derivation::fromJSON( \ - *store, \ - STR ## _json)); \ +#define TEST_JSON(NAME, STR, VAL) \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + (VAL).toJSON(*store)); \ + } \ + \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _from_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + (VAL), \ + Derivation::fromJSON( \ + *store, \ + STR ## _json)); \ } +#define TEST_ATERM(NAME, STR, VAL, DRV_NAME) \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _to_aterm) { \ + ASSERT_EQ( \ + STR, \ + (VAL).unparse(*store, false)); \ + } \ + \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _from_aterm) { \ + auto parsed = parseDerivation( \ + *store, \ + STR, \ + DRV_NAME); \ + ASSERT_EQ( \ + (VAL).toJSON(*store), \ + parsed.toJSON(*store)); \ + ASSERT_EQ( \ + (VAL), \ + parsed); \ + } + +Derivation makeSimpleDrv(const Store & store) { + Derivation drv; + drv.name = "simple-derivation"; + drv.inputSrcs = { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + { + "cat", + "dog", + }, + }, + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = { + { + "BIG_BAD", + "WOLF", + }, + }; + return drv; +} + TEST_JSON(simple, R"({ - "name": "my-derivation", + "name": "simple-derivation", "inputSrcs": [ "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" ], @@ -183,37 +233,14 @@ TEST_JSON(simple, }, "outputs": {} })", - ({ - Derivation drv; - drv.name = "my-derivation"; - drv.inputSrcs = { - store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), - }; - drv.inputDrvs = { - { - store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), - { - "cat", - "dog", - }, - } - }; - drv.platform = "wasm-sel4"; - drv.builder = "foo"; - drv.args = { - "bar", - "baz", - }; - drv.env = { - { - "BIG_BAD", - "WOLF", - }, - }; - drv; - }), - "drv-name") + makeSimpleDrv(*store)) + +TEST_ATERM(simple, + R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", + makeSimpleDrv(*store), + "simple-derivation") #undef TEST_JSON +#undef TEST_ATERM } From 2b3a17820f202ea2108184a399f3c573ca11c1b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christina=20S=C3=B8rensen?= Date: Wed, 6 Sep 2023 04:19:40 +0000 Subject: [PATCH 238/909] Fix globals.hh typo --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 7009f6bb8..cf10edebd 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -554,7 +554,7 @@ public: R"( This option determines the maximum size of the `tmpfs` filesystem mounted on `/dev/shm` in Linux sandboxes. For the format, see the - description of the `size` option of `tmpfs` in mount8. The default + description of the `size` option of `tmpfs` in mount(8). The default is `50%`. )"}; From 5c23d3a90cb1a76a0ea464be380bb16dc9f999c7 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 6 Sep 2023 09:09:02 +0200 Subject: [PATCH 239/909] disambiguate output from output path --- doc/manual/src/glossary.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 57a3334dd..b7c340d36 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -197,9 +197,15 @@ [closure]: #gloss-closure +- [output]{#gloss-output} + + A [store object] produced by a [derivation]. + + [output]: #gloss-output + - [output path]{#gloss-output-path} - A [store path] produced by a [derivation]. + The [store path] to the [output] of a [derivation]. [output path]: #gloss-output-path From 02c2679f0e94013b5cae6305e0b15fce8d971eb8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 6 Sep 2023 09:49:00 +0200 Subject: [PATCH 240/909] dedent common options listing; one sentence per line this is a pure reformatting, contents were not changed one sentence per line makes reviewing diffs and making suggestions much more convenient. the indentation was an artifat of the DocBook migration. --- doc/manual/src/command-ref/opt-common.md | 335 +++++++++++------------ 1 file changed, 161 insertions(+), 174 deletions(-) diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 54c0a1d0d..0647228c2 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -2,217 +2,204 @@ Most Nix commands accept the following command-line options: - - [`--help`](#opt-help)\ - Prints out a summary of the command syntax and exits. +- [`--help`](#opt-help) - - [`--version`](#opt-version)\ - Prints out the Nix version number on standard output and exits. + Prints out a summary of the command syntax and exits. - - [`--verbose`](#opt-verbose) / `-v`\ - Increases the level of verbosity of diagnostic messages printed on - standard error. For each Nix operation, the information printed on - standard output is well-defined; any diagnostic information is - printed on standard error, never on standard output. +- [`--version`](#opt-version) - This option may be specified repeatedly. Currently, the following - verbosity levels exist: + Prints out the Nix version number on standard output and exits. - - 0\ - “Errors only”: only print messages explaining why the Nix - invocation failed. +- [`--verbose`](#opt-verbose) / `-v` - - 1\ - “Informational”: print *useful* messages about what Nix is - doing. This is the default. + Increases the level of verbosity of diagnostic messages printed on standard error. + For each Nix operation, the information printed on standard output is well-defined; + any diagnostic information is printed on standard error, never on standard output. - - 2\ - “Talkative”: print more informational messages. + This option may be specified repeatedly. + Currently, the following verbosity levels exist: - - 3\ - “Chatty”: print even more informational messages. + - `0` “Errors only” - - 4\ - “Debug”: print debug information. + Only print messages explaining why the Nix invocation failed. - - 5\ - “Vomit”: print vast amounts of debug information. + - `1` “Informational” - - [`--quiet`](#opt-quiet)\ - Decreases the level of verbosity of diagnostic messages printed on - standard error. This is the inverse option to `-v` / `--verbose`. + Print *useful* messages about what Nix is doing. + This is the default. - This option may be specified repeatedly. See the previous verbosity - levels list. + - `2` “Talkative” - - [`--log-format`](#opt-log-format) *format*\ - This option can be used to change the output of the log format, with - *format* being one of: + Print more informational messages. - - raw\ - This is the raw format, as outputted by nix-build. + - `3` “Chatty” - - internal-json\ - Outputs the logs in a structured manner. + Print even more informational messages. - > **Warning** - > - > While the schema itself is relatively stable, the format of - > the error-messages (namely of the `msg`-field) can change - > between releases. + - `4` “Debug” + + Print debug information. - - bar\ - Only display a progress bar during the builds. + - `5` “Vomit” - - bar-with-logs\ - Display the raw logs, with the progress bar at the bottom. + Print vast amounts of debug information. - - [`--no-build-output`](#opt-no-build-output) / `-Q`\ - By default, output written by builders to standard output and - standard error is echoed to the Nix command's standard error. This - option suppresses this behaviour. Note that the builder's standard - output and error are always written to a log file in - `prefix/nix/var/log/nix`. +- [`--quiet`](#opt-quiet) - - [`--max-jobs`](#opt-max-jobs) / `-j` *number*\ - Sets the maximum number of build jobs that Nix will perform in - parallel to the specified number. Specify `auto` to use the number - of CPUs in the system. The default is specified by the `max-jobs` - configuration setting, which itself defaults to `1`. A higher - value is useful on SMP systems or to exploit I/O latency. + Decreases the level of verbosity of diagnostic messages printed on standard error. + This is the inverse option to `-v` / `--verbose`. - Setting it to `0` disallows building on the local machine, which is - useful when you want builds to happen only on remote builders. + This option may be specified repeatedly. + See the previous verbosity levels list. - - [`--cores`](#opt-cores)\ - Sets the value of the `NIX_BUILD_CORES` environment variable in - the invocation of builders. Builders can use this variable at - their discretion to control the maximum amount of parallelism. For - instance, in Nixpkgs, if the derivation attribute - `enableParallelBuilding` is set to `true`, the builder passes the - `-jN` flag to GNU Make. It defaults to the value of the `cores` - configuration setting, if set, or `1` otherwise. The value `0` - means that the builder should use all available CPU cores in the - system. +- [`--log-format`](#opt-log-format) *format* - - [`--max-silent-time`](#opt-max-silent-time)\ - Sets the maximum number of seconds that a builder can go without - producing any data on standard output or standard error. The - default is specified by the `max-silent-time` configuration - setting. `0` means no time-out. + This option can be used to change the output of the log format, with *format* being one of: - - [`--timeout`](#opt-timeout)\ - Sets the maximum number of seconds that a builder can run. The - default is specified by the `timeout` configuration setting. `0` - means no timeout. + - `raw` - - [`--keep-going`](#opt-keep-going) / `-k`\ - Keep going in case of failed builds, to the greatest extent - possible. That is, if building an input of some derivation fails, - Nix will still build the other inputs, but not the derivation - itself. Without this option, Nix stops if any build fails (except - for builds of substitutes), possibly killing builds in progress (in - case of parallel or distributed builds). + This is the raw format, as outputted by nix-build. - - [`--keep-failed`](#opt-keep-failed) / `-K`\ - Specifies that in case of a build failure, the temporary directory - (usually in `/tmp`) in which the build takes place should not be - deleted. The path of the build directory is printed as an - informational message. + - `internal-json` - - [`--fallback`](#opt-fallback)\ - Whenever Nix attempts to build a derivation for which substitutes - are known for each output path, but realising the output paths - through the substitutes fails, fall back on building the derivation. + Outputs the logs in a structured manner. - The most common scenario in which this is useful is when we have - registered substitutes in order to perform binary distribution from, - say, a network repository. If the repository is down, the - realisation of the derivation will fail. When this option is - specified, Nix will build the derivation instead. Thus, installation - from binaries falls back on installation from source. This option is - not the default since it is generally not desirable for a transient - failure in obtaining the substitutes to lead to a full build from - source (with the related consumption of resources). + > **Warning** + > + > While the schema itself is relatively stable, the format of + > the error-messages (namely of the `msg`-field) can change + > between releases. - - [`--readonly-mode`](#opt-readonly-mode)\ - When this option is used, no attempt is made to open the Nix - database. Most Nix operations do need database access, so those - operations will fail. + - `bar` - - [`--arg`](#opt-arg) *name* *value*\ - This option is accepted by `nix-env`, `nix-instantiate`, - `nix-shell` and `nix-build`. When evaluating Nix expressions, the - expression evaluator will automatically try to call functions that - it encounters. It can automatically call functions for which every - argument has a [default - value](@docroot@/language/constructs.md#functions) (e.g., - `{ argName ? defaultValue }: ...`). With `--arg`, you can also - call functions that have arguments without a default value (or - override a default value). That is, if the evaluator encounters a - function with an argument named *name*, it will call it with value - *value*. + Only display a progress bar during the builds. - For instance, the top-level `default.nix` in Nixpkgs is actually a - function: + - `bar-with-logs` - ```nix - { # The system (e.g., `i686-linux') for which to build the packages. - system ? builtins.currentSystem - ... - }: ... - ``` + Display the raw logs, with the progress bar at the bottom. - So if you call this Nix expression (e.g., when you do `nix-env --install --attr - pkgname`), the function will be called automatically using the - value [`builtins.currentSystem`](@docroot@/language/builtins.md) for - the `system` argument. You can override this using `--arg`, e.g., - `nix-env --install --attr pkgname --arg system \"i686-freebsd\"`. (Note that - since the argument is a Nix string literal, you have to escape the - quotes.) +- [`--no-build-output`](#opt-no-build-output) / `-Q` - - [`--argstr`](#opt-argstr) *name* *value*\ - This option is like `--arg`, only the value is not a Nix - expression but a string. So instead of `--arg system - \"i686-linux\"` (the outer quotes are to keep the shell happy) you - can say `--argstr system i686-linux`. + By default, output written by builders to standard output and standard error is echoed to the Nix command's standard error. + This option suppresses this behaviour. + Note that the builder's standard output and error are always written to a log file in `prefix/nix/var/log/nix`. - - [`--attr`](#opt-attr) / `-A` *attrPath*\ - Select an attribute from the top-level Nix expression being - evaluated. (`nix-env`, `nix-instantiate`, `nix-build` and - `nix-shell` only.) The *attribute path* *attrPath* is a sequence - of attribute names separated by dots. For instance, given a - top-level Nix expression *e*, the attribute path `xorg.xorgserver` - would cause the expression `e.xorg.xorgserver` to be used. See - [`nix-env --install`](@docroot@/command-ref/nix-env/install.md) for some - concrete examples. +- [`--max-jobs`](#opt-max-jobs) / `-j` *number* - In addition to attribute names, you can also specify array indices. - For instance, the attribute path `foo.3.bar` selects the `bar` - attribute of the fourth element of the array in the `foo` attribute - of the top-level expression. + Sets the maximum number of build jobs that Nix will perform in parallel to the specified number. + Specify `auto` to use the number of CPUs in the system. + The default is specified by the `max-jobs` configuration setting, which itself defaults to `1`. + A higher value is useful on SMP systems or to exploit I/O latency. - - [`--expr`](#opt-expr) / `-E`\ - Interpret the command line arguments as a list of Nix expressions to - be parsed and evaluated, rather than as a list of file names of Nix - expressions. (`nix-instantiate`, `nix-build` and `nix-shell` only.) + Setting it to `0` disallows building on the local machine, which is useful when you want builds to happen only on remote builders. - For `nix-shell`, this option is commonly used to give you a shell in - which you can build the packages returned by the expression. If you - want to get a shell which contain the *built* packages ready for - use, give your expression to the `nix-shell --packages ` convenience flag - instead. +- [`--cores`](#opt-cores) - - [`-I`](#opt-I) *path*\ - Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). - This option may be given multiple times. - Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). + Sets the value of the `NIX_BUILD_CORES` environment variable in the invocation of builders. + Builders can use this variable at their discretion to control the maximum amount of parallelism. + For instance, in Nixpkgs, if the derivation attribute `enableParallelBuilding` is set to `true`, the builder passes the `-jN` flag to GNU Make. + It defaults to the value of the `cores` configuration setting, if set, or `1` otherwise. + The value `0` means that the builder should use all available CPU cores in the system. - - [`--option`](#opt-option) *name* *value*\ - Set the Nix configuration option *name* to *value*. This overrides - settings in the Nix configuration file (see nix.conf5). +- [`--max-silent-time`](#opt-max-silent-time) - - [`--repair`](#opt-repair)\ - Fix corrupted or missing store paths by redownloading or rebuilding - them. Note that this is slow because it requires computing a - cryptographic hash of the contents of every path in the closure of - the build. Also note the warning under `nix-store --repair-path`. + Sets the maximum number of seconds that a builder can go without producing any data on standard output or standard error. + The default is specified by the `max-silent-time` configuration setting. + `0` means no time-out. + +- [`--timeout`](#opt-timeout) + + Sets the maximum number of seconds that a builder can run. + The default is specified by the `timeout` configuration setting. + `0` means no timeout. + +- [`--keep-going`](#opt-keep-going) / `-k` + + Keep going in case of failed builds, to the greatest extent possible. + That is, if building an input of some derivation fails, Nix will still build the other inputs, but not the derivation itself. + Without this option, Nix stops if any build fails (except for builds of substitutes), possibly killing builds in progress (in case of parallel or distributed builds). + +- [`--keep-failed`](#opt-keep-failed) / `-K` + + Specifies that in case of a build failure, the temporary directory (usually in `/tmp`) in which the build takes place should not be deleted. + The path of the build directory is printed as an informational message. + +- [`--fallback`](#opt-fallback) + + Whenever Nix attempts to build a derivation for which substitutes are known for each output path, but realising the output paths through the substitutes fails, fall back on building the derivation. + + The most common scenario in which this is useful is when we have registered substitutes in order to perform binary distribution from, say, a network repository. + If the repository is down, the realisation of the derivation will fail. + When this option is specified, Nix will build the derivation instead. + Thus, installation from binaries falls back on installation from source. + This option is not the default since it is generally not desirable for a transient failure in obtaining the substitutes to lead to a full build from source (with the related consumption of resources). + +- [`--readonly-mode`](#opt-readonly-mode) + + When this option is used, no attempt is made to open the Nix database. + Most Nix operations do need database access, so those operations will fail. + +- [`--arg`](#opt-arg) *name* *value* + + This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell` and `nix-build`. + When evaluating Nix expressions, the expression evaluator will automatically try to call functions that it encounters. + It can automatically call functions for which every argument has a [default value](@docroot@/language/constructs.md#functions) (e.g., `{ argName ? defaultValue }: ...`). + + With `--arg`, you can also call functions that have arguments without a default value (or override a default value). + That is, if the evaluator encounters a function with an argument named *name*, it will call it with value *value*. + + For instance, the top-level `default.nix` in Nixpkgs is actually a function: + + ```nix + { # The system (e.g., `i686-linux') for which to build the packages. + system ? builtins.currentSystem + ... + }: ... + ``` + + So if you call this Nix expression (e.g., when you do `nix-env --install --attr pkgname`), the function will be called automatically using the value [`builtins.currentSystem`](@docroot@/language/builtins.md) for the `system` argument. + You can override this using `--arg`, e.g., `nix-env --install --attr pkgname --arg system \"i686-freebsd\"`. + (Note that since the argument is a Nix string literal, you have to escape the quotes.) + +- [`--argstr`](#opt-argstr) *name* *value* + + This option is like `--arg`, only the value is not a Nix expression but a string. + So instead of `--arg system \"i686-linux\"` (the outer quotes are to keep the shell happy) you can say `--argstr system i686-linux`. + +- [`--attr`](#opt-attr) / `-A` *attrPath* + + Select an attribute from the top-level Nix expression being evaluated. + (`nix-env`, `nix-instantiate`, `nix-build` and `nix-shell` only.) + The *attribute path* *attrPath* is a sequence of attribute names separated by dots. + For instance, given a top-level Nix expression *e*, the attribute path `xorg.xorgserver` would cause the expression `e.xorg.xorgserver` to be used. + See [`nix-env --install`](@docroot@/command-ref/nix-env/install.md) for some concrete examples. + + In addition to attribute names, you can also specify array indices. + For instance, the attribute path `foo.3.bar` selects the `bar` + attribute of the fourth element of the array in the `foo` attribute + of the top-level expression. + +- [`--expr`](#opt-expr) / `-E` + + Interpret the command line arguments as a list of Nix expressions to be parsed and evaluated, rather than as a list of file names of Nix expressions. + (`nix-instantiate`, `nix-build` and `nix-shell` only.) + + For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. + If you want to get a shell which contain the *built* packages ready for use, give your expression to the `nix-shell --packages ` convenience flag instead. + +- [`-I`](#opt-I) *path* + + Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). + This option may be given multiple times. + Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). + +- [`--option`](#opt-option) *name* *value* + + Set the Nix configuration option *name* to *value*. + This overrides settings in the Nix configuration file (see nix.conf5). + +- [`--repair`](#opt-repair) + + Fix corrupted or missing store paths by redownloading or rebuilding them. + Note that this is slow because it requires computing a cryptographic hash of the contents of every path in the closure of the build. + Also note the warning under `nix-store --repair-path`. From 391f18063c0279e2de189176335a1b2863b9533a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 6 Sep 2023 09:52:56 +0200 Subject: [PATCH 241/909] add anchors to option listings --- doc/manual/generate-manpage.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 65eec42d0..bc4d2c5bc 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -98,7 +98,7 @@ let (option ? labels) (concatStringsSep " " (map (s: "*${s}*") option.labels)); in trim '' - - `--${name}` ${shortName} ${labels} + - [`--${name}`](#opt-${name}) ${shortName} ${labels} ${option.description} ''; From d568877eab59a490694e0bc3edec99d13049552c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 6 Sep 2023 08:43:16 -0400 Subject: [PATCH 242/909] Retitle section as Robert suggests Co-authored-by: Robert Hensing --- doc/manual/src/SUMMARY.md.in | 2 +- doc/manual/src/protocols/derivation-aterm.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 3de053a17..701a6ea69 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -100,7 +100,7 @@ - [File System Object](architecture/file-system-object.md) - [Protocols](protocols/protocols.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - - [Derivation on-disk "ATerm" format](protocols/derivation-aterm.md) + - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md index 2377e51ae..c9dc00ef7 100644 --- a/doc/manual/src/protocols/derivation-aterm.md +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -1,4 +1,4 @@ -# Derivation on-disk "ATerm" format +# Derivation "ATerm" file format For historical reasons, [derivations](@docroot@/glossary.md#gloss-store-derivation) are stored on-disk in [ATerm](https://homepages.cwi.nl/~daybuild/daily-books/technology/aterm-guide/aterm-guide.html) format. From b7edc2099fffd7d54638122d2d9950e3d3a751f6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 6 Sep 2023 11:35:01 -0400 Subject: [PATCH 243/909] Improve derivation parsing - Don't assert: Derivation ATerms are not necessarily produced by Nix, and parsers should always throw graceful errors - Improve error message from `static void except(..)`, shows both what we expected and what we actually got. The intention is that we backport it, and then hopefully a few people might get slightly better errors if they try out new experimental drv files (for RFC 92) with an old version of Nix. --- src/libstore/derivations.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index dc32c3847..43d3dc4a2 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -154,8 +154,9 @@ static void expect(std::istream & str, std::string_view s) { char s2[s.size()]; str.read(s2, s.size()); - if (std::string(s2, s.size()) != s) - throw FormatError("expected string '%1%'", s); + std::string_view s2View { s2, s.size() }; + if (s2View != s) + throw FormatError("expected string '%s', got '%s'", s, s2View); } @@ -223,7 +224,8 @@ static DerivationOutput parseDerivationOutput(const Store & store, const auto hashType = parseHashType(hashAlgo); if (hashS == "impure") { experimentalFeatureSettings.require(Xp::ImpureDerivations); - assert(pathS == ""); + if (pathS != "") + throw FormatError("impure derivation output should not specify output path"); return DerivationOutput::Impure { .method = std::move(method), .hashType = std::move(hashType), @@ -239,7 +241,8 @@ static DerivationOutput parseDerivationOutput(const Store & store, }; } else { experimentalFeatureSettings.require(Xp::CaDerivations); - assert(pathS == ""); + if (pathS != "") + throw FormatError("content-addressed derivation output should not specify output path"); return DerivationOutput::CAFloating { .method = std::move(method), .hashType = std::move(hashType), From 589fd897fb8414561b7b328e864e998876ff822b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 6 Sep 2023 09:14:56 +0200 Subject: [PATCH 244/909] add todo on store docs --- doc/manual/generate-manpage.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index bc4d2c5bc..0f91a9742 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -70,6 +70,7 @@ let * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description} ''; + # TODO: move this confusing special case out of here when implementing #8496 maybeDocumentation = optionalString (details ? doc) (replaceStrings ["@stores@"] [storeDocs] details.doc); From b0fe7f560ddb31fea55546a79a89c44a63cd56ce Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 6 Sep 2023 09:15:03 +0200 Subject: [PATCH 245/909] add missing link --- doc/manual/generate-manpage.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 0f91a9742..705fc0b69 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -148,7 +148,7 @@ let To use this store, you need to make sure the corresponding experimental feature, [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), is enabled. - For example, include the following in [`nix.conf`](#): + For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): ``` extra-experimental-features = ${experimentalFeature} From 09eb7f1ef653822616e7371c906ef1ae34b3ba09 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 6 Sep 2023 09:30:32 +0200 Subject: [PATCH 246/909] do not show configuration override flags for each command this removes a lot of noise from the web search, which precludes finding the actual documentation. some configuration settings have enough documentation to warrant individual pages, so the alternative of including full setting documentation in each command page doesn't make much sense here. this change technically means that the command line flags to override settings are "invisible", and not exported as JSON. this may or may not be desirable. a more explicit approach would be adding a `hidden` field to the flag's JSON output, but would also require adjusting post-processing of that JSON for manual rendering. --- doc/manual/generate-manpage.nix | 4 ++++ src/libutil/args.cc | 1 + 2 files changed, 5 insertions(+) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 705fc0b69..87891fa7a 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -79,6 +79,10 @@ let # Options ${showOptions details.flags toplevel.flags} + + > **Note** + > + > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. ''; showOptions = options: commonOptions: diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 00281ce6f..8db293762 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -236,6 +236,7 @@ nlohmann::json Args::toJSON() for (auto & [name, flag] : longFlags) { auto j = nlohmann::json::object(); + if (hiddenCategories.count(flag->category)) continue; if (flag->aliases.count(name)) continue; if (flag->shortName) j["shortName"] = std::string(1, flag->shortName); From aa46f536e80ca20e0fc45664af867b47597a87f3 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 6 Sep 2023 09:40:51 +0200 Subject: [PATCH 247/909] add note on overriding settings for stable commands --- doc/manual/src/command-ref/opt-common.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 0647228c2..114b292f9 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -203,3 +203,7 @@ Most Nix commands accept the following command-line options: Fix corrupted or missing store paths by redownloading or rebuilding them. Note that this is slow because it requires computing a cryptographic hash of the contents of every path in the closure of the build. Also note the warning under `nix-store --repair-path`. + +> **Note** +> +> See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. From 829d4d3e03e0f7935c1fd3e1ef760980c8cf6046 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 7 Sep 2023 15:13:22 +0200 Subject: [PATCH 248/909] fix invalid anchor link --- src/libexpr/eval-settings.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index e6666061a..19122bc31 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -21,7 +21,7 @@ struct EvalSettings : Config R"( List of directories to be searched for `<...>` file references - In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of + In particular, outside of [pure evaluation mode](#conf-pure-eval), this determines the value of [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). )"}; From 7ad66cb3ef7615b51a35a099e232640d528c006c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 1 Oct 2021 18:05:53 -0400 Subject: [PATCH 249/909] Allow dynamic derivation deps in `inputDrvs` We use the same nested map representation we used for goals, again in order to save space. We might someday want to combine with `inputDrvs`, by doing `V = bool` instead of `V = std::set`, but we are not doing that yet for sake of a smaller diff. The ATerm format for Derivations also needs to be extended, in addition to the in-memory format. To accomodate this, we added a new basic versioning scheme, so old versions of Nix will get nice errors. (And going forward, if the ATerm format changes again the errors will be even better.) `parsedStrings`, an internal function used as part of parsing derivations in A-Term format, used to consume the final `]` but expect the initial `[` to already be consumed. This made for what looked like unbalanced brackets at callsites, which was confusing. Now it consumes both which is hopefully less confusing. As part of testing, we also created a unit test for the A-Term format for regular non-experimental derivations too. Co-authored-by: Robert Hensing Co-authored-by: Valentin Gagarin Apply suggestions from code review Co-authored-by: Robert Hensing --- doc/manual/src/protocols/derivation-aterm.md | 18 +- perl/lib/Nix/Store.xs | 2 +- src/build-remote/build-remote.cc | 2 +- src/libcmd/built-path.cc | 22 ++ src/libcmd/built-path.hh | 4 + src/libexpr/primops.cc | 10 +- src/libstore/build/derivation-goal.cc | 83 +++-- src/libstore/build/worker.cc | 9 +- src/libstore/build/worker.hh | 3 +- src/libstore/derivations.cc | 331 +++++++++++++++---- src/libstore/derivations.hh | 15 +- src/libstore/derived-path-map.cc | 41 ++- src/libstore/derived-path-map.hh | 25 ++ src/libstore/misc.cc | 67 ++-- src/libstore/tests/derivation.cc | 163 +++++++-- src/libutil/comparator.hh | 77 +++-- src/nix-build/nix-build.cc | 37 ++- src/nix/app.cc | 19 +- tests/dyn-drv/dep-built-drv.sh | 4 +- 19 files changed, 741 insertions(+), 191 deletions(-) diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md index c9dc00ef7..e58b602a3 100644 --- a/doc/manual/src/protocols/derivation-aterm.md +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -2,8 +2,18 @@ For historical reasons, [derivations](@docroot@/glossary.md#gloss-store-derivation) are stored on-disk in [ATerm](https://homepages.cwi.nl/~daybuild/daily-books/technology/aterm-guide/aterm-guide.html) format. -Derivations are serialised in the following format: +Derivations are serialised in one of the following formats: -``` -Derive(...) -``` +- ``` + Derive(...) + ``` + + For all stable derivations. + +- ``` + DrvWithVersion(, ...) + ``` + + The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md): + + - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index c38ea2d2b..e96885e4c 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -324,7 +324,7 @@ SV * derivationFromPath(char * drvPath) hv_stores(hash, "outputs", newRV((SV *) outputs)); AV * inputDrvs = newAV(); - for (auto & i : drv.inputDrvs) + for (auto & i : drv.inputDrvs.map) av_push(inputDrvs, newSVpv(store()->printStorePath(i.first).c_str(), 0)); // !!! ignores i->second hv_stores(hash, "inputDrvs", newRV((SV *) inputDrvs)); diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index c1f03a8ef..d69d3a0c2 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -314,7 +314,7 @@ connected: // // 2. Changing the `inputSrcs` set changes the associated // output ids, which break CA derivations - if (!drv.inputDrvs.empty()) + if (!drv.inputDrvs.map.empty()) drv.inputSrcs = store->parseStorePathSet(inputs); optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv); auto & result = *optResult; diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index c6cc7fa9c..9a2dce806 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -58,6 +58,28 @@ StorePathSet BuiltPath::outPaths() const ); } +SingleDerivedPath::Built SingleBuiltPath::Built::discardOutputPath() const +{ + return SingleDerivedPath::Built { + .drvPath = make_ref(drvPath->discardOutputPath()), + .output = output.first, + }; +} + +SingleDerivedPath SingleBuiltPath::discardOutputPath() const +{ + return std::visit( + overloaded{ + [](const SingleBuiltPath::Opaque & p) -> SingleDerivedPath { + return p; + }, + [](const SingleBuiltPath::Built & b) -> SingleDerivedPath { + return b.discardOutputPath(); + }, + }, raw() + ); +} + nlohmann::json BuiltPath::Built::toJSON(const Store & store) const { nlohmann::json res; diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index 747bcc440..e677bc810 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -9,6 +9,8 @@ struct SingleBuiltPathBuilt { ref drvPath; std::pair output; + SingleDerivedPathBuilt discardOutputPath() const; + std::string to_string(const Store & store) const; static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); nlohmann::json toJSON(const Store & store) const; @@ -34,6 +36,8 @@ struct SingleBuiltPath : _SingleBuiltPathRaw { StorePath outPath() const; + SingleDerivedPath discardOutputPath() const; + static SingleBuiltPath parse(const Store & store, std::string_view); nlohmann::json toJSON(const Store & store) const; }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e2b1ac4f6..cd9a05bb2 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1252,15 +1252,13 @@ drvName, Bindings * attrs, Value & v) state.store->computeFSClosure(d.drvPath, refs); for (auto & j : refs) { drv.inputSrcs.insert(j); - if (j.isDerivation()) - drv.inputDrvs[j] = state.store->readDerivation(j).outputNames(); + if (j.isDerivation()) { + drv.inputDrvs.map[j].value = state.store->readDerivation(j).outputNames(); + } } }, [&](const NixStringContextElem::Built & b) { - if (auto * p = std::get_if(&*b.drvPath)) - drv.inputDrvs[p->path].insert(b.output); - else - throw UnimplementedError("Dependencies on the outputs of dynamic derivations are not yet supported"); + drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output); }, [&](const NixStringContextElem::Opaque & o) { drv.inputSrcs.insert(o.path); diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index bec0bc538..6472ecd99 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -350,25 +350,37 @@ void DerivationGoal::gaveUpOnSubstitution() /* The inputs must be built before we can build this goal. */ inputDrvOutputs.clear(); - if (useDerivation) - for (auto & i : dynamic_cast(drv.get())->inputDrvs) { + if (useDerivation) { + std::function, const DerivedPathMap::ChildNode &)> addWaiteeDerivedPath; + + addWaiteeDerivedPath = [&](ref inputDrv, const DerivedPathMap::ChildNode & inputNode) { + if (!inputNode.value.empty()) + addWaitee(worker.makeGoal( + DerivedPath::Built { + .drvPath = inputDrv, + .outputs = inputNode.value, + }, + buildMode == bmRepair ? bmRepair : bmNormal)); + for (const auto & [outputName, childNode] : inputNode.childMap) + addWaiteeDerivedPath( + make_ref(SingleDerivedPath::Built { inputDrv, outputName }), + childNode); + }; + + for (const auto & [inputDrvPath, inputNode] : dynamic_cast(drv.get())->inputDrvs.map) { /* Ensure that pure, non-fixed-output derivations don't depend on impure derivations. */ if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) { - auto inputDrv = worker.evalStore.readDerivation(i.first); + auto inputDrv = worker.evalStore.readDerivation(inputDrvPath); if (!inputDrv.type().isPure()) throw Error("pure derivation '%s' depends on impure derivation '%s'", worker.store.printStorePath(drvPath), - worker.store.printStorePath(i.first)); + worker.store.printStorePath(inputDrvPath)); } - addWaitee(worker.makeGoal( - DerivedPath::Built { - .drvPath = makeConstantStorePathRef(i.first), - .outputs = i.second, - }, - buildMode == bmRepair ? bmRepair : bmNormal)); + addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode); } + } /* Copy the input sources from the eval store to the build store. */ @@ -501,7 +513,7 @@ void DerivationGoal::inputsRealised() return ia.deferred; }, [&](const DerivationType::ContentAddressed & ca) { - return !fullDrv.inputDrvs.empty() && ( + return !fullDrv.inputDrvs.map.empty() && ( ca.fixed /* Can optionally resolve if fixed, which is good for avoiding unnecessary rebuilds. */ @@ -515,7 +527,7 @@ void DerivationGoal::inputsRealised() } }, drvType.raw); - if (resolveDrv && !fullDrv.inputDrvs.empty()) { + if (resolveDrv && !fullDrv.inputDrvs.map.empty()) { experimentalFeatureSettings.require(Xp::CaDerivations); /* We are be able to resolve this derivation based on the @@ -552,11 +564,13 @@ void DerivationGoal::inputsRealised() return; } - for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { + std::function::ChildNode &)> accumInputPaths; + + accumInputPaths = [&](const StorePath & depDrvPath, const DerivedPathMap::ChildNode & inputNode) { /* Add the relevant output closures of the input derivation `i' as input paths. Only add the closures of output paths that are specified as inputs. */ - for (auto & j : wantedDepOutputs) { + auto getOutput = [&](const std::string & outputName) { /* TODO (impure derivations-induced tech debt): Tracking input derivation outputs statefully through the goals is error prone and has led to bugs. @@ -568,21 +582,30 @@ void DerivationGoal::inputsRealised() a representation in the store, which is a usability problem in itself. When implementing this logic entirely with lookups make sure that they're cached. */ - if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) { - worker.store.computeFSClosure(*outPath, inputPaths); + if (auto outPath = get(inputDrvOutputs, { depDrvPath, outputName })) { + return *outPath; } else { auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath); - auto outMapPath = outMap.find(j); + auto outMapPath = outMap.find(outputName); if (outMapPath == outMap.end()) { throw Error( "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); + worker.store.printStorePath(drvPath), outputName, worker.store.printStorePath(depDrvPath)); } - worker.store.computeFSClosure(outMapPath->second, inputPaths); + return outMapPath->second; } - } - } + }; + + for (auto & outputName : inputNode.value) + worker.store.computeFSClosure(getOutput(outputName), inputPaths); + + for (auto & [outputName, childNode] : inputNode.childMap) + accumInputPaths(getOutput(outputName), childNode); + }; + + for (auto & [depDrvPath, depNode] : fullDrv.inputDrvs.map) + accumInputPaths(depDrvPath, depNode); } /* Second, the input sources. */ @@ -1475,22 +1498,24 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) if (!useDerivation) return; auto & fullDrv = *dynamic_cast(drv.get()); - auto * dg = tryGetConcreteDrvGoal(waitee); - if (!dg) return; + std::optional info = tryGetConcreteDrvGoal(waitee); + if (!info) return; + const auto & [dg, drvReq] = *info; - auto outputs = fullDrv.inputDrvs.find(dg->drvPath); - if (outputs == fullDrv.inputDrvs.end()) return; + auto * nodeP = fullDrv.inputDrvs.findSlot(drvReq.get()); + if (!nodeP) return; + auto & outputs = nodeP->value; - for (auto & outputName : outputs->second) { - auto buildResult = dg->getBuildResult(DerivedPath::Built { - .drvPath = makeConstantStorePathRef(dg->drvPath), + for (auto & outputName : outputs) { + auto buildResult = dg.get().getBuildResult(DerivedPath::Built { + .drvPath = makeConstantStorePathRef(dg.get().drvPath), .outputs = OutputsSpec::Names { outputName }, }); if (buildResult.success()) { auto i = buildResult.builtOutputs.find(outputName); if (i != buildResult.builtOutputs.end()) inputDrvOutputs.insert_or_assign( - { dg->drvPath, outputName }, + { dg.get().drvPath, outputName }, i->second.outPath); } } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index f65f63b99..b4a634e7b 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -594,11 +594,14 @@ GoalPtr upcast_goal(std::shared_ptr subGoal) return subGoal; } -const DerivationGoal * tryGetConcreteDrvGoal(GoalPtr waitee) +std::optional, std::reference_wrapper>> tryGetConcreteDrvGoal(GoalPtr waitee) { auto * odg = dynamic_cast(&*waitee); - if (!odg) return nullptr; - return &*odg->concreteDrvGoal; + if (!odg) return std::nullopt; + return {{ + std::cref(*odg->concreteDrvGoal), + std::cref(*odg->drvReq), + }}; } } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index a778e311c..6f6d25d7d 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -49,7 +49,8 @@ typedef std::chrono::time_point steady_time_point; * we have made the function, written in `worker.cc` where all the goal * types are visible, and use it instead. */ -const DerivationGoal * tryGetConcreteDrvGoal(GoalPtr waitee); + +std::optional, std::reference_wrapper>> tryGetConcreteDrvGoal(GoalPtr waitee); /** * A mapping used to remember for each child process to what goal it diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 43d3dc4a2..67069c3c9 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -136,7 +136,7 @@ StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair, bool readOnly) { auto references = drv.inputSrcs; - for (auto & i : drv.inputDrvs) + for (auto & i : drv.inputDrvs.map) references.insert(i.first); /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be @@ -208,22 +208,25 @@ static bool endOfList(std::istream & str) static StringSet parseStrings(std::istream & str, bool arePaths) { StringSet res; + expect(str, "["); while (!endOfList(str)) res.insert(arePaths ? parsePath(str) : parseString(str)); return res; } -static DerivationOutput parseDerivationOutput(const Store & store, - std::string_view pathS, std::string_view hashAlgo, std::string_view hashS) +static DerivationOutput parseDerivationOutput( + const Store & store, + std::string_view pathS, std::string_view hashAlgo, std::string_view hashS, + const ExperimentalFeatureSettings & xpSettings) { if (hashAlgo != "") { ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo); if (method == TextIngestionMethod {}) - experimentalFeatureSettings.require(Xp::DynamicDerivations); + xpSettings.require(Xp::DynamicDerivations); const auto hashType = parseHashType(hashAlgo); if (hashS == "impure") { - experimentalFeatureSettings.require(Xp::ImpureDerivations); + xpSettings.require(Xp::ImpureDerivations); if (pathS != "") throw FormatError("impure derivation output should not specify output path"); return DerivationOutput::Impure { @@ -240,7 +243,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, }, }; } else { - experimentalFeatureSettings.require(Xp::CaDerivations); + xpSettings.require(Xp::CaDerivations); if (pathS != "") throw FormatError("content-addressed derivation output should not specify output path"); return DerivationOutput::CAFloating { @@ -259,29 +262,116 @@ static DerivationOutput parseDerivationOutput(const Store & store, } } -static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str) +static DerivationOutput parseDerivationOutput( + const Store & store, std::istringstream & str, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings) { expect(str, ","); const auto pathS = parseString(str); expect(str, ","); const auto hashAlgo = parseString(str); expect(str, ","); const auto hash = parseString(str); expect(str, ")"); - return parseDerivationOutput(store, pathS, hashAlgo, hash); + return parseDerivationOutput(store, pathS, hashAlgo, hash, xpSettings); +} + +/** + * All ATerm Derivation format versions currently known. + * + * Unknown versions are rejected at the parsing stage. + */ +enum struct DerivationATermVersion { + /** + * Older unversioned form + */ + Traditional, + + /** + * Newer versioned form; only this version so far. + */ + DynamicDerivations, +}; + +static DerivedPathMap::ChildNode parseDerivedPathMapNode( + const Store & store, + std::istringstream & str, + DerivationATermVersion version) +{ + DerivedPathMap::ChildNode node; + + auto parseNonDynamic = [&]() { + node.value = parseStrings(str, false); + }; + + // Older derivation should never use new form, but newer + // derivaiton can use old form. + switch (version) { + case DerivationATermVersion::Traditional: + parseNonDynamic(); + break; + case DerivationATermVersion::DynamicDerivations: + switch (str.peek()) { + case '[': + parseNonDynamic(); + break; + case '(': + expect(str, "("); + node.value = parseStrings(str, false); + expect(str, ",["); + while (!endOfList(str)) { + expect(str, "("); + auto outputName = parseString(str); + expect(str, ","); + node.childMap.insert_or_assign(outputName, parseDerivedPathMapNode(store, str, version)); + expect(str, ")"); + } + expect(str, ")"); + break; + default: + throw FormatError("invalid inputDrvs entry in derivation"); + } + break; + default: + // invalid format, not a parse error but internal error + assert(false); + } + return node; } -Derivation parseDerivation(const Store & store, std::string && s, std::string_view name) +Derivation parseDerivation( + const Store & store, std::string && s, std::string_view name, + const ExperimentalFeatureSettings & xpSettings) { Derivation drv; drv.name = name; std::istringstream str(std::move(s)); - expect(str, "Derive(["); + expect(str, "D"); + DerivationATermVersion version; + switch (str.peek()) { + case 'e': + expect(str, "erive("); + version = DerivationATermVersion::Traditional; + break; + case 'r': + expect(str, "rvWithVersion("); + auto versionS = parseString(str); + if (versionS == "xp-dyn-drv") { + // Only verison we have so far + version = DerivationATermVersion::DynamicDerivations; + xpSettings.require(Xp::DynamicDerivations); + } else { + throw FormatError("Unknown derivation ATerm format version '%s'", versionS); + } + expect(str, ","); + break; + } /* Parse the list of outputs. */ + expect(str, "["); while (!endOfList(str)) { expect(str, "("); std::string id = parseString(str); - auto output = parseDerivationOutput(store, str); + auto output = parseDerivationOutput(store, str, xpSettings); drv.outputs.emplace(std::move(id), std::move(output)); } @@ -290,12 +380,12 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi while (!endOfList(str)) { expect(str, "("); Path drvPath = parsePath(str); - expect(str, ",["); - drv.inputDrvs.insert_or_assign(store.parseStorePath(drvPath), parseStrings(str, false)); + expect(str, ","); + drv.inputDrvs.map.insert_or_assign(store.parseStorePath(drvPath), parseDerivedPathMapNode(store, str, version)); expect(str, ")"); } - expect(str, ",["); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true)); + expect(str, ","); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true)); expect(str, ","); drv.platform = parseString(str); expect(str, ","); drv.builder = parseString(str); @@ -379,14 +469,67 @@ static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIt } +static void unparseDerivedPathMapNode(const Store & store, std::string & s, const DerivedPathMap::ChildNode & node) +{ + s += ','; + if (node.childMap.empty()) { + printUnquotedStrings(s, node.value.begin(), node.value.end()); + } else { + s += "("; + printUnquotedStrings(s, node.value.begin(), node.value.end()); + s += ",["; + bool first = true; + for (auto & [outputName, childNode] : node.childMap) { + if (first) first = false; else s += ','; + s += '('; printUnquotedString(s, outputName); + unparseDerivedPathMapNode(store, s, childNode); + s += ')'; + } + s += "])"; + } +} + + +/** + * Does the derivation have a dependency on the output of a dynamic + * derivation? + * + * In other words, does it on the output of derivation that is itself an + * ouput of a derivation? This corresponds to a dependency that is an + * inductive derived path with more than one layer of + * `DerivedPath::Built`. + */ +static bool hasDynamicDrvDep(const Derivation & drv) +{ + return + std::find_if( + drv.inputDrvs.map.begin(), + drv.inputDrvs.map.end(), + [](auto & kv) { return !kv.second.childMap.empty(); }) + != drv.inputDrvs.map.end(); +} + + std::string Derivation::unparse(const Store & store, bool maskOutputs, - std::map * actualInputs) const + DerivedPathMap::ChildNode::Map * actualInputs) const { std::string s; s.reserve(65536); - s += "Derive(["; + + /* Use older unversioned form if possible, for wider compat. Use + newer form only if we need it, which we do for + `Xp::DynamicDerivations`. */ + if (hasDynamicDrvDep(*this)) { + s += "DrvWithVersion("; + // Only version we have so far + printUnquotedString(s, "xp-dyn-drv"); + s += ","; + } else { + s += "Derive("; + } bool first = true; + s += "["; for (auto & i : outputs) { if (first) first = false; else s += ','; s += '('; printUnquotedString(s, i.first); @@ -424,17 +567,17 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, s += "],["; first = true; if (actualInputs) { - for (auto & i : *actualInputs) { + for (auto & [drvHashModulo, childMap] : *actualInputs) { if (first) first = false; else s += ','; - s += '('; printUnquotedString(s, i.first); - s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end()); + s += '('; printUnquotedString(s, drvHashModulo); + unparseDerivedPathMapNode(store, s, childMap); s += ')'; } } else { - for (auto & i : inputDrvs) { + for (auto & [drvPath, childMap] : inputDrvs.map) { if (first) first = false; else s += ','; - s += '('; printUnquotedString(s, store.printStorePath(i.first)); - s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end()); + s += '('; printUnquotedString(s, store.printStorePath(drvPath)); + unparseDerivedPathMapNode(store, s, childMap); s += ')'; } } @@ -668,18 +811,16 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut } }, drv.type().raw); - std::map inputs2; - for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) { - // Avoid lambda capture restriction with standard / Clang - auto & inputOutputs = inputOutputs0; + DerivedPathMap::ChildNode::Map inputs2; + for (auto & [drvPath, node] : drv.inputDrvs.map) { const auto & res = pathDerivationModulo(store, drvPath); if (res.kind == DrvHash::Kind::Deferred) kind = DrvHash::Kind::Deferred; - for (auto & outputName : inputOutputs) { + for (auto & outputName : node.value) { const auto h = get(res.hashes, outputName); if (!h) throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); - inputs2[h->to_string(Base16, false)].insert(outputName); + inputs2[h->to_string(Base16, false)].value.insert(outputName); } } @@ -709,7 +850,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) const auto hashAlgo = readString(in); const auto hash = readString(in); - return parseDerivationOutput(store, pathS, hashAlgo, hash); + return parseDerivationOutput(store, pathS, hashAlgo, hash, experimentalFeatureSettings); } StringSet BasicDerivation::outputNames() const @@ -824,6 +965,8 @@ std::string hashPlaceholder(const OutputNameView outputName) static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) { + debug("Rewriting the derivation"); + for (auto & rewrite : rewrites) { debug("rewriting %s as %s", rewrite.first, rewrite.second); } @@ -862,14 +1005,70 @@ std::optional Derivation::tryResolve(Store & store) const { std::map, StorePath> inputDrvOutputs; - for (auto & input : inputDrvs) - for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first)) - if (outputPath) - inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath); + std::function::ChildNode &)> accum; + accum = [&](auto & inputDrv, auto & node) { + for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv)) { + if (outputPath) { + inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath); + if (auto p = get(node.childMap, outputName)) + accum(*outputPath, *p); + } + } + }; + + for (auto & [inputDrv, node] : inputDrvs.map) + accum(inputDrv, node); return tryResolve(store, inputDrvOutputs); } +static bool tryResolveInput( + Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites, + const DownstreamPlaceholder * placeholderOpt, + const StorePath & inputDrv, const DerivedPathMap::ChildNode & inputNode, + const std::map, StorePath> & inputDrvOutputs) +{ + auto getOutput = [&](const std::string & outputName) { + auto * actualPathOpt = get(inputDrvOutputs, { inputDrv, outputName }); + if (!actualPathOpt) + warn("output %s of input %s missing, aborting the resolving", + outputName, + store.printStorePath(inputDrv) + ); + return actualPathOpt; + }; + + auto getPlaceholder = [&](const std::string & outputName) { + return placeholderOpt + ? DownstreamPlaceholder::unknownDerivation(*placeholderOpt, outputName) + : DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName); + }; + + for (auto & outputName : inputNode.value) { + auto actualPathOpt = getOutput(outputName); + if (!actualPathOpt) return false; + auto actualPath = *actualPathOpt; + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + inputRewrites.emplace( + getPlaceholder(outputName).render(), + store.printStorePath(actualPath)); + } + inputSrcs.insert(std::move(actualPath)); + } + + for (auto & [outputName, childNode] : inputNode.childMap) { + auto actualPathOpt = getOutput(outputName); + if (!actualPathOpt) return false; + auto actualPath = *actualPathOpt; + auto nextPlaceholder = getPlaceholder(outputName); + if (!tryResolveInput(store, inputSrcs, inputRewrites, + &nextPlaceholder, actualPath, childNode, + inputDrvOutputs)) + return false; + } + return true; +} + std::optional Derivation::tryResolve( Store & store, const std::map, StorePath> & inputDrvOutputs) const @@ -879,23 +1078,10 @@ std::optional Derivation::tryResolve( // Input paths that we'll want to rewrite in the derivation StringMap inputRewrites; - for (auto & [inputDrv, inputOutputs] : inputDrvs) { - for (auto & outputName : inputOutputs) { - if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) { - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - inputRewrites.emplace( - DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(), - store.printStorePath(*actualPath)); - } - resolved.inputSrcs.insert(*actualPath); - } else { - warn("output '%s' of input '%s' missing, aborting the resolving", - outputName, - store.printStorePath(inputDrv)); - return {}; - } - } - } + for (auto & [inputDrv, inputNode] : inputDrvs.map) + if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, + nullptr, inputDrv, inputNode, inputDrvOutputs)) + return std::nullopt; rewriteDerivation(store, resolved, inputRewrites); @@ -1084,10 +1270,25 @@ nlohmann::json Derivation::toJSON(const Store & store) const } { - auto& inputDrvsObj = res["inputDrvs"]; - inputDrvsObj = nlohmann::json ::object(); - for (auto & input : inputDrvs) - inputDrvsObj[store.printStorePath(input.first)] = input.second; + std::function::ChildNode &)> doInput; + doInput = [&](const auto & inputNode) { + auto value = nlohmann::json::object(); + value["outputs"] = inputNode.value; + { + auto next = nlohmann::json::object(); + for (auto & [outputId, childNode] : inputNode.childMap) + next[outputId] = doInput(childNode); + value["dynamicOutputs"] = std::move(next); + } + return value; + }; + { + auto& inputDrvsObj = res["inputDrvs"]; + inputDrvsObj = nlohmann::json::object(); + for (auto & [inputDrv, inputNode] : inputDrvs.map) { + inputDrvsObj[store.printStorePath(inputDrv)] = doInput(inputNode); + } + } } res["system"] = platform; @@ -1101,7 +1302,8 @@ nlohmann::json Derivation::toJSON(const Store & store) const Derivation Derivation::fromJSON( const Store & store, - const nlohmann::json & json) + const nlohmann::json & json, + const ExperimentalFeatureSettings & xpSettings) { using nlohmann::detail::value_t; @@ -1133,12 +1335,21 @@ Derivation Derivation::fromJSON( } try { + std::function::ChildNode(const nlohmann::json &)> doInput; + doInput = [&](const auto & json) { + DerivedPathMap::ChildNode node; + node.value = static_cast( + ensureType(valueAt(json, "outputs"), value_t::array)); + for (auto & [outputId, childNode] : ensureType(valueAt(json, "dynamicOutputs"), value_t::object).items()) { + xpSettings.require(Xp::DynamicDerivations); + node.childMap[outputId] = doInput(childNode); + } + return node; + }; auto & inputDrvsObj = ensureType(valueAt(json, "inputDrvs"), value_t::object); - for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) { - ensureType(inputOutputs, value_t::array); - res.inputDrvs[store.parseStorePath(inputDrvPath)] = - static_cast(inputOutputs); - } + for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) + res.inputDrvs.map[store.parseStorePath(inputDrvPath)] = + doInput(inputOutputs); } catch (Error & e) { e.addTrace({}, "while reading key 'inputDrvs'"); throw; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 106056f2d..fa14e7536 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -6,7 +6,7 @@ #include "hash.hh" #include "content-address.hh" #include "repair-flag.hh" -#include "derived-path.hh" +#include "derived-path-map.hh" #include "sync.hh" #include "comparator.hh" #include "variant-wrapper.hh" @@ -323,13 +323,13 @@ struct Derivation : BasicDerivation /** * inputs that are sub-derivations */ - DerivationInputs inputDrvs; + DerivedPathMap> inputDrvs; /** * Print a derivation. */ std::string unparse(const Store & store, bool maskOutputs, - std::map * actualInputs = nullptr) const; + DerivedPathMap::ChildNode::Map * actualInputs = nullptr) const; /** * Return the underlying basic derivation but with these changes: @@ -368,7 +368,8 @@ struct Derivation : BasicDerivation nlohmann::json toJSON(const Store & store) const; static Derivation fromJSON( const Store & store, - const nlohmann::json & json); + const nlohmann::json & json, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); GENERATE_CMP(Derivation, static_cast(*me), @@ -389,7 +390,11 @@ StorePath writeDerivation(Store & store, /** * Read a derivation from a file. */ -Derivation parseDerivation(const Store & store, std::string && s, std::string_view name); +Derivation parseDerivation( + const Store & store, + std::string && s, + std::string_view name, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** * \todo Remove. diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index 5c8c7a4f2..437b6a71a 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -21,6 +21,32 @@ typename DerivedPathMap::ChildNode & DerivedPathMap::ensureSlot(const Sing return initIter(k); } +template +typename DerivedPathMap::ChildNode * DerivedPathMap::findSlot(const SingleDerivedPath & k) +{ + std::function initIter; + initIter = [&](const auto & k) { + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + auto it = map.find(bo.path); + return it != map.end() + ? &it->second + : nullptr; + }, + [&](const SingleDerivedPath::Built & bfd) { + auto * n = initIter(*bfd.drvPath); + if (!n) return (ChildNode *)nullptr; + + auto it = n->childMap.find(bfd.output); + return it != n->childMap.end() + ? &it->second + : nullptr; + }, + }, k.raw()); + }; + return initIter(k); +} + } // instantiations @@ -30,4 +56,17 @@ namespace nix { template struct DerivedPathMap>; -} +GENERATE_CMP_EXT( + template<>, + DerivedPathMap>::ChildNode, + me->value, + me->childMap); + +GENERATE_CMP_EXT( + template<>, + DerivedPathMap>, + me->map); + +template struct DerivedPathMap>; + +}; diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh index 9ce58206e..4a2c90733 100644 --- a/src/libstore/derived-path-map.hh +++ b/src/libstore/derived-path-map.hh @@ -48,6 +48,8 @@ struct DerivedPathMap { * The map of the root node. */ Map childMap; + + DECLARE_CMP(ChildNode); }; /** @@ -60,6 +62,8 @@ struct DerivedPathMap { */ Map map; + DECLARE_CMP(DerivedPathMap); + /** * Find the node for `k`, creating it if needed. * @@ -68,6 +72,27 @@ struct DerivedPathMap { * by changing this node. */ ChildNode & ensureSlot(const SingleDerivedPath & k); + + /** + * Like `ensureSlot` but does not create the slot if it doesn't exist. + * + * Read the entire description of `ensureSlot` to understand an + * important caveat here that "have slot" does *not* imply "key is + * set in map". To ensure a key is set one would need to get the + * child node (with `findSlot` or `ensureSlot`) *and* check the + * `ChildNode::value`. + */ + ChildNode * findSlot(const SingleDerivedPath & k); }; + +DECLARE_CMP_EXT( + template<>, + DerivedPathMap>::, + DerivedPathMap>); +DECLARE_CMP_EXT( + template<>, + DerivedPathMap>::ChildNode::, + DerivedPathMap>::ChildNode); + } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index c043b9b93..1035691c7 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -125,14 +125,26 @@ void Store::queryMissing(const std::vector & targets, std::function doPath; + std::function, const DerivedPathMap::ChildNode &)> enqueueDerivedPaths; + + enqueueDerivedPaths = [&](ref inputDrv, const DerivedPathMap::ChildNode & inputNode) { + if (!inputNode.value.empty()) + pool.enqueue(std::bind(doPath, DerivedPath::Built { inputDrv, inputNode.value })); + for (const auto & [outputName, childNode] : inputNode.childMap) + enqueueDerivedPaths( + make_ref(SingleDerivedPath::Built { inputDrv, outputName }), + childNode); + }; + auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { { auto state(state_.lock()); state->willBuild.insert(drvPath); } - for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, DerivedPath::Built { makeConstantStorePathRef(i.first), i.second })); + for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) { + enqueueDerivedPaths(makeConstantStorePathRef(inputDrv), inputNode); + } }; auto checkOutput = [&]( @@ -322,24 +334,41 @@ std::map drvOutputReferences( { std::set inputRealisations; - for (const auto & [inputDrv, outputNames] : drv.inputDrvs) { - const auto outputHashes = - staticOutputHashes(store, store.readDerivation(inputDrv)); - for (const auto & outputName : outputNames) { - auto outputHash = get(outputHashes, outputName); - if (!outputHash) - throw Error( - "output '%s' of derivation '%s' isn't realised", outputName, - store.printStorePath(inputDrv)); - auto thisRealisation = store.queryRealisation( - DrvOutput{*outputHash, outputName}); - if (!thisRealisation) - throw Error( - "output '%s' of derivation '%s' isn't built", outputName, - store.printStorePath(inputDrv)); - inputRealisations.insert(*thisRealisation); + std::function::ChildNode &)> accumRealisations; + + accumRealisations = [&](const StorePath & inputDrv, const DerivedPathMap::ChildNode & inputNode) { + if (!inputNode.value.empty()) { + auto outputHashes = + staticOutputHashes(store, store.readDerivation(inputDrv)); + for (const auto & outputName : inputNode.value) { + auto outputHash = get(outputHashes, outputName); + if (!outputHash) + throw Error( + "output '%s' of derivation '%s' isn't realised", outputName, + store.printStorePath(inputDrv)); + auto thisRealisation = store.queryRealisation( + DrvOutput{*outputHash, outputName}); + if (!thisRealisation) + throw Error( + "output '%s' of derivation '%s' isn’t built", outputName, + store.printStorePath(inputDrv)); + inputRealisations.insert(*thisRealisation); + } } - } + if (!inputNode.value.empty()) { + auto d = makeConstantStorePathRef(inputDrv); + for (const auto & [outputName, childNode] : inputNode.childMap) { + SingleDerivedPath next = SingleDerivedPath::Built { d, outputName }; + accumRealisations( + // TODO deep resolutions for dynamic derivations, issue #8947, would go here. + resolveDerivedPath(store, next), + childNode); + } + } + }; + + for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) + accumRealisations(inputDrv, inputNode); auto info = store.queryPathInfo(outputPath); diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 7fd7f9278..c360c9707 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -42,6 +42,26 @@ class ImpureDerivationTest : public DerivationTest } }; +TEST_F(DerivationTest, BadATerm_version) { + ASSERT_THROW( + parseDerivation( + *store, + R"(DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", + "whatever", + mockXpSettings), + FormatError); +} + +TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { + ASSERT_THROW( + parseDerivation( + *store, + R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", + "dyn-dep-derivation", + mockXpSettings), + FormatError); +} + #define TEST_JSON(FIXTURE, NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ using nlohmann::literals::operator "" _json; \ @@ -143,35 +163,37 @@ TEST_JSON(ImpureDerivationTest, impure, #undef TEST_JSON -#define TEST_JSON(NAME, STR, VAL) \ - TEST_F(DerivationTest, Derivation_ ## NAME ## _to_json) { \ +#define TEST_JSON(FIXTURE, NAME, STR, VAL) \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \ using nlohmann::literals::operator "" _json; \ ASSERT_EQ( \ STR ## _json, \ (VAL).toJSON(*store)); \ } \ \ - TEST_F(DerivationTest, Derivation_ ## NAME ## _from_json) { \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \ using nlohmann::literals::operator "" _json; \ ASSERT_EQ( \ (VAL), \ Derivation::fromJSON( \ *store, \ - STR ## _json)); \ + STR ## _json, \ + mockXpSettings)); \ } -#define TEST_ATERM(NAME, STR, VAL, DRV_NAME) \ - TEST_F(DerivationTest, Derivation_ ## NAME ## _to_aterm) { \ +#define TEST_ATERM(FIXTURE, NAME, STR, VAL, DRV_NAME) \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \ ASSERT_EQ( \ STR, \ (VAL).unparse(*store, false)); \ } \ \ - TEST_F(DerivationTest, Derivation_ ## NAME ## _from_aterm) { \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \ auto parsed = parseDerivation( \ - *store, \ - STR, \ - DRV_NAME); \ + *store, \ + STR, \ + DRV_NAME, \ + mockXpSettings); \ ASSERT_EQ( \ (VAL).toJSON(*store), \ parsed.toJSON(*store)); \ @@ -187,11 +209,15 @@ Derivation makeSimpleDrv(const Store & store) { store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), }; drv.inputDrvs = { - { - store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + .map = { { - "cat", - "dog", + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + { + .value = { + "cat", + "dog", + }, + }, }, }, }; @@ -210,17 +236,20 @@ Derivation makeSimpleDrv(const Store & store) { return drv; } -TEST_JSON(simple, +TEST_JSON(DerivationTest, simple, R"({ "name": "simple-derivation", "inputSrcs": [ "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" ], "inputDrvs": { - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": [ - "cat", - "dog" - ] + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": {}, + "outputs": [ + "cat", + "dog" + ] + } }, "system": "wasm-sel4", "builder": "foo", @@ -235,11 +264,105 @@ TEST_JSON(simple, })", makeSimpleDrv(*store)) -TEST_ATERM(simple, +TEST_ATERM(DerivationTest, simple, R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", makeSimpleDrv(*store), "simple-derivation") +Derivation makeDynDepDerivation(const Store & store) { + Derivation drv; + drv.name = "dyn-dep-derivation"; + drv.inputSrcs = { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + .map = { + { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + DerivedPathMap::ChildNode { + .value = { + "cat", + "dog", + }, + .childMap = { + { + "cat", + DerivedPathMap::ChildNode { + .value = { + "kitten", + }, + }, + }, + { + "goose", + DerivedPathMap::ChildNode { + .value = { + "gosling", + }, + }, + }, + }, + }, + }, + }, + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = { + { + "BIG_BAD", + "WOLF", + }, + }; + return drv; +} + +TEST_JSON(DynDerivationTest, dynDerivationDeps, + R"({ + "name": "dyn-dep-derivation", + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": { + "cat": { + "dynamicOutputs": {}, + "outputs": ["kitten"] + }, + "goose": { + "dynamicOutputs": {}, + "outputs": ["gosling"] + } + }, + "outputs": [ + "cat", + "dog" + ] + } + }, + "system": "wasm-sel4", + "builder": "foo", + "args": [ + "bar", + "baz" + ], + "env": { + "BIG_BAD": "WOLF" + }, + "outputs": {} + })", + makeDynDepDerivation(*store)) + +TEST_ATERM(DynDerivationTest, dynDerivationDeps, + R"(DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", + makeDynDepDerivation(*store), + "dyn-dep-derivation") + #undef TEST_JSON #undef TEST_ATERM diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index 7982fdc5e..a4d20a675 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -1,21 +1,48 @@ #pragma once ///@file +#define DECLARE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE) \ + PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const; +#define DECLARE_EQUAL(prefix, qualification, my_type) \ + DECLARE_ONE_CMP(prefix, qualification, ==, my_type) +#define DECLARE_LEQ(prefix, qualification, my_type) \ + DECLARE_ONE_CMP(prefix, qualification, <, my_type) +#define DECLARE_NEQ(prefix, qualification, my_type) \ + DECLARE_ONE_CMP(prefix, qualification, !=, my_type) + +#define GENERATE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE, ...) \ + PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const { \ + __VA_OPT__(const MY_TYPE * me = this;) \ + auto fields1 = std::make_tuple( __VA_ARGS__ ); \ + __VA_OPT__(me = &other;) \ + auto fields2 = std::make_tuple( __VA_ARGS__ ); \ + return fields1 COMPARATOR fields2; \ + } +#define GENERATE_EQUAL(prefix, qualification, my_type, args...) \ + GENERATE_ONE_CMP(prefix, qualification, ==, my_type, args) +#define GENERATE_LEQ(prefix, qualification, my_type, args...) \ + GENERATE_ONE_CMP(prefix, qualification, <, my_type, args) +#define GENERATE_NEQ(prefix, qualification, my_type, args...) \ + GENERATE_ONE_CMP(prefix, qualification, !=, my_type, args) + /** * Declare comparison methods without defining them. */ -#define DECLARE_ONE_CMP(COMPARATOR, MY_TYPE) \ - bool operator COMPARATOR(const MY_TYPE & other) const; -#define DECLARE_EQUAL(my_type) \ - DECLARE_ONE_CMP(==, my_type) -#define DECLARE_LEQ(my_type) \ - DECLARE_ONE_CMP(<, my_type) -#define DECLARE_NEQ(my_type) \ - DECLARE_ONE_CMP(!=, my_type) #define DECLARE_CMP(my_type) \ - DECLARE_EQUAL(my_type) \ - DECLARE_LEQ(my_type) \ - DECLARE_NEQ(my_type) + DECLARE_EQUAL(,,my_type) \ + DECLARE_LEQ(,,my_type) \ + DECLARE_NEQ(,,my_type) + +/** + * @param prefix This is for something before each declaration like + * `template`. + * + * @param my_type the type are defining operators for. + */ +#define DECLARE_CMP_EXT(prefix, qualification, my_type) \ + DECLARE_EQUAL(prefix, qualification, my_type) \ + DECLARE_LEQ(prefix, qualification, my_type) \ + DECLARE_NEQ(prefix, qualification, my_type) /** * Awful hacky generation of the comparison operators by doing a lexicographic @@ -33,18 +60,18 @@ * } * ``` */ -#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, ...) \ - bool operator COMPARATOR(const MY_TYPE& other) const { \ - __VA_OPT__(const MY_TYPE* me = this;) \ - auto fields1 = std::make_tuple( __VA_ARGS__ ); \ - __VA_OPT__(me = &other;) \ - auto fields2 = std::make_tuple( __VA_ARGS__ ); \ - return fields1 COMPARATOR fields2; \ - } -#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) -#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args) -#define GENERATE_NEQ(args...) GENERATE_ONE_CMP(!=, args) #define GENERATE_CMP(args...) \ - GENERATE_EQUAL(args) \ - GENERATE_LEQ(args) \ - GENERATE_NEQ(args) + GENERATE_EQUAL(,,args) \ + GENERATE_LEQ(,,args) \ + GENERATE_NEQ(,,args) + +/** + * @param prefix This is for something before each declaration like + * `template`. + * + * @param my_type the type are defining operators for. + */ +#define GENERATE_CMP_EXT(prefix, my_type, args...) \ + GENERATE_EQUAL(prefix, my_type ::, my_type, args) \ + GENERATE_LEQ(prefix, my_type ::, my_type, args) \ + GENERATE_NEQ(prefix, my_type ::, my_type, args) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 66f319c3e..e2189fc66 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -406,8 +406,22 @@ static void main_nix_build(int argc, char * * argv) } } + std::function, const DerivedPathMap::ChildNode &)> accumDerivedPath; + + accumDerivedPath = [&](ref inputDrv, const DerivedPathMap::ChildNode & inputNode) { + if (!inputNode.value.empty()) + pathsToBuild.push_back(DerivedPath::Built { + .drvPath = inputDrv, + .outputs = OutputsSpec::Names { inputNode.value }, + }); + for (const auto & [outputName, childNode] : inputNode.childMap) + accumDerivedPath( + make_ref(SingleDerivedPath::Built { inputDrv, outputName }), + childNode); + }; + // Build or fetch all dependencies of the derivation. - for (const auto & [inputDrv0, inputOutputs] : drv.inputDrvs) { + for (const auto & [inputDrv0, inputNode] : drv.inputDrvs.map) { // To get around lambda capturing restrictions in the // standard. const auto & inputDrv = inputDrv0; @@ -416,10 +430,7 @@ static void main_nix_build(int argc, char * * argv) return !std::regex_search(store->printStorePath(inputDrv), std::regex(exclude)); })) { - pathsToBuild.push_back(DerivedPath::Built { - .drvPath = makeConstantStorePathRef(inputDrv), - .outputs = OutputsSpec::Names { inputOutputs }, - }); + accumDerivedPath(makeConstantStorePathRef(inputDrv), inputNode); pathsToCopy.insert(inputDrv); } } @@ -482,13 +493,21 @@ static void main_nix_build(int argc, char * * argv) if (env.count("__json")) { StorePathSet inputs; - for (auto & [depDrvPath, wantedDepOutputs] : drv.inputDrvs) { - auto outputs = evalStore->queryPartialDerivationOutputMap(depDrvPath); - for (auto & i : wantedDepOutputs) { + + std::function::ChildNode &)> accumInputClosure; + + accumInputClosure = [&](const StorePath & inputDrv, const DerivedPathMap::ChildNode & inputNode) { + auto outputs = evalStore->queryPartialDerivationOutputMap(inputDrv); + for (auto & i : inputNode.value) { auto o = outputs.at(i); store->computeFSClosure(*o, inputs); } - } + for (const auto & [outputName, childNode] : inputNode.childMap) + accumInputClosure(*outputs.at(outputName), childNode); + }; + + for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) + accumInputClosure(inputDrv, inputNode); ParsedDerivation parsedDrv(drvInfo.requireDrvPath(), drv); diff --git a/src/nix/app.cc b/src/nix/app.cc index 67c5ef344..34fac9935 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -20,15 +20,22 @@ StringPairs resolveRewrites( const std::vector & dependencies) { StringPairs res; - for (auto & dep : dependencies) - if (auto drvDep = std::get_if(&dep.path)) - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) - for (auto & [ outputName, outputPath ] : drvDep->outputs) + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + for (auto & dep : dependencies) { + if (auto drvDep = std::get_if(&dep.path)) { + for (auto & [ outputName, outputPath ] : drvDep->outputs) { res.emplace( - DownstreamPlaceholder::unknownCaOutput( - drvDep->drvPath->outPath(), outputName).render(), + DownstreamPlaceholder::fromSingleDerivedPathBuilt( + SingleDerivedPath::Built { + .drvPath = make_ref(drvDep->drvPath->discardOutputPath()), + .output = outputName, + }).render(), store.printStorePath(outputPath) ); + } + } + } + } return res; } diff --git a/tests/dyn-drv/dep-built-drv.sh b/tests/dyn-drv/dep-built-drv.sh index 8c4a45e3b..c3daf2c59 100644 --- a/tests/dyn-drv/dep-built-drv.sh +++ b/tests/dyn-drv/dep-built-drv.sh @@ -6,4 +6,6 @@ out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link) clearStore -expectStderr 1 nix-build ./text-hashed-output.nix -A wrapper --no-out-link | grepQuiet "Dependencies on the outputs of dynamic derivations are not yet supported" +out2=$(nix-build ./text-hashed-output.nix -A wrapper --no-out-link) + +diff -r $out1 $out2 From 80d7994f52ccefca2f2fbe4ee64741a6f49884ff Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 6 Sep 2023 22:57:37 -0400 Subject: [PATCH 250/909] Special-case error message to add extra information The Derivation parser and old ATerm unfortunately leaves few ways to get nice errors when an old version of Nix encounters a new version of the format. The most likely scenario for this to occur is with a new client making a derivation that the old daemon it is communicating with cannot understand. The extensions we just created for dynamic derivation deps will add a version field, solving the problem going forward, but there is still the issue of what to do about old versions of Nix up to now. The solution here is to carefully catch the bad error from the daemon that is likely to indicate this problem, and add some extra context to it. There is another "Ugly backwards compatibility hack" in `remote-store.cc` that also works by transforming an error. Co-authored-by: Robert Hensing --- src/libstore/remote-store.cc | 19 ++++++++++++++++++- tests/dyn-drv/local.mk | 3 ++- tests/dyn-drv/old-daemon-error-hack.nix | 20 ++++++++++++++++++++ tests/dyn-drv/old-daemon-error-hack.sh | 11 +++++++++++ 4 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 tests/dyn-drv/old-daemon-error-hack.nix create mode 100644 tests/dyn-drv/old-daemon-error-hack.sh diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 58f72beb9..a639346d1 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -172,7 +172,24 @@ void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, auto ex = handle->processStderr(sink, source, flush); if (ex) { daemonException = true; - std::rethrow_exception(ex); + try { + std::rethrow_exception(ex); + } catch (const Error & e) { + // Nix versions before #4628 did not have an adequate behavior for reporting that the derivation format was upgraded. + // To avoid having to add compatibility logic in many places, we expect to catch almost all occurrences of the + // old incomprehensible error here, so that we can explain to users what's going on when their daemon is + // older than #4628 (2023). + if (experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations) && + GET_PROTOCOL_MINOR(handle->daemonVersion) <= 35) + { + auto m = e.msg(); + if (m.find("parsing derivation") != std::string::npos && + m.find("expected string") != std::string::npos && + m.find("Derive([") != std::string::npos) + throw Error("%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw dervation is in the form '%s'", std::move(m), "DrvWithVersion(..)"); + } + throw; + } } } diff --git a/tests/dyn-drv/local.mk b/tests/dyn-drv/local.mk index 0ce7cd37d..6b435499b 100644 --- a/tests/dyn-drv/local.mk +++ b/tests/dyn-drv/local.mk @@ -3,7 +3,8 @@ dyn-drv-tests := \ $(d)/recursive-mod-json.sh \ $(d)/build-built-drv.sh \ $(d)/eval-outputOf.sh \ - $(d)/dep-built-drv.sh + $(d)/dep-built-drv.sh \ + $(d)/old-daemon-error-hack.sh install-tests-groups += dyn-drv diff --git a/tests/dyn-drv/old-daemon-error-hack.nix b/tests/dyn-drv/old-daemon-error-hack.nix new file mode 100644 index 000000000..c9d4a62d4 --- /dev/null +++ b/tests/dyn-drv/old-daemon-error-hack.nix @@ -0,0 +1,20 @@ +with import ./config.nix; + +# A simple content-addressed derivation. +# The derivation can be arbitrarily modified by passing a different `seed`, +# but the output will always be the same +rec { + stub = mkDerivation { + name = "stub"; + buildCommand = '' + echo stub > $out + ''; + }; + wrapper = mkDerivation { + name = "has-dynamic-drv-dep"; + buildCommand = '' + exit 1 # we're not building this derivation + ${builtins.outputOf stub.outPath "out"} + ''; + }; +} diff --git a/tests/dyn-drv/old-daemon-error-hack.sh b/tests/dyn-drv/old-daemon-error-hack.sh new file mode 100644 index 000000000..43b049973 --- /dev/null +++ b/tests/dyn-drv/old-daemon-error-hack.sh @@ -0,0 +1,11 @@ +# Purposely bypassing our usual common for this subgroup +source ../common.sh + +# Need backend to support text-hashing too +isDaemonNewer "2.18.0pre20230906" && skipTest "Daemon is too new" + +enableFeatures "ca-derivations dynamic-derivations" + +restartDaemon + +expectStderr 1 nix-instantiate --read-write-mode ./old-daemon-error-hack.nix | grepQuiet "the daemon is too old to understand dependencies on dynamic derivations" From 5473e10249e02f95375b6126e232700d51bdf429 Mon Sep 17 00:00:00 2001 From: thenbe <33713262+thenbe@users.noreply.github.com> Date: Thu, 7 Sep 2023 22:25:32 +0000 Subject: [PATCH 251/909] fix: `nix shell` multiple commands example (#8950) The `-c` flag belongs to `sh` not `nix shell`. As it stands, the command errors with: ``` $ nix shell nixpkgs#gnumake --command sh --command "cd src && make" sh: --command: invalid option ``` https://github.com/NixOS/nix/pull/8276 was good for readability, but it missed this since that PR used a find/replace script. --- src/nix/shell.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/shell.md b/src/nix/shell.md index 1668104b1..f36919575 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -26,7 +26,7 @@ R""( * Run multiple commands in a shell environment: ```console - # nix shell nixpkgs#gnumake --command sh --command "cd src && make" + # nix shell nixpkgs#gnumake --command sh -c "cd src && make" ``` * Run GNU Hello in a chroot store: From 2cdc9c32e746a620458a1dd87655d2a6c3800f64 Mon Sep 17 00:00:00 2001 From: Emil Nikolov Date: Sat, 9 Sep 2023 08:54:39 +0200 Subject: [PATCH 252/909] docs: fixed the default priority of nix-env --install (#8945) --- doc/manual/src/command-ref/nix-env/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-env/install.md b/doc/manual/src/command-ref/nix-env/install.md index ad179cbc7..5dc04c385 100644 --- a/doc/manual/src/command-ref/nix-env/install.md +++ b/doc/manual/src/command-ref/nix-env/install.md @@ -30,7 +30,7 @@ a number of possible ways: derivation with the highest *priority* is used. A derivation can define a priority by declaring the `meta.priority` attribute. This attribute should be a number, with a higher value denoting a lower - priority. The default priority is `0`. + priority. The default priority is `5`. If there are multiple matching derivations with the same priority, then the derivation with the highest version will be installed. From 82ddb130984c7bdc45cdffc14e81bed720089200 Mon Sep 17 00:00:00 2001 From: Graham Bennett Date: Fri, 29 Apr 2022 15:15:25 -0400 Subject: [PATCH 253/909] Unlock output paths when a derivation is already built Without this change, nix build processes will not drop the locks for derivation goals which have already been built by another process when the current process gets round to building them. This means the locks are held until the process terminates. If there are other nix build processes in a similar state, they will also try to acquire the same locks when they try to build the same derivation, and so will wait until the lock holder terminates (which might be a very long time if it has a lot to build). In some pathological cases, those processes might be holding their own locks on other derivations due to the same issue, and this can lead to deadlock. Resolves #6468 --- src/libstore/build/derivation-goal.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 6472ecd99..befbfd10e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1456,6 +1456,7 @@ void DerivationGoal::done( SingleDrvOutputs builtOutputs, std::optional ex) { + outputLocks.unlock(); buildResult.status = status; if (ex) buildResult.errorMsg = fmt("%s", normaltxt(ex->info().msg)); From 682dbcab9a7a81c210dc9c8aa3cfc58ccc1723aa Mon Sep 17 00:00:00 2001 From: maralorn Date: Sat, 9 Sep 2023 18:01:10 +0200 Subject: [PATCH 254/909] Print parent activity field in json log --- src/libutil/logging.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 54702e4ea..9d7a141b3 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -220,8 +220,8 @@ struct JSONLogger : Logger { json["level"] = lvl; json["type"] = type; json["text"] = s; + json["parent"] = parent; addFields(json, fields); - // FIXME: handle parent write(json); } From 07545add53ab39b4560d8e7cec1748423ae1a22e Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Sun, 10 Sep 2023 12:18:03 +0200 Subject: [PATCH 255/909] Drop dead code localPath is unused --- src/libexpr/flake/flake.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 6a27ea2e8..90427e064 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -520,11 +520,6 @@ LockedFlake lockFlake( } } - auto localPath(parentPath); - // If this input is a path, recurse it down. - // This allows us to resolve path inputs relative to the current flake. - if ((*input.ref).input.getType() == "path") - localPath = absPath(*input.ref->input.getSourcePath(), parentPath); computeLocks( mustRefetch ? getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath).inputs From 3720e811fa2883bca1a2698543146070e4d8d32c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 12 Sep 2023 13:21:55 +0200 Subject: [PATCH 256/909] libexpr: Add nrExprs to NIX_SHOW_STATS --- src/libexpr/eval.cc | 1 + src/libexpr/nixexpr.hh | 4 ++++ src/libexpr/parser.y | 1 + 3 files changed, 6 insertions(+) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 6d445fd96..afa864730 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2506,6 +2506,7 @@ void EvalState::printStats() {"elements", nrValuesInEnvs}, {"bytes", bEnvs}, }; + topObj["nrExprs"] = Expr::nrExprs; topObj["list"] = { {"elements", nrListElems}, {"bytes", bLists}, diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 5ca3d1fa6..b80c07f46 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -155,6 +155,10 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) struct Expr { + static unsigned long nrExprs; + inline Expr() { + nrExprs++; + } virtual ~Expr() { }; virtual void show(const SymbolTable & symbols, std::ostream & str) const; virtual void bindVars(EvalState & es, const std::shared_ptr & env); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 792f51fde..a3d3c7041 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -653,6 +653,7 @@ formal namespace nix { +unsigned long Expr::nrExprs = 0; Expr * EvalState::parse( char * text, From bf8deb4991286a44e1b6cc2142721a594276dd12 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 12 Sep 2023 13:45:45 +0200 Subject: [PATCH 257/909] Expr: remove redundant int and float fields --- src/libexpr/nixexpr.cc | 4 ++-- src/libexpr/nixexpr.hh | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 4566a1388..22be8e68c 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -76,12 +76,12 @@ void Expr::show(const SymbolTable & symbols, std::ostream & str) const void ExprInt::show(const SymbolTable & symbols, std::ostream & str) const { - str << n; + str << v.integer; } void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const { - str << nf; + str << v.fpoint; } void ExprString::show(const SymbolTable & symbols, std::ostream & str) const diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index b80c07f46..17b2d1136 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -175,18 +175,16 @@ struct Expr struct ExprInt : Expr { - NixInt n; Value v; - ExprInt(NixInt n) : n(n) { v.mkInt(n); }; + ExprInt(NixInt n) { v.mkInt(n); }; Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; struct ExprFloat : Expr { - NixFloat nf; Value v; - ExprFloat(NixFloat nf) : nf(nf) { v.mkFloat(nf); }; + ExprFloat(NixFloat nf) { v.mkFloat(nf); }; Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; From 21783cff1649f236cb31f27f788e3934802c42c9 Mon Sep 17 00:00:00 2001 From: Emil Nikolov Date: Tue, 12 Sep 2023 17:15:36 +0200 Subject: [PATCH 258/909] docs: make the nix develop --command example unambiguous (#8952) --- src/nix/develop.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/develop.md b/src/nix/develop.md index 1b5a8aeba..c49b39669 100644 --- a/src/nix/develop.md +++ b/src/nix/develop.md @@ -69,7 +69,7 @@ R""( * Run a series of script commands: ```console - # nix develop --command bash --command "mkdir build && cmake .. && make" + # nix develop --command bash -c "mkdir build && cmake .. && make" ``` # Description From 408055a9ddf855e32a59e07e564ad893b2cb2cdf Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 18 Sep 2023 20:08:48 +0200 Subject: [PATCH 259/909] add clarifying example to `nix-env` output selection there is a very confusing warning in the Nixpkgs manual that mischaracterises `nix-env` behavior, and this example shows what's really happening. note that it doesn't use `pkgs.runCommand` or other `pkgs.stdenv` facilities, as deep down those set `meta.outputsToInstall` to very particular defaults that do not generally apply to Nix. --- doc/manual/src/command-ref/nix-env/install.md | 60 ++++++++++++++++++- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/command-ref/nix-env/install.md b/doc/manual/src/command-ref/nix-env/install.md index 5dc04c385..95648e9e0 100644 --- a/doc/manual/src/command-ref/nix-env/install.md +++ b/doc/manual/src/command-ref/nix-env/install.md @@ -19,12 +19,15 @@ current generation of the active profile, to which a set of store paths described by *args* is added. The arguments *args* map to store paths in a number of possible ways: - - By default, *args* is a set of derivation names denoting derivations + + - By default, *args* is a set of [derivation] names denoting derivations in the active Nix expression. These are realised, and the resulting output paths are installed. Currently installed derivations with a name equal to the name of a derivation being added are removed unless the option `--preserve-installed` is specified. + [derivation]: @docroot@/language/derivations.md + If there are multiple derivations matching a name in *args* that have the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the derivation with the highest *priority* is used. A derivation can @@ -66,8 +69,59 @@ a number of possible ways: - If *args* are store paths that are not store derivations, then these are [realised](@docroot@/command-ref/nix-store/realise.md) and installed. - - By default all outputs are installed for each derivation. That can - be reduced by setting `meta.outputsToInstall`. + - By default all outputs are installed for each derivation. + This can be overridden by adding a `meta.outputsToInstall` attribute on the derivation listing a subset of the output names. + + + + Example: + + The file `example.nix` defines a [derivation] with two outputs `foo` and `bar`, each containing a file. + + ```nix + # example.nix + let + pkgs = import {}; + command = '' + ${pkgs.coreutils}/bin/mkdir -p $foo $bar + echo foo > $foo/foo-file + echo bar > $bar/bar-file + ''; + in + derivation { + name = "example"; + builder = "${pkgs.bash}/bin/bash"; + args = [ "-c" command ]; + outputs = [ "foo" "bar" ]; + system = builtins.currentSystem; + } + ``` + + Installing from this Nix expression will make files from both outputs appear in the current profile. + + ```console + $ nix-env --install --file example.nix + installing 'example' + $ ls ~/.nix-profile + foo-file + bar-file + manifest.nix + ``` + + Adding `meta.outputsToInstall` to that derivation will make `nix-env` only install files from the specified outputs. + + ```nix + # example-outputs.nix + import ./example.nix // { meta.outputsToInstall = [ "bar" ]; } + ``` + + ```console + $ nix-env --install --file example-outputs.nix + installing 'example' + $ ls ~/.nix-profile + bar-file + manifest.nix + ``` # Flags From dd3bf4dbda02b83a185efc1f8f4bced06a6b07ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 22:16:55 +0000 Subject: [PATCH 260/909] Bump docker/login-action from 2 to 3 Bumps [docker/login-action](https://github.com/docker/login-action) from 2 to 3. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 515548a4e..0c9c24dad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,7 +127,7 @@ jobs: - run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION - run: docker tag nix:$NIX_VERSION nixos/nix:master - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} From c8afa01bc25ba81883f8007ef83532ca550c731d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 19 Sep 2023 14:51:50 +0200 Subject: [PATCH 261/909] Try aws-sdk-cpp fix --- flake.lock | 12 ++++++------ flake.nix | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/flake.lock b/flake.lock index 2bc503258..75b6ae6a7 100644 --- a/flake.lock +++ b/flake.lock @@ -34,16 +34,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1693494129, - "narHash": "sha256-YrHlSbniFmhcz0ORe8MMFttifKR4hTRzyX2OQUO9VxA=", - "owner": "NixOS", + "lastModified": 1695124524, + "narHash": "sha256-trXDytVCqf3KryQQQrHOZKUabu1/lB8/ndOAuZKQrOE=", + "owner": "edolstra", "repo": "nixpkgs", - "rev": "22a584b861ab31a2dca52121999079832f0e0f73", + "rev": "a3d30b525535e3158221abc1a957ce798ab159fe", "type": "github" }, "original": { - "owner": "NixOS", - "ref": "nixos-23.05-small", + "owner": "edolstra", + "ref": "fix-aws-sdk-cpp", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 49aa9c4ce..249da2a04 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,8 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; + #inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; + inputs.nixpkgs.url = "github:edolstra/nixpkgs/fix-aws-sdk-cpp"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; From 126e2645f2a060197655f9a699ffa4dc3a464bdd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 19 Sep 2023 16:04:00 +0200 Subject: [PATCH 262/909] Disable rapidcheck tests in the coverage run https://hydra.nixos.org/build/233688539 --- flake.nix | 2 ++ src/libexpr/tests/derived-path.cc | 4 ++++ src/libexpr/tests/value/context.cc | 4 ++++ src/libstore/tests/derived-path.cc | 4 ++++ src/libstore/tests/outputs-spec.cc | 4 ++++ src/libstore/tests/path.cc | 4 ++++ 6 files changed, 22 insertions(+) diff --git a/flake.nix b/flake.nix index 249da2a04..dfb2dc1f8 100644 --- a/flake.nix +++ b/flake.nix @@ -588,6 +588,8 @@ lcovFilter = [ "*/boost/*" "*-tab.*" ]; hardeningDisable = ["fortify"]; + + NIX_CFLAGS_COMPILE = "-DCOVERAGE=1"; }; # API docs for Nix's unstable internal C++ interfaces. diff --git a/src/libexpr/tests/derived-path.cc b/src/libexpr/tests/derived-path.cc index d01735db8..d5fc6f201 100644 --- a/src/libexpr/tests/derived-path.cc +++ b/src/libexpr/tests/derived-path.cc @@ -18,6 +18,8 @@ TEST_F(DerivedPathExpressionTest, force_init) { } +#ifndef COVERAGE + RC_GTEST_FIXTURE_PROP( DerivedPathExpressionTest, prop_opaque_path_round_trip, @@ -61,4 +63,6 @@ RC_GTEST_FIXTURE_PROP( RC_ASSERT(SingleDerivedPath { b } == d); } +#endif + } /* namespace nix */ diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index 19407d071..92d4889ab 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -147,6 +147,8 @@ Gen Arbitrary::arbitrary() namespace nix { +#ifndef COVERAGE + RC_GTEST_PROP( NixStringContextElemTest, prop_round_rip, @@ -155,4 +157,6 @@ RC_GTEST_PROP( RC_ASSERT(o == NixStringContextElem::parse(o.to_string())); } +#endif + } diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc index d6549f66f..3fa3c0801 100644 --- a/src/libstore/tests/derived-path.cc +++ b/src/libstore/tests/derived-path.cc @@ -130,6 +130,8 @@ TEST_F(DerivedPathTest, built_built_xp) { MissingExperimentalFeature); } +#ifndef COVERAGE + RC_GTEST_FIXTURE_PROP( DerivedPathTest, prop_legacy_round_rip, @@ -146,4 +148,6 @@ RC_GTEST_FIXTURE_PROP( RC_ASSERT(o == DerivedPath::parse(*store, o.to_string(*store))); } +#endif + } diff --git a/src/libstore/tests/outputs-spec.cc b/src/libstore/tests/outputs-spec.cc index bf8deaa9d..952945185 100644 --- a/src/libstore/tests/outputs-spec.cc +++ b/src/libstore/tests/outputs-spec.cc @@ -224,6 +224,8 @@ Gen Arbitrary::arbitrary() namespace nix { +#ifndef COVERAGE + RC_GTEST_PROP( OutputsSpec, prop_round_rip, @@ -232,4 +234,6 @@ RC_GTEST_PROP( RC_ASSERT(o == OutputsSpec::parse(o.to_string())); } +#endif + } diff --git a/src/libstore/tests/path.cc b/src/libstore/tests/path.cc index 430aa0099..efa35ef2b 100644 --- a/src/libstore/tests/path.cc +++ b/src/libstore/tests/path.cc @@ -134,6 +134,8 @@ Gen Arbitrary::arbitrary() namespace nix { +#ifndef COVERAGE + RC_GTEST_FIXTURE_PROP( StorePathTest, prop_regex_accept, @@ -150,4 +152,6 @@ RC_GTEST_FIXTURE_PROP( RC_ASSERT(p == store->parseStorePath(store->printStorePath(p))); } +#endif + } From c6953d1ff62fb6dc4fbd89c03e7949c552c19382 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 19 Sep 2023 17:03:21 +0200 Subject: [PATCH 263/909] Disable systemd-nspawn test This is broken because of a change in systemd in NixOS 23.05. It fails with Failed to mount proc (type proc) on /proc (MS_NOSUID|MS_NODEV|MS_NOEXEC ""): Operation not permitted --- tests/nixos/containers/containers.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/nixos/containers/containers.nix b/tests/nixos/containers/containers.nix index c8ee78a4a..e721be48f 100644 --- a/tests/nixos/containers/containers.nix +++ b/tests/nixos/containers/containers.nix @@ -56,8 +56,8 @@ host.fail("nix build -v --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-6 --arg uidRange true") # Run systemd-nspawn in a Nix build. - host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./systemd-nspawn.nix} --argstr nixpkgs ${nixpkgs}") - host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]") + #host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./systemd-nspawn.nix} --argstr nixpkgs ${nixpkgs}") + #host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]") ''; } From b6b2a0aea99995d73f0faa6115fb33a137e57b23 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 19 Sep 2023 17:21:07 +0200 Subject: [PATCH 264/909] Use "touch -h" https://hydra.nixos.org/build/235888160 This is needed because Nixpkgs now contains dangling symlinks (pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/pkgs/by-name/fo/foo/foo.nix). --- tests/nixos/github-flakes.nix | 2 +- tests/nixos/sourcehut-flakes.nix | 2 +- tests/nixos/tarball-flakes.nix | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index e4d347691..6de702d17 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -82,7 +82,7 @@ let dir=NixOS-nixpkgs-${nixpkgs.shortRev} cp -prd ${nixpkgs} $dir # Set the correct timestamp in the tarball. - find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- + find $dir -print0 | xargs -0 touch -h -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- tar cfz $out/archive/${nixpkgs.rev}.tar.gz $dir --hard-dereference ''; in diff --git a/tests/nixos/sourcehut-flakes.nix b/tests/nixos/sourcehut-flakes.nix index a76fed020..6e8d884a0 100644 --- a/tests/nixos/sourcehut-flakes.nix +++ b/tests/nixos/sourcehut-flakes.nix @@ -47,7 +47,7 @@ let cp -prd ${nixpkgs} $dir # Set the correct timestamp in the tarball. - find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- + find $dir -print0 | xargs -0 touch -h -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- mkdir -p $out/archive tar cfz $out/archive/${nixpkgs.rev}.tar.gz $dir --hard-dereference diff --git a/tests/nixos/tarball-flakes.nix b/tests/nixos/tarball-flakes.nix index 1d43a5d04..e30d15739 100644 --- a/tests/nixos/tarball-flakes.nix +++ b/tests/nixos/tarball-flakes.nix @@ -11,7 +11,7 @@ let dir=nixpkgs-${nixpkgs.shortRev} cp -prd ${nixpkgs} $dir # Set the correct timestamp in the tarball. - find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- + find $dir -print0 | xargs -0 touch -h -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- tar cfz $out/stable/${nixpkgs.rev}.tar.gz $dir --hard-dereference echo 'Redirect "/latest.tar.gz" "/stable/${nixpkgs.rev}.tar.gz"' > $out/.htaccess From 10ad052f7d14101525daabbd63a147eb0247c3a1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Sep 2023 11:42:49 +0200 Subject: [PATCH 265/909] Release notes --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/release-notes/rl-2.18.md | 28 ++++++++++++++++++++++++ doc/manual/src/release-notes/rl-next.md | 29 ------------------------- 3 files changed, 29 insertions(+), 29 deletions(-) create mode 100644 doc/manual/src/release-notes/rl-2.18.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 701a6ea69..c23186511 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -110,6 +110,7 @@ - [C++ style guide](contributing/cxx.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md) - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md) diff --git a/doc/manual/src/release-notes/rl-2.18.md b/doc/manual/src/release-notes/rl-2.18.md new file mode 100644 index 000000000..4bbc52b50 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.18.md @@ -0,0 +1,28 @@ +# Release 2.18 (2023-09-20) + +- Two new builtin functions, + [`builtins.parseFlakeRef`](@docroot@/language/builtins.md#builtins-parseFlakeRef) + and + [`builtins.flakeRefToString`](@docroot@/language/builtins.md#builtins-flakeRefToString), + have been added. + These functions are useful for converting between flake references encoded as attribute sets and URLs. + +- [`builtins.toJSON`](@docroot@/language/builtins.md#builtins-parseFlakeRef) now prints [--show-trace](@docroot@/command-ref/conf-file.html#conf-show-trace) items for the path in which it finds an evaluation error. + +- Error messages regarding malformed input to [`nix derivation add`](@docroot@/command-ref/new-cli/nix3-derivation-add.md) are now clearer and more detailed. + +- The `discard-references` feature has been stabilized. + This means that the + [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) + attribute is no longer guarded by an experimental flag and can be used + freely. + +- The JSON output for derived paths which are store paths is now a string, not an object with a single `path` field. + This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. + +- A new builtin [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) has been added. + It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + +- Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. + +- [`nix-store --query`](@docroot@/command-ref/nix-store/query.md) gained a new type of query: `--valid-derivers`. It returns all `.drv` files in the local store that *can be* used to build the output passed in argument. This is in contrast to `--deriver`, which returns the single `.drv` file that *was actually* used to build the output passed in argument. In case the output was substituted from a binary cache, this `.drv` file may only exist on said binary cache and not locally. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 02ad0a661..c869b5e2f 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,30 +1 @@ # Release X.Y (202?-??-??) - -- Two new builtin functions, - [`builtins.parseFlakeRef`](@docroot@/language/builtins.md#builtins-parseFlakeRef) - and - [`builtins.flakeRefToString`](@docroot@/language/builtins.md#builtins-flakeRefToString), - have been added. - These functions are useful for converting between flake references encoded as attribute sets and URLs. - -- [`builtins.toJSON`](@docroot@/language/builtins.md#builtins-parseFlakeRef) now prints [--show-trace](@docroot@/command-ref/conf-file.html#conf-show-trace) items for the path in which it finds an evaluation error. - -- Error messages regarding malformed input to [`derivation add`](@docroot@/command-ref/new-cli/nix3-derivation-add.md) are now clearer and more detailed. - -- The `discard-references` feature has been stabilized. - This means that the - [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) - attribute is no longer guarded by an experimental flag and can be used - freely. - -- The JSON output for derived paths with are store paths is now a string, not an object with a single `path` field. - This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. - -- Introduce a new [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) builtin. - It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. - -- Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. - -- [`nix-store --query`](@docroot@/command-ref/nix-store/query.md) gained a new type of query: `--valid-derivers`. It returns all `.drv` files in the local store that *can be* used to build the output passed in argument. -This is in contrast to `--deriver`, which returns the single `.drv` file that *was actually* used to build the output passed in argument. In case the output was substituted from a binary cache, -this `.drv` file may only exist on said binary cache and not locally. From e44d2a6bbef113b3d8f162f75bd5c94e0101075f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 30 Aug 2023 22:57:59 -0400 Subject: [PATCH 266/909] Add FreeBSD and NetBSD cross to Nix's flake --- flake.nix | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index dfb2dc1f8..871f0d7ba 100644 --- a/flake.nix +++ b/flake.nix @@ -25,8 +25,11 @@ linuxSystems = linux32BitSystems ++ linux64BitSystems; darwinSystems = [ "x86_64-darwin" "aarch64-darwin" ]; systems = linuxSystems ++ darwinSystems; - - crossSystems = [ "armv6l-linux" "armv7l-linux" ]; + + crossSystems = [ + "armv6l-linux" "armv7l-linux" + "x86_64-freebsd13" "x86_64-netbsd" + ]; stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ]; @@ -94,7 +97,14 @@ nixpkgsFor = forAllSystems (system: let make-pkgs = crossSystem: stdenv: import nixpkgs { - inherit system crossSystem; + localSystem = { + inherit system; + }; + crossSystem = if crossSystem == null then null else { + system = crossSystem; + } // lib.optionalAttrs (crossSystem == "x86_64-freebsd13") { + useLLVM = true; + }; overlays = [ (overlayFor (p: p.${stdenv})) ]; From 28850ee90095e337bf47fbe03a0d937cb98784e2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 30 Aug 2023 23:27:53 -0400 Subject: [PATCH 267/909] Make dev shells work for cross Need to get tools from right package set. Could build clang tools but I don't want to wait :D. --- flake.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 871f0d7ba..24d9520ea 100644 --- a/flake.nix +++ b/flake.nix @@ -750,7 +750,11 @@ outputs = [ "out" "dev" "doc" ]; nativeBuildInputs = nativeBuildDeps - ++ (lib.optionals stdenv.cc.isClang [ pkgs.bear pkgs.clang-tools ]); + ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear + ++ lib.optional + (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) + pkgs.buildPackages.clang-tools + ; buildInputs = buildDeps ++ propagatedDeps ++ awsDeps ++ checkDeps ++ internalApiDocsDeps; From 564392b57bc44c407303e4eaf6138d61a8ea50b0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 30 Aug 2023 23:29:17 -0400 Subject: [PATCH 268/909] Make libsodium an unconditional dependency The configure script will not tolerate it being missing. --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 24d9520ea..50a226347 100644 --- a/flake.nix +++ b/flake.nix @@ -190,9 +190,9 @@ libarchive boost lowdown-nix + libsodium ] ++ lib.optionals stdenv.isLinux [libseccomp] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; checkDeps = [ From 0db251e4ad07338375fd59134fb467e9ebc4176a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 30 Aug 2023 23:43:48 -0400 Subject: [PATCH 269/909] Do not build docs in cross devShell Coppied from the main build; we really should deduplicate this more. --- flake.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 50a226347..36937057d 100644 --- a/flake.nix +++ b/flake.nix @@ -743,6 +743,9 @@ devShells = let makeShell = pkgs: stdenv: + let + canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + in with commonDeps { inherit pkgs; }; stdenv.mkDerivation { name = "nix"; @@ -760,7 +763,8 @@ ++ awsDeps ++ checkDeps ++ internalApiDocsDeps; configureFlags = configureFlags - ++ testConfigureFlags ++ internalApiDocsConfigureFlags; + ++ testConfigureFlags ++ internalApiDocsConfigureFlags + ++ lib.optional (!canRunInstalled) "--disable-doc-gen"; enableParallelBuilding = true; From 7f76d7f038fbb5cb7982cf9aa951b9730566e275 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 30 Aug 2023 23:44:23 -0400 Subject: [PATCH 270/909] Rename an identifier of ours called `stdout` This is a reserved identifier on NetBSD --- it is replaced by a macro on that platform --- and so we cannot use it. --- src/libmain/shared.cc | 6 +++--- src/libmain/shared.hh | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 56f47a4ac..9c2ad039a 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -379,9 +379,9 @@ RunPager::RunPager() }); pid.setKillSignal(SIGINT); - stdout = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0); + std_out = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0); if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); + throw SysError("dupping standard output"); } @@ -390,7 +390,7 @@ RunPager::~RunPager() try { if (pid != -1) { std::cout.flush(); - dup2(stdout, STDOUT_FILENO); + dup2(std_out, STDOUT_FILENO); pid.wait(); } } catch (...) { diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 7a9e83c6c..9415be78a 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -85,8 +85,9 @@ struct LegacyArgs : public MixCommonArgs void showManPage(const std::string & name); /** - * The constructor of this class starts a pager if stdout is a - * terminal and $PAGER is set. Stdout is redirected to the pager. + * The constructor of this class starts a pager if standard output is a + * terminal and $PAGER is set. Standard output is redirected to the + * pager. */ class RunPager { @@ -96,7 +97,7 @@ public: private: Pid pid; - int stdout; + int std_out; }; extern volatile ::sig_atomic_t blockInt; From c18911602eb4260d59acf8c17f1c3b4c7fcf7cee Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 31 Aug 2023 00:07:49 -0400 Subject: [PATCH 271/909] Fix `boehmgc-coroutine-sp-fallback.diff` for FreeBSD Our FreeBSD headers have `pthread_getattr_np`, but we get a link-time error that is missing. The good news is that there is another similar function which does exist, and the upstream project elsewhere does just the [fallback code] we need. As the fallback code indicates, the two functions are not identical however as the other one needs explicit initialization. NetBSD supports both in fact, and its [manpage] is therefore a good resource on what the differences are. [fallback code]: https://github.com/ivmai/bdwgc/blob/07a6d0ee8889bca5eaeadc13cabadc363725d216/os_dep.c#L1266-L1272 [manpage]: https://man.netbsd.org/pthread_attr_get_np.3 --- boehmgc-coroutine-sp-fallback.diff | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff index 5066d8278..2afbe9671 100644 --- a/boehmgc-coroutine-sp-fallback.diff +++ b/boehmgc-coroutine-sp-fallback.diff @@ -59,12 +59,18 @@ index b5d71e62..aed7b0bf 100644 GC_bool found_me = FALSE; size_t nthreads = 0; int i; -@@ -851,6 +853,31 @@ GC_INNER void GC_push_all_stacks(void) +@@ -851,6 +853,37 @@ GC_INNER void GC_push_all_stacks(void) hi = p->altstack + p->altstack_size; /* FIXME: Need to scan the normal stack too, but how ? */ /* FIXME: Assume stack grows down */ + } else { -+ if (pthread_getattr_np(p->id, &pattr)) { ++#ifdef HAVE_PTHREAD_ATTR_GET_NP ++ if (!pthread_attr_init(&pattr) ++ || !pthread_attr_get_np(p->id, &pattr)) ++#else /* HAVE_PTHREAD_GETATTR_NP */ ++ if (pthread_getattr_np(p->id, &pattr)) ++#endif ++ { + ABORT("GC_push_all_stacks: pthread_getattr_np failed!"); + } + if (pthread_attr_getstacksize(&pattr, &stack_limit)) { From b7acef1ceb756c72462a896d90c1b17ddc32a6df Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Sep 2023 15:21:21 +0200 Subject: [PATCH 272/909] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index cf8690732..ef0f38abe 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.18.0 +2.19.0 From 883092e3f78d4efb1066a2e24e343b307035a04c Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Wed, 20 Sep 2023 09:09:01 -0700 Subject: [PATCH 273/909] Re-enable systemd-nspawn test It was disabled in c6953d1ff62fb6dc4fbd89c03e7949c552c19382 because a recent Nixpkgs bump brought in a new systemd which changed how systemd-nspawn worked. As far as I can tell, the issue was caused by this upstream systemd commit: https://github.com/systemd/systemd/commit/b71a0192c040f585397cfc6fc2ca025bf839733d Bind-mounting the host's `/sys` and `/proc` into the container's `/run/host/{sys,proc}` fixes the issue and allows the test to succeed. --- tests/nixos/containers/containers.nix | 4 ++-- tests/nixos/containers/systemd-nspawn.nix | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/nixos/containers/containers.nix b/tests/nixos/containers/containers.nix index e721be48f..c8ee78a4a 100644 --- a/tests/nixos/containers/containers.nix +++ b/tests/nixos/containers/containers.nix @@ -56,8 +56,8 @@ host.fail("nix build -v --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-6 --arg uidRange true") # Run systemd-nspawn in a Nix build. - #host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./systemd-nspawn.nix} --argstr nixpkgs ${nixpkgs}") - #host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]") + host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./systemd-nspawn.nix} --argstr nixpkgs ${nixpkgs}") + host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]") ''; } diff --git a/tests/nixos/containers/systemd-nspawn.nix b/tests/nixos/containers/systemd-nspawn.nix index f54f32f2a..1dad4ebd7 100644 --- a/tests/nixos/containers/systemd-nspawn.nix +++ b/tests/nixos/containers/systemd-nspawn.nix @@ -73,6 +73,8 @@ runCommand "test" --resolv-conf=off \ --bind-ro=/nix/store \ --bind=$out \ + --bind=/proc:/run/host/proc \ + --bind=/sys:/run/host/sys \ --private-network \ $toplevel/init '' From e0e5943db28271cd6f1dcf56f3a0b12d0bc37b6c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:51:39 +0200 Subject: [PATCH 274/909] remove IRC from links in README the community has moved away from IRC a long time ago --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 85b0902b1..46a4218d9 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,6 @@ to set up a development environment and build Nix from source. - [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) - [NixOS Discourse](https://discourse.nixos.org/) - [Matrix - #nix:nixos.org](https://matrix.to/#/#nix:nixos.org) -- [IRC - #nixos on libera.chat](irc://irc.libera.chat/#nixos) ## License From 747b2baf21f5bc3924945b02d10cd37b17e2e1bb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:51:54 +0200 Subject: [PATCH 275/909] fix rendering error for consecutive spaces --- doc/manual/src/language/index.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index 29950a52d..a26e43a05 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -83,7 +83,8 @@ This is an incomplete overview of language features, by example. - A multi-line string. Strips common prefixed whitespace. Evaluates to `"multi\n line\n string"`. + + A multi-line string. Strips common prefixed whitespace. Evaluates to `"multi\n line\n  string"`. From 954890a42f143896d5892066f7b5a0f2b120fbfe Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:52:05 +0200 Subject: [PATCH 276/909] add information on release cycle and backports --- doc/manual/src/release-notes/release-notes.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/manual/src/release-notes/release-notes.md b/doc/manual/src/release-notes/release-notes.md index b05d5ee0a..dbda0babd 100644 --- a/doc/manual/src/release-notes/release-notes.md +++ b/doc/manual/src/release-notes/release-notes.md @@ -1 +1,7 @@ # Nix Release Notes + +Nix has a release cycle of roughly 6 weeks. +Notable changes and additions are announced in the release notes for each version. + +Bugfixes can be backported on request to the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). + From 02649d247b7336697f059186ea8e9f3076dcda0a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:52:28 +0200 Subject: [PATCH 277/909] move test coverage section to testing page --- doc/manual/src/contributing/hacking.md | 14 -------------- doc/manual/src/contributing/testing.md | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 4b0a3a3e5..22f629c6f 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -271,17 +271,3 @@ or inside a `nix develop` shell by running: # make internal-api-html # xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html ``` - -## Coverage analysis - -A coverage analysis report is [available -online](https://hydra.nixos.org/job/nix/master/coverage/latest/download-by-type/report/coverage). You -can build it yourself: - -``` -# nix build .#hydraJobs.coverage -# xdg-open ./result/coverage/index.html -``` - -Metrics about the change in line/function coverage over time are also -[available](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts). diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index cd94d5cfb..881118505 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -1,5 +1,19 @@ # Running tests +## Coverage analysis + +A [coverage analysis report] is available online +You can build it yourself: + +[coverage analysis report]: https://hydra.nixos.org/job/nix/master/coverage/latest/download-by-type/report/coverage + +``` +# nix build .#hydraJobs.coverage +# xdg-open ./result/coverage/index.html +``` + +[Extensive records of build metrics](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts), such as test coverage over time, are also available online. + ## Unit-tests The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined From 4685260a77266382b5c2c2b68ff1e6e0ddc8a908 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:52:58 +0200 Subject: [PATCH 278/909] fix links to configuration settings --- src/libutil/experimental-features.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 782331283..211374bf6 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -171,7 +171,7 @@ constexpr std::array xpFeatureDetails = {{ .name = "auto-allocate-uids", .description = R"( Allows Nix to automatically pick UIDs for builds, rather than creating - `nixbld*` user accounts. See the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting for details. + `nixbld*` user accounts. See the [`auto-allocate-uids`](@docroot@/command-ref/conf-file.md#conf-auto-allocate-uids) setting for details. )", }, { @@ -179,7 +179,7 @@ constexpr std::array xpFeatureDetails = {{ .name = "cgroups", .description = R"( Allows Nix to execute builds inside cgroups. See - the [`use-cgroups`](#conf-use-cgroups) setting for details. + the [`use-cgroups`](@docroot@/command-ref/conf-file.md#conf-use-cgroups) setting for details. )", }, { From 8e25450ff4bbf2b2c6951ebc78fd1bdec7097660 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:53:11 +0200 Subject: [PATCH 279/909] refer to nix.dev for installation instructions there are currently multiple places with installation instructions that all have to be updated when a change to any of them is accepted. this reduces the number of places by one, and directs beginners to the maintained and curated resource for Nix learning materials. --- README.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 85b0902b1..1440239bc 100644 --- a/README.md +++ b/README.md @@ -7,16 +7,11 @@ Nix is a powerful package manager for Linux and other Unix systems that makes pa management reliable and reproducible. Please refer to the [Nix manual](https://nixos.org/nix/manual) for more details. -## Installation +## Installation and first steps -On Linux and macOS the easiest way to install Nix is to run the following shell command -(as a user other than root): +Visit [nix.dev](https://nix.dev) for installation instructions and beginner tutorials. -```console -$ curl -L https://nixos.org/nix/install | sh -``` - -Information on additional installation methods is available on the [Nix download page](https://nixos.org/download.html). +Full reference documentation can be found in the [Nix manual](https://nixos.org/nix/manual). ## Building And Developing From 984bd4cb0e2973ad756b6b1692919980f14f5df2 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:53:36 +0200 Subject: [PATCH 280/909] README: link to CONTRIBUTING --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 85b0902b1..3b31b4d93 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,10 @@ Information on additional installation methods is available on the [Nix download See our [Hacking guide](https://nixos.org/manual/nix/unstable/contributing/hacking.html) in our manual for instruction on how to to set up a development environment and build Nix from source. +## Contributing + +Check the [contributing guide](./CONTRIBUTING.md) if you want to get involved with developing Nix. + ## Additional Resources - [Nix manual](https://nixos.org/nix/manual) From cf6ba7256f849ec1fe209b18c311e61ecd7719d3 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:57:00 +0200 Subject: [PATCH 281/909] delete unused files --- doc/manual/src/command-ref/opt-common-syn.md | 57 -------------------- doc/manual/src/command-ref/opt-inst-syn.md | 15 ------ 2 files changed, 72 deletions(-) delete mode 100644 doc/manual/src/command-ref/opt-common-syn.md delete mode 100644 doc/manual/src/command-ref/opt-inst-syn.md diff --git a/doc/manual/src/command-ref/opt-common-syn.md b/doc/manual/src/command-ref/opt-common-syn.md deleted file mode 100644 index b66d318c2..000000000 --- a/doc/manual/src/command-ref/opt-common-syn.md +++ /dev/null @@ -1,57 +0,0 @@ -\--help - -\--version - -\--verbose - -\-v - -\--quiet - -\--log-format - -format - -\--no-build-output - -\-Q - -\--max-jobs - -\-j - -number - -\--cores - -number - -\--max-silent-time - -number - -\--timeout - -number - -\--keep-going - -\-k - -\--keep-failed - -\-K - -\--fallback - -\--readonly-mode - -\-I - -path - -\--option - -name - -value diff --git a/doc/manual/src/command-ref/opt-inst-syn.md b/doc/manual/src/command-ref/opt-inst-syn.md deleted file mode 100644 index 1703c40e3..000000000 --- a/doc/manual/src/command-ref/opt-inst-syn.md +++ /dev/null @@ -1,15 +0,0 @@ -\--prebuilt-only - -\-b - -\--attr - -\-A - -\--from-expression - -\-E - -\--from-profile - -path From 1a412a8d782b11b3af08be5b7ee3b4000b72cb32 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 11:38:13 +0200 Subject: [PATCH 282/909] fix typo in docstring --- src/libutil/config.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index cc8532587..96c2cd75d 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -36,8 +36,8 @@ namespace nix { * * std::map settings; * config.getSettings(settings); - * config["system"].description == "the current system" - * config["system"].value == "x86_64-linux" + * settings["system"].description == "the current system" + * settings["system"].value == "x86_64-linux" * * * The above retrieves all currently known settings from the `Config` object From 1b560ea502cf0308b829e4b3425307f80fc3528a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 23:23:14 +0200 Subject: [PATCH 283/909] more detail on backports Co-authored-by: John Ericson --- doc/manual/src/release-notes/release-notes.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/release-notes.md b/doc/manual/src/release-notes/release-notes.md index dbda0babd..1fa2f5580 100644 --- a/doc/manual/src/release-notes/release-notes.md +++ b/doc/manual/src/release-notes/release-notes.md @@ -3,5 +3,9 @@ Nix has a release cycle of roughly 6 weeks. Notable changes and additions are announced in the release notes for each version. -Bugfixes can be backported on request to the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). +Bugfixes can be backported on request to previous Nix releases. +We typically backport only as far back as the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). + +Backports never skip releases (if a feature is backported to `x.y`, it must also be available in `x.(y+1)`; +This ensures that upgrading from an older version with backports is still safe and no backported functionality will go missing. From f264d9ff0862277523a207c7afaed4894a40dc11 Mon Sep 17 00:00:00 2001 From: Gerg-L Date: Thu, 21 Sep 2023 21:00:35 -0400 Subject: [PATCH 284/909] flake: complete update to 23.05 --- flake.lock | 12 ++++++------ flake.nix | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/flake.lock b/flake.lock index 75b6ae6a7..56df9c3fb 100644 --- a/flake.lock +++ b/flake.lock @@ -34,16 +34,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1695124524, - "narHash": "sha256-trXDytVCqf3KryQQQrHOZKUabu1/lB8/ndOAuZKQrOE=", - "owner": "edolstra", + "lastModified": 1695283060, + "narHash": "sha256-CJz71xhCLlRkdFUSQEL0pIAAfcnWFXMzd9vXhPrnrEg=", + "owner": "NixOS", "repo": "nixpkgs", - "rev": "a3d30b525535e3158221abc1a957ce798ab159fe", + "rev": "31ed632c692e6a36cfc18083b88ece892f863ed4", "type": "github" }, "original": { - "owner": "edolstra", - "ref": "fix-aws-sdk-cpp", + "owner": "NixOS", + "ref": "nixos-23.05-small", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index dfb2dc1f8..a7680a759 100644 --- a/flake.nix +++ b/flake.nix @@ -1,8 +1,7 @@ { description = "The purely functional package manager"; - #inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; - inputs.nixpkgs.url = "github:edolstra/nixpkgs/fix-aws-sdk-cpp"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; From d8cebae939932f2a5b458b5d92c82cd1a025dc0a Mon Sep 17 00:00:00 2001 From: Rasmus Rendal Date: Sat, 4 Jun 2022 12:07:08 +0200 Subject: [PATCH 285/909] Add a test for flake paths with spaces in them --- tests/flakes/common.sh | 10 +- tests/flakes/flakes.sh | 302 +++++++++++++++++++++-------------------- 2 files changed, 157 insertions(+), 155 deletions(-) diff --git a/tests/flakes/common.sh b/tests/flakes/common.sh index 427abcdde..8aed296e6 100644 --- a/tests/flakes/common.sh +++ b/tests/flakes/common.sh @@ -61,10 +61,10 @@ createGitRepo() { local repo="$1" local extraArgs="${2-}" - rm -rf $repo $repo.tmp - mkdir -p $repo + rm -rf "$repo" "$repo".tmp + mkdir -p "$repo" - git -C $repo init $extraArgs - git -C $repo config user.email "foobar@example.com" - git -C $repo config user.name "Foobar" + git -C "$repo" init $extraArgs + git -C "$repo" config user.email "foobar@example.com" + git -C "$repo" config user.name "Foobar" } diff --git a/tests/flakes/flakes.sh b/tests/flakes/flakes.sh index 128f759ea..70de28628 100644 --- a/tests/flakes/flakes.sh +++ b/tests/flakes/flakes.sh @@ -6,27 +6,29 @@ clearStore rm -rf $TEST_HOME/.cache $TEST_HOME/.config flake1Dir=$TEST_ROOT/flake1 -flake2Dir=$TEST_ROOT/flake2 -flake3Dir=$TEST_ROOT/flake3 +flake2Dir=$TEST_ROOT/flake\ 2 +percentEncodedFlake2Dir=$TEST_ROOT/flake%202 +flake3Dir=$TEST_ROOT/flake%20 +percentEncodedFlake3Dir=$TEST_ROOT/flake%2520 flake5Dir=$TEST_ROOT/flake5 flake7Dir=$TEST_ROOT/flake7 nonFlakeDir=$TEST_ROOT/nonFlake badFlakeDir=$TEST_ROOT/badFlake flakeGitBare=$TEST_ROOT/flakeGitBare -for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $nonFlakeDir; do +for repo in "$flake1Dir" "$flake2Dir" "$flake3Dir" "$flake7Dir" "$nonFlakeDir"; do # Give one repo a non-main initial branch. extraArgs= - if [[ $repo == $flake2Dir ]]; then + if [[ "$repo" == "$flake2Dir" ]]; then extraArgs="--initial-branch=main" fi createGitRepo "$repo" "$extraArgs" done -createSimpleGitFlake $flake1Dir +createSimpleGitFlake "$flake1Dir" -cat > $flake2Dir/flake.nix < "$flake2Dir/flake.nix" < $flake2Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/default.nix < "$flake3Dir/default.nix" < $nonFlakeDir/README.md < "$nonFlakeDir/README.md" < $flake1Dir/foo -git -C $flake1Dir add $flake1Dir/foo +echo foo > "$flake1Dir/foo" +git -C "$flake1Dir" add $flake1Dir/foo [[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]] -echo -n '# foo' >> $flake1Dir/flake.nix -flake1OriginalCommit=$(git -C $flake1Dir rev-parse HEAD) -git -C $flake1Dir commit -a -m 'Foo' -flake1NewCommit=$(git -C $flake1Dir rev-parse HEAD) +echo -n '# foo' >> "$flake1Dir/flake.nix" +flake1OriginalCommit=$(git -C "$flake1Dir" rev-parse HEAD) +git -C "$flake1Dir" commit -a -m 'Foo' +flake1NewCommit=$(git -C "$flake1Dir" rev-parse HEAD) hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision) [[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "null" ]] [[ $hash1 != $hash2 ]] # Test 'nix build' on a flake. -nix build -o $TEST_ROOT/result flake1#foo -[[ -e $TEST_ROOT/result/hello ]] +nix build -o "$TEST_ROOT/result" flake1#foo +[[ -e "$TEST_ROOT/result/hello" ]] # Test packages.default. -nix build -o $TEST_ROOT/result flake1 -[[ -e $TEST_ROOT/result/hello ]] +nix build -o "$TEST_ROOT/result" flake1 +[[ -e "$TEST_ROOT/result/hello" ]] -nix build -o $TEST_ROOT/result $flake1Dir -nix build -o $TEST_ROOT/result git+file://$flake1Dir +nix build -o "$TEST_ROOT/result" "$flake1Dir" +nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir" # Check that store symlinks inside a flake are not interpreted as flakes. -nix build -o $flake1Dir/result git+file://$flake1Dir -nix path-info $flake1Dir/result +nix build -o "$flake1Dir/result" "git+file://$flake1Dir" +nix path-info "$flake1Dir/result" # 'getFlake' on an unlocked flakeref should fail in pure mode, but # succeed in impure mode. -(! nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default") -nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default" --impure +(! nix build -o "$TEST_ROOT/result" --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default") +nix build -o "$TEST_ROOT/result" --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default" --impure # 'getFlake' on a locked flakeref should succeed even in pure mode. -nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"git+file://$flake1Dir?rev=$hash2\").packages.$system.default" +nix build -o "$TEST_ROOT/result" --expr "(builtins.getFlake \"git+file://$flake1Dir?rev=$hash2\").packages.$system.default" # Building a flake with an unlocked dependency should fail in pure mode. -(! nix build -o $TEST_ROOT/result flake2#bar --no-registries) -(! nix build -o $TEST_ROOT/result flake2#bar --no-use-registries) +(! nix build -o "$TEST_ROOT/result" flake2#bar --no-registries) +(! nix build -o "$TEST_ROOT/result" flake2#bar --no-use-registries) (! nix eval --expr "builtins.getFlake \"$flake2Dir\"") # But should succeed in impure mode. -(! nix build -o $TEST_ROOT/result flake2#bar --impure) -nix build -o $TEST_ROOT/result flake2#bar --impure --no-write-lock-file +(! nix build -o "$TEST_ROOT/result" flake2#bar --impure) +nix build -o "$TEST_ROOT/result" flake2#bar --impure --no-write-lock-file nix eval --expr "builtins.getFlake \"$flake2Dir\"" --impure # Building a local flake with an unlocked dependency should fail with --no-update-lock-file. -expect 1 nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' +expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file 2>&1 | grep 'requires lock file changes' # But it should succeed without that flag. -nix build -o $TEST_ROOT/result $flake2Dir#bar --no-write-lock-file -expect 1 nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' -nix build -o $TEST_ROOT/result $flake2Dir#bar --commit-lock-file -[[ -e $flake2Dir/flake.lock ]] -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] +nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-write-lock-file +expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file 2>&1 | grep 'requires lock file changes' +nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --commit-lock-file +[[ -e "$flake2Dir/flake.lock" ]] +[[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] # Rerunning the build should not change the lockfile. -nix build -o $TEST_ROOT/result $flake2Dir#bar -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] +nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" +[[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] # Building with a lockfile should not require a fetch of the registry. -nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-registries $flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-use-registries $flake2Dir#bar --refresh +nix build -o "$TEST_ROOT/result" --flake-registry file:///no-registry.json "$flake2Dir#bar" --refresh +nix build -o "$TEST_ROOT/result" --no-registries "$flake2Dir#bar" --refresh +nix build -o "$TEST_ROOT/result" --no-use-registries "$flake2Dir#bar" --refresh # Updating the flake should not change the lockfile. -nix flake lock $flake2Dir -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] +nix flake lock "$flake2Dir" +[[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] # Now we should be able to build the flake in pure mode. -nix build -o $TEST_ROOT/result flake2#bar +nix build -o "$TEST_ROOT/result" flake2#bar # Or without a registry. -nix build -o $TEST_ROOT/result --no-registries git+file://$flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-use-registries git+file://$flake2Dir#bar --refresh +nix build -o "$TEST_ROOT/result" --no-registries "git+file://$percentEncodedFlake2Dir#bar" --refresh +nix build -o "$TEST_ROOT/result" --no-use-registries "git+file://$percentEncodedFlake2Dir#bar" --refresh # Test whether indirect dependencies work. -nix build -o $TEST_ROOT/result $flake3Dir#xyzzy -git -C $flake3Dir add flake.lock +nix build -o "$TEST_ROOT/result" "$flake3Dir#xyzzy" +git -C "$flake3Dir" add flake.lock # Add dependency to flake3. -rm $flake3Dir/flake.nix +rm "$flake3Dir/flake.nix" -cat > $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix <' -A x) = 123 ]] [[ $(NIX_PATH=flake3=flake:flake3 nix-instantiate --eval '' -A x) = 123 ]] # Test alternate lockfile paths. -nix flake lock $flake2Dir --output-lock-file $TEST_ROOT/flake2.lock -cmp $flake2Dir/flake.lock $TEST_ROOT/flake2.lock >/dev/null # lockfiles should be identical, since we're referencing flake2's original one +nix flake lock "$flake2Dir" --output-lock-file $TEST_ROOT/flake2.lock +cmp "$flake2Dir/flake.lock" $TEST_ROOT/flake2.lock >/dev/null # lockfiles should be identical, since we're referencing flake2's original one -nix flake lock $flake2Dir --output-lock-file $TEST_ROOT/flake2-overridden.lock --override-input flake1 git+file://$flake1Dir?rev=$flake1OriginalCommit -expectStderr 1 cmp $flake2Dir/flake.lock $TEST_ROOT/flake2-overridden.lock -nix flake metadata $flake2Dir --reference-lock-file $TEST_ROOT/flake2-overridden.lock | grepQuiet $flake1OriginalCommit +nix flake lock "$flake2Dir" --output-lock-file $TEST_ROOT/flake2-overridden.lock --override-input flake1 git+file://$flake1Dir?rev=$flake1OriginalCommit +expectStderr 1 cmp "$flake2Dir/flake.lock" $TEST_ROOT/flake2-overridden.lock +nix flake metadata "$flake2Dir" --reference-lock-file $TEST_ROOT/flake2-overridden.lock | grepQuiet $flake1OriginalCommit # reference-lock-file can only be used if allow-dirty is set. -expectStderr 1 nix flake metadata $flake2Dir --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock +expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock From 50e61f579cfc8cf64031845f3e73f82593ad2ff3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 31 May 2023 10:36:43 +0200 Subject: [PATCH 286/909] Allow special characters in flake paths Support using nix flakes in paths with spaces or abitrary unicode characters. This introduces the convention that the path part of the URL should be percent-encoded when dealing with `path:` urls and not when using filepaths (following the convention of firefox). Co-authored-by: Rendal --- src/libexpr/flake/flakeref.cc | 85 +++++++++++++++++++---------------- src/libutil/tests/url.cc | 9 ++++ src/libutil/url.cc | 2 +- 3 files changed, 57 insertions(+), 39 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index e1bce90bc..2155c5f12 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -77,14 +77,6 @@ std::pair parseFlakeRefWithFragment( { using namespace fetchers; - static std::string fnRegex = "[0-9a-zA-Z-._~!$&'\"()*+,;=]+"; - - static std::regex pathUrlRegex( - "(/?" + fnRegex + "(?:/" + fnRegex + ")*/?)" - + "(?:\\?(" + queryRegex + "))?" - + "(?:#(" + queryRegex + "))?", - std::regex::ECMAScript); - static std::regex flakeRegex( "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)" + "(?:#(" + queryRegex + "))?", @@ -92,26 +84,23 @@ std::pair parseFlakeRefWithFragment( std::smatch match; - /* Check if 'url' is a flake ID. This is an abbreviated syntax for - 'flake:?ref=&rev='. */ - - if (std::regex_match(url, match, flakeRegex)) { - auto parsedURL = ParsedURL{ - .url = url, - .base = "flake:" + match.str(1), - .scheme = "flake", - .authority = "", - .path = match[1], - }; - - return std::make_pair( - FlakeRef(Input::fromURL(parsedURL, isFlake), ""), - percentDecode(match.str(6))); - } - - else if (std::regex_match(url, match, pathUrlRegex)) { - std::string path = match[1]; - std::string fragment = percentDecode(match.str(3)); + auto parsePathFlakeRef = [&]() { + std::string path = url; + std::string fragment = ""; + std::map query = {}; + auto pathEnd = url.find_first_of("#?"); + auto fragmentStart = pathEnd; + if (pathEnd != std::string::npos && url[pathEnd] == '?') + fragmentStart = url.find("#"); + if (pathEnd != std::string::npos) { + path = url.substr(0, pathEnd); + } + if (fragmentStart != std::string::npos) { + fragment = percentDecode(url.substr(fragmentStart+1)); + } + if (fragmentStart != std::string::npos && pathEnd != std::string::npos) { + query = decodeQuery(url.substr(pathEnd+1, fragmentStart)); + } if (baseDir) { /* Check if 'url' is a path (either absolute or relative @@ -163,7 +152,7 @@ std::pair parseFlakeRefWithFragment( .scheme = "git+file", .authority = "", .path = flakeRoot, - .query = decodeQuery(match[2]), + .query = query, }; if (subdir != "") { @@ -188,7 +177,6 @@ std::pair parseFlakeRefWithFragment( } else { if (!hasPrefix(path, "/")) throw BadURL("flake reference '%s' is not an absolute path", url); - auto query = decodeQuery(match[2]); path = canonPath(path + "/" + getOr(query, "dir", "")); } @@ -197,19 +185,40 @@ std::pair parseFlakeRefWithFragment( attrs.insert_or_assign("path", path); return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment); + }; + + /* Check if 'url' is a flake ID. This is an abbreviated syntax for + 'flake:?ref=&rev='. */ + + if (std::regex_match(url, match, flakeRegex)) { + auto parsedURL = ParsedURL{ + .url = url, + .base = "flake:" + match.str(1), + .scheme = "flake", + .authority = "", + .path = match[1], + }; + + return std::make_pair( + FlakeRef(Input::fromURL(parsedURL), ""), + percentDecode(match.str(6))); } else { - auto parsedURL = parseURL(url); - std::string fragment; - std::swap(fragment, parsedURL.fragment); + try { + auto parsedURL = parseURL(url); + std::string fragment; + std::swap(fragment, parsedURL.fragment); - auto input = Input::fromURL(parsedURL, isFlake); - input.parent = baseDir; + auto input = Input::fromURL(parsedURL, isFlake); + input.parent = baseDir; - return std::make_pair( - FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), - fragment); + return std::make_pair( + FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), + fragment); + } catch (BadURL &) { + return parsePathFlakeRef(); + } } } diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc index a908631e6..a678dad20 100644 --- a/src/libutil/tests/url.cc +++ b/src/libutil/tests/url.cc @@ -335,4 +335,13 @@ namespace nix { ASSERT_EQ(d, s); } + TEST(percentEncode, yen) { + // https://en.wikipedia.org/wiki/Percent-encoding#Character_data + std::string s = reinterpret_cast(u8"円"); + std::string e = "%E5%86%86"; + + ASSERT_EQ(percentEncode(s), e); + ASSERT_EQ(percentDecode(e), s); + } + } diff --git a/src/libutil/url.cc b/src/libutil/url.cc index a8f7d39fd..e700c8eaf 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -103,7 +103,7 @@ std::string percentEncode(std::string_view s, std::string_view keep) || keep.find(c) != std::string::npos) res += c; else - res += fmt("%%%02X", (unsigned int) c); + res += fmt("%%%02X", c & 0xFF); return res; } From e8113747e1abf7fa6225f7fcd31ca9e71e03cfbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 20 Apr 2023 06:51:47 +0200 Subject: [PATCH 287/909] Split the parseFlakeRefWithFragment function Was starting to be very complex and hard to follow. Now the different cases should be easier to understand. --- src/libexpr/flake/flakeref.cc | 279 +++++++++++++++++++--------------- 1 file changed, 155 insertions(+), 124 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 2155c5f12..f7ef19b3f 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -69,127 +69,130 @@ std::optional maybeParseFlakeRef( } } -std::pair parseFlakeRefWithFragment( +std::pair parsePathFlakeRefWithFragment( const std::string & url, const std::optional & baseDir, bool allowMissing, bool isFlake) { - using namespace fetchers; + std::string path = url; + std::string fragment = ""; + std::map query; + auto pathEnd = url.find_first_of("#?"); + auto fragmentStart = pathEnd; + if (pathEnd != std::string::npos && url[pathEnd] == '?') { + fragmentStart = url.find("#"); + } + if (pathEnd != std::string::npos) { + path = url.substr(0, pathEnd); + } + if (fragmentStart != std::string::npos) { + fragment = percentDecode(url.substr(fragmentStart+1)); + } + if (pathEnd != std::string::npos && fragmentStart != std::string::npos) { + query = decodeQuery(url.substr(pathEnd+1, fragmentStart)); + } + + if (baseDir) { + /* Check if 'url' is a path (either absolute or relative + to 'baseDir'). If so, search upward to the root of the + repo (i.e. the directory containing .git). */ + + path = absPath(path, baseDir); + + if (isFlake) { + + if (!allowMissing && !pathExists(path + "/flake.nix")){ + notice("path '%s' does not contain a 'flake.nix', searching up",path); + + // Save device to detect filesystem boundary + dev_t device = lstat(path).st_dev; + bool found = false; + while (path != "/") { + if (pathExists(path + "/flake.nix")) { + found = true; + break; + } else if (pathExists(path + "/.git")) + throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path); + else { + if (lstat(path).st_dev != device) + throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path); + } + path = dirOf(path); + } + if (!found) + throw BadURL("could not find a flake.nix file"); + } + + if (!S_ISDIR(lstat(path).st_mode)) + throw BadURL("path '%s' is not a flake (because it's not a directory)", path); + + if (!allowMissing && !pathExists(path + "/flake.nix")) + throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); + + auto flakeRoot = path; + std::string subdir; + + while (flakeRoot != "/") { + if (pathExists(flakeRoot + "/.git")) { + auto base = std::string("git+file://") + flakeRoot; + + auto parsedURL = ParsedURL{ + .url = base, // FIXME + .base = base, + .scheme = "git+file", + .authority = "", + .path = flakeRoot, + .query = query, + }; + + if (subdir != "") { + if (parsedURL.query.count("dir")) + throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url); + parsedURL.query.insert_or_assign("dir", subdir); + } + + if (pathExists(flakeRoot + "/.git/shallow")) + parsedURL.query.insert_or_assign("shallow", "1"); + + return std::make_pair( + FlakeRef(fetchers::Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")), + fragment); + } + + subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); + flakeRoot = dirOf(flakeRoot); + } + } + + } else { + if (!hasPrefix(path, "/")) + throw BadURL("flake reference '%s' is not an absolute path", url); + path = canonPath(path + "/" + getOr(query, "dir", "")); + } + + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "path"); + attrs.insert_or_assign("path", path); + + return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(std::move(attrs)), ""), fragment); +}; + + +/* Check if 'url' is a flake ID. This is an abbreviated syntax for + 'flake:?ref=&rev='. */ +std::optional> parseFlakeIdRef( + const std::string & url, + bool isFlake +) +{ + std::smatch match; static std::regex flakeRegex( "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)" + "(?:#(" + queryRegex + "))?", std::regex::ECMAScript); - std::smatch match; - - auto parsePathFlakeRef = [&]() { - std::string path = url; - std::string fragment = ""; - std::map query = {}; - auto pathEnd = url.find_first_of("#?"); - auto fragmentStart = pathEnd; - if (pathEnd != std::string::npos && url[pathEnd] == '?') - fragmentStart = url.find("#"); - if (pathEnd != std::string::npos) { - path = url.substr(0, pathEnd); - } - if (fragmentStart != std::string::npos) { - fragment = percentDecode(url.substr(fragmentStart+1)); - } - if (fragmentStart != std::string::npos && pathEnd != std::string::npos) { - query = decodeQuery(url.substr(pathEnd+1, fragmentStart)); - } - - if (baseDir) { - /* Check if 'url' is a path (either absolute or relative - to 'baseDir'). If so, search upward to the root of the - repo (i.e. the directory containing .git). */ - - path = absPath(path, baseDir); - - if (isFlake) { - - if (!allowMissing && !pathExists(path + "/flake.nix")){ - notice("path '%s' does not contain a 'flake.nix', searching up",path); - - // Save device to detect filesystem boundary - dev_t device = lstat(path).st_dev; - bool found = false; - while (path != "/") { - if (pathExists(path + "/flake.nix")) { - found = true; - break; - } else if (pathExists(path + "/.git")) - throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path); - else { - if (lstat(path).st_dev != device) - throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path); - } - path = dirOf(path); - } - if (!found) - throw BadURL("could not find a flake.nix file"); - } - - if (!S_ISDIR(lstat(path).st_mode)) - throw BadURL("path '%s' is not a flake (because it's not a directory)", path); - - if (!allowMissing && !pathExists(path + "/flake.nix")) - throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); - - auto flakeRoot = path; - std::string subdir; - - while (flakeRoot != "/") { - if (pathExists(flakeRoot + "/.git")) { - auto base = std::string("git+file://") + flakeRoot; - - auto parsedURL = ParsedURL{ - .url = base, // FIXME - .base = base, - .scheme = "git+file", - .authority = "", - .path = flakeRoot, - .query = query, - }; - - if (subdir != "") { - if (parsedURL.query.count("dir")) - throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url); - parsedURL.query.insert_or_assign("dir", subdir); - } - - if (pathExists(flakeRoot + "/.git/shallow")) - parsedURL.query.insert_or_assign("shallow", "1"); - - return std::make_pair( - FlakeRef(Input::fromURL(parsedURL, isFlake), getOr(parsedURL.query, "dir", "")), - fragment); - } - - subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); - flakeRoot = dirOf(flakeRoot); - } - } - - } else { - if (!hasPrefix(path, "/")) - throw BadURL("flake reference '%s' is not an absolute path", url); - path = canonPath(path + "/" + getOr(query, "dir", "")); - } - - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "path"); - attrs.insert_or_assign("path", path); - - return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment); - }; - - /* Check if 'url' is a flake ID. This is an abbreviated syntax for - 'flake:?ref=&rev='. */ - if (std::regex_match(url, match, flakeRegex)) { auto parsedURL = ParsedURL{ .url = url, @@ -200,25 +203,53 @@ std::pair parseFlakeRefWithFragment( }; return std::make_pair( - FlakeRef(Input::fromURL(parsedURL), ""), + FlakeRef(fetchers::Input::fromURL(parsedURL, isFlake), ""), percentDecode(match.str(6))); } - else { - try { - auto parsedURL = parseURL(url); - std::string fragment; - std::swap(fragment, parsedURL.fragment); + return {}; +} - auto input = Input::fromURL(parsedURL, isFlake); - input.parent = baseDir; +std::optional> parseURLFlakeRef( + const std::string & url, + const std::optional & baseDir, + bool isFlake +) +{ + ParsedURL parsedURL; + try { + parsedURL = parseURL(url); + } catch (BadURL &) { + return std::nullopt; + } - return std::make_pair( - FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), - fragment); - } catch (BadURL &) { - return parsePathFlakeRef(); - } + std::string fragment; + std::swap(fragment, parsedURL.fragment); + + auto input = fetchers::Input::fromURL(parsedURL, isFlake); + input.parent = baseDir; + + return std::make_pair( + FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), + fragment); +} + +std::pair parseFlakeRefWithFragment( + const std::string & url, + const std::optional & baseDir, + bool allowMissing, + bool isFlake) +{ + using namespace fetchers; + + std::smatch match; + + if (auto res = parseFlakeIdRef(url, isFlake)) { + return *res; + } else if (auto res = parseURLFlakeRef(url, baseDir, isFlake)) { + return *res; + } else { + return parsePathFlakeRefWithFragment(url, baseDir, allowMissing, isFlake); } } From 34115076969f20be935ae8eaf579928ac07981e3 Mon Sep 17 00:00:00 2001 From: Rasmus Rendal Date: Tue, 14 Jun 2022 17:38:48 +0200 Subject: [PATCH 288/909] Document the percent-encoding mechanism --- doc/manual/src/release-notes/rl-next.md | 4 ++++ src/nix/flake.md | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index c869b5e2f..2467e7ccc 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1 +1,5 @@ # Release X.Y (202?-??-??) + +- [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. + +- [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). diff --git a/src/nix/flake.md b/src/nix/flake.md index 92f477917..1c512b315 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -67,6 +67,13 @@ inputs.nixpkgs = { }; ``` +Following [RFC 3986](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1), +characters outside of the allowed range (i.e. are neither [reserved +character](https://datatracker.ietf.org/doc/html/rfc3986#section-2.2) or +[unreserved +characters](https://datatracker.ietf.org/doc/html/rfc3986#section-2.3)) must be +percent-encoded. + ### Examples Here are some examples of flake references in their URL-like representation: @@ -103,10 +110,15 @@ The semantic of such a path is as follows: 2. The filesystem root (/), or 3. A folder on a different mount point. +Contrary to URL-like reference, path-like flake references can contain +arbitrary unicode characters (except `#` and `?`). + ### Examples * `.`: The flake to which the current directory belongs to. * `/home/alice/src/patchelf`: A flake in some other directory. +* `./../sub directory/with Ûñî©ôδ€`: A flake in another relative directory that + has Unicode characters in its name. ## Flake reference attributes From 39ba81a4eb7404f9a12cf0768da51c0e0e26b588 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Sep 2023 22:52:57 -0400 Subject: [PATCH 289/909] Improve internal API docs for two file hashing functions Co-Authored-By: Matthew Bauer Co-Authored-By: Carlo Nucera --- src/libutil/hash.hh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ae3ee40f4..c3aa5cd81 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -145,13 +145,17 @@ std::string printHash16or32(const Hash & hash); Hash hashString(HashType ht, std::string_view s); /** - * Compute the hash of the given file. + * Compute the hash of the given file, hashing its contents directly. + * + * (Metadata, such as the executable permission bit, is ignored.) */ Hash hashFile(HashType ht, const Path & path); /** - * Compute the hash of the given path. The hash is defined as - * (essentially) hashString(ht, dumpPath(path)). + * Compute the hash of the given path, serializing as a Nix Archive and + * then hashing that. + * + * The hash is defined as (essentially) hashString(ht, dumpPath(path)). */ typedef std::pair HashResult; HashResult hashPath(HashType ht, const Path & path, From 9d6114313b1b06a020bed230cfe7f05e6075b875 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Sep 2023 22:31:19 -0400 Subject: [PATCH 290/909] Move `ParseSink` to its own header We will soon add a new implemenation so the one for NARs in `archive.cc` isn't the only one. Co-Authored-By: Matthew Bauer Co-Authored-By: Carlo Nucera --- src/libutil/archive.hh | 17 +---------------- src/libutil/fs-sink.hh | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 16 deletions(-) create mode 100644 src/libutil/fs-sink.hh diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 2cf164a41..3530783c1 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "serialise.hh" +#include "fs-sink.hh" namespace nix { @@ -72,22 +73,6 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink, */ void dumpString(std::string_view s, Sink & sink); -/** - * \todo Fix this API, it sucks. - */ -struct ParseSink -{ - virtual void createDirectory(const Path & path) { }; - - virtual void createRegularFile(const Path & path) { }; - virtual void closeRegularFile() { }; - virtual void isExecutable() { }; - virtual void preallocateContents(uint64_t size) { }; - virtual void receiveContents(std::string_view data) { }; - - virtual void createSymlink(const Path & path, const std::string & target) { }; -}; - /** * If the NAR archive contains a single file at top-level, then save * the contents of the file to `s`. Otherwise barf. diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh new file mode 100644 index 000000000..ed6484e61 --- /dev/null +++ b/src/libutil/fs-sink.hh @@ -0,0 +1,25 @@ +#pragma once +///@file + +#include "types.hh" +#include "serialise.hh" + +namespace nix { + +/** + * \todo Fix this API, it sucks. + */ +struct ParseSink +{ + virtual void createDirectory(const Path & path) { }; + + virtual void createRegularFile(const Path & path) { }; + virtual void closeRegularFile() { }; + virtual void isExecutable() { }; + virtual void preallocateContents(uint64_t size) { }; + virtual void receiveContents(std::string_view data) { }; + + virtual void createSymlink(const Path & path, const std::string & target) { }; +}; + +} From 8a416e819c9727c2e9efd70f1d2a4b0bfda11663 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Sep 2023 22:49:01 -0400 Subject: [PATCH 291/909] Move `RestoreSink` to `fs-sink.cc` Co-Authored-By: Matthew Bauer Co-Authored-By: Carlo Nucera --- src/libutil/archive.cc | 67 ---------------------------------- src/libutil/fs-sink.cc | 82 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 67 deletions(-) create mode 100644 src/libutil/fs-sink.cc diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 268a798d9..0f0cae7c0 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -27,8 +27,6 @@ struct ArchiveSettings : Config #endif "use-case-hack", "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; - Setting preallocateContents{this, false, "preallocate-contents", - "Whether to preallocate files when writing objects with known size."}; }; static ArchiveSettings archiveSettings; @@ -302,71 +300,6 @@ void parseDump(ParseSink & sink, Source & source) } -struct RestoreSink : ParseSink -{ - Path dstPath; - AutoCloseFD fd; - - void createDirectory(const Path & path) override - { - Path p = dstPath + path; - if (mkdir(p.c_str(), 0777) == -1) - throw SysError("creating directory '%1%'", p); - }; - - void createRegularFile(const Path & path) override - { - Path p = dstPath + path; - fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); - if (!fd) throw SysError("creating file '%1%'", p); - } - - void closeRegularFile() override - { - /* Call close explicitly to make sure the error is checked */ - fd.close(); - } - - void isExecutable() override - { - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("fstat"); - if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) - throw SysError("fchmod"); - } - - void preallocateContents(uint64_t len) override - { - if (!archiveSettings.preallocateContents) - return; - -#if HAVE_POSIX_FALLOCATE - if (len) { - errno = posix_fallocate(fd.get(), 0, len); - /* Note that EINVAL may indicate that the underlying - filesystem doesn't support preallocation (e.g. on - OpenSolaris). Since preallocation is just an - optimisation, ignore it. */ - if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) - throw SysError("preallocating file of %1% bytes", len); - } -#endif - } - - void receiveContents(std::string_view data) override - { - writeFull(fd.get(), data); - } - - void createSymlink(const Path & path, const std::string & target) override - { - Path p = dstPath + path; - nix::createSymlink(target, p); - } -}; - - void restorePath(const Path & path, Source & source) { RestoreSink sink; diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc new file mode 100644 index 000000000..fcffd3216 --- /dev/null +++ b/src/libutil/fs-sink.cc @@ -0,0 +1,82 @@ +#include + +#include "config.hh" +#include "fs-sink.hh" + +namespace nix { + +struct RestoreSinkSettings : Config +{ + Setting preallocateContents{this, false, "preallocate-contents", + "Whether to preallocate files when writing objects with known size."}; +}; + +static RestoreSinkSettings restoreSinkSettings; + +static GlobalConfig::Register r1(&restoreSinkSettings); + +struct RestoreSink : ParseSink +{ + Path dstPath; + AutoCloseFD fd; + + void createDirectory(const Path & path) override + { + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) + throw SysError("creating directory '%1%'", p); + }; + + void createRegularFile(const Path & path) override + { + Path p = dstPath + path; + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); + if (!fd) throw SysError("creating file '%1%'", p); + } + + void closeRegularFile() override + { + /* Call close explicitly to make sure the error is checked */ + fd.close(); + } + + void isExecutable() override + { + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("fstat"); + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); + } + + void preallocateContents(uint64_t len) override + { + if (!archiveSettings.preallocateContents) + return; + +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError("preallocating file of %1% bytes", len); + } +#endif + } + + void receiveContents(std::string_view data) override + { + writeFull(fd.get(), data); + } + + void createSymlink(const Path & path, const std::string & target) override + { + Path p = dstPath + path; + nix::createSymlink(target, p); + } +}; + +} From f2e201fbdb93925b9588f78a03d67fcd91c74b65 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Sep 2023 22:49:01 -0400 Subject: [PATCH 292/909] Expose `RestoreSink` in header (`fs-sink.hh`) Co-Authored-By: Matthew Bauer Co-Authored-By: Carlo Nucera --- src/libutil/fs-sink.cc | 117 ++++++++++++++++++++--------------------- src/libutil/fs-sink.hh | 17 ++++++ 2 files changed, 73 insertions(+), 61 deletions(-) diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index fcffd3216..a08a723a4 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -15,68 +15,63 @@ static RestoreSinkSettings restoreSinkSettings; static GlobalConfig::Register r1(&restoreSinkSettings); -struct RestoreSink : ParseSink + +void RestoreSink::createDirectory(const Path & path) { - Path dstPath; - AutoCloseFD fd; - - void createDirectory(const Path & path) override - { - Path p = dstPath + path; - if (mkdir(p.c_str(), 0777) == -1) - throw SysError("creating directory '%1%'", p); - }; - - void createRegularFile(const Path & path) override - { - Path p = dstPath + path; - fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); - if (!fd) throw SysError("creating file '%1%'", p); - } - - void closeRegularFile() override - { - /* Call close explicitly to make sure the error is checked */ - fd.close(); - } - - void isExecutable() override - { - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("fstat"); - if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) - throw SysError("fchmod"); - } - - void preallocateContents(uint64_t len) override - { - if (!archiveSettings.preallocateContents) - return; - -#if HAVE_POSIX_FALLOCATE - if (len) { - errno = posix_fallocate(fd.get(), 0, len); - /* Note that EINVAL may indicate that the underlying - filesystem doesn't support preallocation (e.g. on - OpenSolaris). Since preallocation is just an - optimisation, ignore it. */ - if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) - throw SysError("preallocating file of %1% bytes", len); - } -#endif - } - - void receiveContents(std::string_view data) override - { - writeFull(fd.get(), data); - } - - void createSymlink(const Path & path, const std::string & target) override - { - Path p = dstPath + path; - nix::createSymlink(target, p); - } + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) + throw SysError("creating directory '%1%'", p); }; +void RestoreSink::createRegularFile(const Path & path) +{ + Path p = dstPath + path; + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); + if (!fd) throw SysError("creating file '%1%'", p); +} + +void RestoreSink::closeRegularFile() +{ + /* Call close explicitly to make sure the error is checked */ + fd.close(); +} + +void RestoreSink::isExecutable() +{ + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("fstat"); + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); +} + +void RestoreSink::preallocateContents(uint64_t len) +{ + if (!restoreSinkSettings.preallocateContents) + return; + +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError("preallocating file of %1% bytes", len); + } +#endif +} + +void RestoreSink::receiveContents(std::string_view data) +{ + writeFull(fd.get(), data); +} + +void RestoreSink::createSymlink(const Path & path, const std::string & target) +{ + Path p = dstPath + path; + nix::createSymlink(target, p); +} + } diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index ed6484e61..6837e2fc4 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -22,4 +22,21 @@ struct ParseSink virtual void createSymlink(const Path & path, const std::string & target) { }; }; +struct RestoreSink : ParseSink +{ + Path dstPath; + AutoCloseFD fd; + + + void createDirectory(const Path & path) override; + + void createRegularFile(const Path & path) override; + void closeRegularFile() override; + void isExecutable() override; + void preallocateContents(uint64_t size) override; + void receiveContents(std::string_view data) override; + + void createSymlink(const Path & path, const std::string & target) override; +}; + } From 694810ba34a74226eba22daf5c6264b9d81e2926 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 22 Sep 2023 23:54:04 -0400 Subject: [PATCH 293/909] doc: `showOptions`: Move union to caller `showOptions` itself doesn't care, so it shouldn't take two separate arguments. --- doc/manual/generate-manpage.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 87891fa7a..b85c488c9 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -78,16 +78,15 @@ let maybeOptions = optionalString (details.flags != {}) '' # Options - ${showOptions details.flags toplevel.flags} + ${showOptions (details.flags // toplevel.flags)} > **Note** > > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. ''; - showOptions = options: commonOptions: + showOptions = allOptions: let - allOptions = options // commonOptions; showCategory = cat: '' ${optionalString (cat != "") "**${cat}:**"} From 9c640c122931aa6515e1f3c80cb4d415a1352cc7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 23 Sep 2023 00:28:16 -0400 Subject: [PATCH 294/909] doc: `showOptions`: Simplify code with `builtins.groupBy` This makes grouping options by category much nicer. No behavior should be changed. --- doc/manual/generate-manpage.nix | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index b85c488c9..e18810594 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -1,8 +1,8 @@ let inherit (builtins) - attrNames attrValues fromJSON listToAttrs mapAttrs + attrNames attrValues fromJSON listToAttrs mapAttrs groupBy concatStringsSep concatMap length lessThan replaceStrings sort; - inherit (import ./utils.nix) concatStrings optionalString filterAttrs trim squash unique showSettings; + inherit (import ./utils.nix) attrsToList concatStrings optionalString filterAttrs trim squash unique showSettings; in commandDump: @@ -87,12 +87,11 @@ let showOptions = allOptions: let - showCategory = cat: '' + showCategory = cat: opts: '' ${optionalString (cat != "") "**${cat}:**"} - ${listOptions (filterAttrs (n: v: v.category == cat) allOptions)} + ${concatStringsSep "\n" (attrValues (mapAttrs showOption opts))} ''; - listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts)); showOption = name: option: let shortName = optionalString @@ -106,8 +105,12 @@ let ${option.description} ''; - categories = sort lessThan (unique (map (cmd: cmd.category) (attrValues allOptions))); - in concatStrings (map showCategory categories); + categories = mapAttrs + (_: listToAttrs) + (groupBy + (cmd: cmd.value.category) + (attrsToList allOptions)); + in concatStrings (attrValues (mapAttrs showCategory categories)); in squash result; appendName = filename: name: (if filename == "nix" then "nix3" else filename) + "-" + name; From 1d9fd3a6f8ac53be4ba7d409defe291c0dbdf97a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 22 Sep 2023 23:55:06 -0400 Subject: [PATCH 295/909] manual / manpages: Adjust option filter filtering, move from C++ to Nix Behavior change: Before we only showed uption if the command-specific options were non-empty. But that is somewhat odd since we also show common options. Now, we do everything based on the union of both sorts of options (with hidden-categories filtered, as before). Implementation change: The JSON dumping once again includes all options; the filtering of hidden categories is done in the Nix instead. This is better separation of "content" vs "presentation", and prepare the way for the HTML manual vs manpages / `--help` doing different things. --- doc/manual/generate-manpage.nix | 8 ++++++-- src/libutil/args.cc | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index e18810594..b41730bc2 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -75,10 +75,14 @@ let (details ? doc) (replaceStrings ["@stores@"] [storeDocs] details.doc); - maybeOptions = optionalString (details.flags != {}) '' + maybeOptions = let + allVisibleOptions = filterAttrs + (_: o: ! o.hiddenCategory) + (details.flags // toplevel.flags); + in optionalString (allVisibleOptions != {}) '' # Options - ${showOptions (details.flags // toplevel.flags)} + ${showOptions allVisibleOptions} > **Note** > diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 8db293762..e410c7eec 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -236,7 +236,7 @@ nlohmann::json Args::toJSON() for (auto & [name, flag] : longFlags) { auto j = nlohmann::json::object(); - if (hiddenCategories.count(flag->category)) continue; + j["hiddenCategory"] = hiddenCategories.count(flag->category) > 0; if (flag->aliases.count(name)) continue; if (flag->shortName) j["shortName"] = std::string(1, flag->shortName); From 9f93972c4d73abdcde789112229209af2cbda520 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 23 Sep 2023 00:35:03 -0400 Subject: [PATCH 296/909] manual / manpages: Make option category names a proper subheader Before they were an "ad-hoc" header with bold and a colon; now they are a proper subheader. For the man pages, this doesn't make much of a difference, but it will help more on for the HTML manual, where things can be restyled. Again, good separation of content vs presentation. --- doc/manual/generate-manpage.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index b41730bc2..21a4802e5 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -92,7 +92,7 @@ let showOptions = allOptions: let showCategory = cat: opts: '' - ${optionalString (cat != "") "**${cat}:**"} + ${optionalString (cat != "") "## ${cat}"} ${concatStringsSep "\n" (attrValues (mapAttrs showOption opts))} ''; From e4b83fbfe2e08ddd2ebd409c26f2dc4c11ee1d4a Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Sun, 24 Sep 2023 14:54:41 +0800 Subject: [PATCH 297/909] libexpr: const rvalue reference -> value for nix::Expr nodes --- src/libexpr/nixexpr.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 5ca3d1fa6..944977e50 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -238,7 +238,7 @@ struct ExprSelect : Expr PosIdx pos; Expr * e, * def; AttrPath attrPath; - ExprSelect(const PosIdx & pos, Expr * e, const AttrPath && attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { }; + ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { }; ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; PosIdx getPos() const override { return pos; } COMMON_METHODS @@ -248,7 +248,7 @@ struct ExprOpHasAttr : Expr { Expr * e; AttrPath attrPath; - ExprOpHasAttr(Expr * e, const AttrPath && attrPath) : e(e), attrPath(std::move(attrPath)) { }; + ExprOpHasAttr(Expr * e, AttrPath attrPath) : e(e), attrPath(std::move(attrPath)) { }; PosIdx getPos() const override { return e->getPos(); } COMMON_METHODS }; From 89e5e68799132bc2f1d2551002dab84b63fde9ac Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 24 Sep 2023 20:28:34 +0200 Subject: [PATCH 298/909] doc/hacking: fix `make` target to build the docs (#9033) Was confused why `make html` didn't work while working on #9032, but then I realized that after this section was written, the target was renamed to `manual-html` in 6910f5dcb6784360589b860b8f80487a5c97fe08. --- doc/manual/src/contributing/hacking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 22f629c6f..a2446b53a 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -228,7 +228,7 @@ This happens late in the process, so `nix build` is not suitable for iterating. To build the manual incrementally, run: ```console -make html -j $NIX_BUILD_CORES +make manual-html -j $NIX_BUILD_CORES ``` In order to reflect changes to the [Makefile], clear all generated files before re-building: From b3433099d4ef3fbc2ef86f97cf19dcd543bd05e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Mon, 25 Sep 2023 09:56:49 +0200 Subject: [PATCH 299/909] Apply suggestions from code review Co-authored-by: Valentin Gagarin --- src/nix/flake.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index 1c512b315..939269c6a 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -68,11 +68,9 @@ inputs.nixpkgs = { ``` Following [RFC 3986](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1), -characters outside of the allowed range (i.e. are neither [reserved -character](https://datatracker.ietf.org/doc/html/rfc3986#section-2.2) or -[unreserved -characters](https://datatracker.ietf.org/doc/html/rfc3986#section-2.3)) must be -percent-encoded. +characters outside of the allowed range (i.e. neither [reserved characters](https://datatracker.ietf.org/doc/html/rfc3986#section-2.2) +nor [unreserved characters](https://datatracker.ietf.org/doc/html/rfc3986#section-2.3)) +must be percent-encoded. ### Examples @@ -110,8 +108,7 @@ The semantic of such a path is as follows: 2. The filesystem root (/), or 3. A folder on a different mount point. -Contrary to URL-like reference, path-like flake references can contain -arbitrary unicode characters (except `#` and `?`). +Contrary to URL-like references, path-like flake references can contain arbitrary unicode characters (except `#` and `?`). ### Examples From 4606a07bb62c650d2261f2d8d5e93e547acf0af6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 25 Sep 2023 08:20:39 -0400 Subject: [PATCH 300/909] generate-manpage.nix: Add comment explaining one bit --- doc/manual/generate-manpage.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 21a4802e5..41725aed6 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -110,6 +110,7 @@ let ${option.description} ''; categories = mapAttrs + # Convert each group from a list of key-value pairs back to an attrset (_: listToAttrs) (groupBy (cmd: cmd.value.category) From 70b5e6050cd135b75942f731ad892e4f473c6c21 Mon Sep 17 00:00:00 2001 From: waalge <47293755+waalge@users.noreply.github.com> Date: Mon, 25 Sep 2023 13:39:11 +0000 Subject: [PATCH 301/909] fix docstring --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index cd9a05bb2..0d503ae26 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2991,7 +2991,7 @@ static RegisterPrimOp primop_tail({ .name = "__tail", .args = {"list"}, .doc = R"( - Return the second to last elements of a list; abort evaluation if + Return the list without its first item; abort evaluation if the argument isn’t a list or is an empty list. > **Warning** From ad213103d85d22b52c608362cb2127fa10e49620 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Mon, 25 Sep 2023 17:45:13 +0100 Subject: [PATCH 302/909] src/libstore/profiles.cc: fix comparison of sign difference Detected by `gcc` as: CXX src/libstore/profiles.o src/libstore/profiles.cc: In function 'void nix::deleteGenerationsGreaterThan(const Path&, GenerationNumber, bool)': src/libstore/profiles.cc:186:50: warning: comparison of integer expressions of different signedness: 'int' and 'nix::GenerationNumber' {aka 'long unsigned int'} [-Wsign-compare] 186 | for (auto keep = 0; i != gens.rend() && keep < max; ++i, ++keep); | ~~~~~^~~~~ --- src/libstore/profiles.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 4e9955948..239047dd6 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -183,7 +183,7 @@ void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bo iterDropUntil(gens, i, [&](auto & g) { return g.number == curGen; }); // Skip over `max` generations, preserving them - for (auto keep = 0; i != gens.rend() && keep < max; ++i, ++keep); + for (GenerationNumber keep = 0; i != gens.rend() && keep < max; ++i, ++keep); // Delete the rest for (; i != gens.rend(); ++i) From bd24176ac57b9d418b4ec51448b188cb74b87945 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 25 Sep 2023 17:51:17 +0100 Subject: [PATCH 303/909] libexpr/nixexpr.hh: Remove redundant inline This is redundant since definitions in C++ record are implicitly inline-ed. Co-authored-by: Yingchi Long --- src/libexpr/nixexpr.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 17b2d1136..5c9242759 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -156,7 +156,7 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) struct Expr { static unsigned long nrExprs; - inline Expr() { + Expr() { nrExprs++; } virtual ~Expr() { }; From cba53b3a18c2758e3d969d9d5781294560afe1b7 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 00:39:11 +0200 Subject: [PATCH 304/909] reformat for readability --- doc/manual/src/command-ref/nix-store/realise.md | 8 ++++++-- doc/manual/src/glossary.md | 10 +++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index a421a4220..5428d57fa 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -15,8 +15,12 @@ Each of *paths* is processed as follows: 1. If it is not [valid], substitute the store derivation file itself. 2. Realise its [output paths]: - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure]. - - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying content-addressed realisation entries in the [Nix database]. - - For any store paths that cannot be substituted, produce the required store objects. This involves first realising all outputs of the derivation's dependencies and then running the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable. + - With [content-addressed derivations] (experimental): + Determine the output paths to realise by querying content-addressed realisation entries in the [Nix database]. + - For any store paths that cannot be substituted, produce the required store objects: + 1. Realise all outputs of the derivation's dependencies + 2. Run the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable + - Otherwise, and if the path is not already valid: Try to fetch the associated [store objects] in the path's [closure] from [substituters]. If no substitutes are available and no store derivation is given, realisation fails. diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index b7c340d36..5345f2deb 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -33,11 +33,15 @@ Ensure a [store path] is [valid][validity]. - This means either running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation], or fetching a pre-built [store object] from a [substituter], or delegating to a [remote builder](@docroot@/advanced-topics/distributed-builds.html) and retrieving the outputs. + This can be achieved by: + - Fetching a pre-built [store object] from a [substituter] + - Running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation] + - Delegating to a [remote builder](@docroot@/advanced-topics/distributed-builds.html) and retrieving the outputs + - See [`nix-build`](./command-ref/nix-build.md) and [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md). + See [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md) for a detailed description of the algorithm. - See [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). + See also [`nix-build`](./command-ref/nix-build.md) and [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). [realise]: #gloss-realise From 541890463d83d1a818dfabc79053d1302bd828a2 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 18 Jul 2023 13:49:59 +0200 Subject: [PATCH 305/909] make separate section for builder execution --- doc/manual/src/language/derivations.md | 116 +++++++++++++------------ 1 file changed, 59 insertions(+), 57 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 79a09122a..df4673cf5 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -93,69 +93,71 @@ wrapper around `derivation` that adds a default value for `system` and always uses Bash as the builder, to which the supplied builder is passed as a command-line argument. See the Nixpkgs manual for details. -The builder is executed as follows: +## Builder execution - - A temporary directory is created under the directory specified by - `TMPDIR` (default `/tmp`) where the build will take place. The - current directory is changed to this directory. +The [`builder`](#attr-builder) is executed as follows: - - The environment is cleared and set to the derivation attributes, as - specified above. +- A temporary directory is created under the directory specified by + `TMPDIR` (default `/tmp`) where the build will take place. The + current directory is changed to this directory. - - In addition, the following variables are set: - - - `NIX_BUILD_TOP` contains the path of the temporary directory for - this build. - - - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the - temporary directory. This is to prevent the builder from - accidentally writing temporary files anywhere else. Doing so - might cause interference by other processes. - - - `PATH` is set to `/path-not-set` to prevent shells from - initialising it to their built-in default value. - - - `HOME` is set to `/homeless-shelter` to prevent programs from - using `/etc/passwd` or the like to find the user's home - directory, which could cause impurity. Usually, when `HOME` is - set, it is used as the location of the home directory, even if - it points to a non-existent path. - - - `NIX_STORE` is set to the path of the top-level Nix store - directory (typically, `/nix/store`). - - - For each output declared in `outputs`, the corresponding - environment variable is set to point to the intended path in the - Nix store for that output. Each output path is a concatenation - of the cryptographic hash of all build inputs, the `name` - attribute and the output name. (The output name is omitted if - it’s `out`.) +- The environment is cleared and set to the derivation attributes, as + specified above. - - If an output path already exists, it is removed. Also, locks are - acquired to prevent multiple Nix instances from performing the same - build at the same time. +- In addition, the following variables are set: - - A log of the combined standard output and error is written to - `/nix/var/log/nix`. + - `NIX_BUILD_TOP` contains the path of the temporary directory for + this build. - - The builder is executed with the arguments specified by the - attribute `args`. If it exits with exit code 0, it is considered to - have succeeded. + - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the + temporary directory. This is to prevent the builder from + accidentally writing temporary files anywhere else. Doing so + might cause interference by other processes. - - The temporary directory is removed (unless the `-K` option was - specified). + - `PATH` is set to `/path-not-set` to prevent shells from + initialising it to their built-in default value. - - If the build was successful, Nix scans each output path for - references to input paths by looking for the hash parts of the input - paths. Since these are potential runtime dependencies, Nix registers - them as dependencies of the output paths. + - `HOME` is set to `/homeless-shelter` to prevent programs from + using `/etc/passwd` or the like to find the user's home + directory, which could cause impurity. Usually, when `HOME` is + set, it is used as the location of the home directory, even if + it points to a non-existent path. - - After the build, Nix sets the last-modified timestamp on all files - in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to - the default group, and sets the mode of the file to 0444 or 0555 - (i.e., read-only, with execute permission enabled if the file was - originally executable). Note that possible `setuid` and `setgid` - bits are cleared. Setuid and setgid programs are not currently - supported by Nix. This is because the Nix archives used in - deployment have no concept of ownership information, and because it - makes the build result dependent on the user performing the build. + - `NIX_STORE` is set to the path of the top-level Nix store + directory (typically, `/nix/store`). + + - For each output declared in `outputs`, the corresponding + environment variable is set to point to the intended path in the + Nix store for that output. Each output path is a concatenation + of the cryptographic hash of all build inputs, the `name` + attribute and the output name. (The output name is omitted if + it’s `out`.) + +- If an output path already exists, it is removed. Also, locks are + acquired to prevent multiple Nix instances from performing the same + build at the same time. + +- A log of the combined standard output and error is written to + `/nix/var/log/nix`. + +- The builder is executed with the arguments specified by the + attribute `args`. If it exits with exit code 0, it is considered to + have succeeded. + +- The temporary directory is removed (unless the `-K` option was + specified). + +- If the build was successful, Nix scans each output path for + references to input paths by looking for the hash parts of the input + paths. Since these are potential runtime dependencies, Nix registers + them as dependencies of the output paths. + +- After the build, Nix sets the last-modified timestamp on all files + in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to + the default group, and sets the mode of the file to 0444 or 0555 + (i.e., read-only, with execute permission enabled if the file was + originally executable). Note that possible `setuid` and `setgid` + bits are cleared. Setuid and setgid programs are not currently + supported by Nix. This is because the Nix archives used in + deployment have no concept of ownership information, and because it + makes the build result dependent on the user performing the build. From e2f118efeda5c0a9b1380db736eb2b86f691b1fa Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 18 Jul 2023 14:03:17 +0200 Subject: [PATCH 306/909] restructure and reword input attributes section on `derivation` --- doc/manual/src/language/derivations.md | 208 +++++++++++++++---------- 1 file changed, 123 insertions(+), 85 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index df4673cf5..56c6818ca 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -1,97 +1,135 @@ # Derivations -The most important built-in function is `derivation`, which is used to -describe a single derivation (a build task). It takes as input a set, -the attributes of which specify the inputs of the build. +The most important built-in function is `derivation`, which is used to describe a single derivation: +a build task to run a process on precisely defined input files and repeatably produce output files at uniquely determined file system paths. - - There must be an attribute named [`system`]{#attr-system} whose value must be a - string specifying a Nix system type, such as `"i686-linux"` or - `"x86_64-darwin"`. (To figure out your system type, run `nix -vv - --version`.) The build can only be performed on a machine and - operating system matching the system type. (Nix can automatically - [forward builds for other - platforms](../advanced-topics/distributed-builds.md) by forwarding - them to other machines.) +It takes as input an attribute set, the attributes of which specify the inputs to the build. +It outputs an attribute set, and produces a [store derivation](@docroot@/glossary.md#gloss-store-derivation) as a side effect of evaluation. - - There must be an attribute named `name` whose value must be a - string. This is used as a symbolic name for the package by - `nix-env`, and it is appended to the output paths of the derivation. + - - There must be an attribute named [`builder`]{#attr-builder} that identifies the - program that is executed to perform the build. It can be either a - derivation or a source (a local file reference, e.g., - `./builder.sh`). +## Input attributes - - Every attribute is passed as an environment variable to the builder. - Attribute values are translated to environment variables as follows: - - - Strings and numbers are just passed verbatim. - - - A *path* (e.g., `../foo/sources.tar`) causes the referenced file - to be copied to the store; its location in the store is put in - the environment variable. The idea is that all sources should - reside in the Nix store, since all inputs to a derivation should - reside in the Nix store. - - - A *derivation* causes that derivation to be built prior to the - present derivation; its default output path is put in the - environment variable. - - - Lists of the previous types are also allowed. They are simply - concatenated, separated by spaces. - - - `true` is passed as the string `1`, `false` and `null` are - passed as an empty string. +### Required - - The optional attribute `args` specifies command-line arguments to be - passed to the builder. It should be a list. +- [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) - - The optional attribute `outputs` specifies a list of symbolic - outputs of the derivation. By default, a derivation produces a - single output path, denoted as `out`. However, derivations can - produce multiple output paths. This is useful because it allows - outputs to be downloaded or garbage-collected separately. For - instance, imagine a library package that provides a dynamic library, - header files, and documentation. A program that links against the - library doesn’t need the header files and documentation at runtime, - and it doesn’t need the documentation at build time. Thus, the - library package could specify: - - ```nix - outputs = [ "lib" "headers" "doc" ]; - ``` - - This will cause Nix to pass environment variables `lib`, `headers` - and `doc` to the builder containing the intended store paths of each - output. The builder would typically do something like - - ```bash - ./configure \ - --libdir=$lib/lib \ - --includedir=$headers/include \ - --docdir=$doc/share/doc - ``` - - for an Autoconf-style package. You can refer to each output of a - derivation by selecting it as an attribute, e.g. - - ```nix - buildInputs = [ pkg.lib pkg.headers ]; - ``` - - The first element of `outputs` determines the *default output*. - Thus, you could also write - - ```nix - buildInputs = [ pkg pkg.headers ]; - ``` - - since `pkg` is equivalent to `pkg.lib`. + Symbolic name for the derivation. + It is appended to the [store paths](@docroot@/glossary.md#gloss-store-path) of the resulting store derivation and its [output paths][output path]. -The function `mkDerivation` in the Nixpkgs standard environment is a -wrapper around `derivation` that adds a default value for `system` and -always uses Bash as the builder, to which the supplied builder is passed -as a command-line argument. See the Nixpkgs manual for details. + Example: `name = "hello";` + +- [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) + + The system type on which the [`builder`](#attr-builder) executable can be run. + + Nix will only build derivations where the `system` attribute matches the current [`system` configuration option]. + It can automatically [build on other platforms](../advanced-topics/distributed-builds.md) by forwarding build requests to other machines. + + Examples: + + `system = "x86_64-linux";` + + `system = builtins.currentSystem;` + + [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. + + [`system` configuration option]: @docroot@/command-ref/conf-file.md#conf-system + +- [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string)) + + Path to an executable that will perform the build. + + Examples: + + `builder = "/bin/bash";` + + `builder = ./builder.sh;` + + `builder = "${pkgs.python}/bin/python";` + +### Optional + +- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) Default: `[ ]` + + Command-line arguments to be passed to the [`builder`](#attr-builder) executable. + + Example: `args = [ "-c" "echo hello world > $out" ];` + +- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) Default: `[ "out" ]` + + Symbolic outputs of the derivation. + Each output name is passed to the [`builder`](#attr-builder) executable as an environment variable with its value set to the corresponding [output path]. + + [output path]: @docroot@/glossary.md#gloss-output-path + + By default, a derivation produces a single output path called `out`. + However, derivations can produce multiple output paths. + This allows the associated [store objects](@docroot@/glossary.md#gloss-store-object) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. + + Examples: + + Imagine a library package that provides a dynamic library, header files, and documentation. + A program that links against the library doesn’t need the header files and documentation at runtime, and it doesn’t need the documentation at build time. + Thus, the library package could specify: + + ```nix + outputs = [ "lib" "headers" "doc" ]; + ``` + + This will cause Nix to pass environment variables `lib`, `headers`, and `doc` to the builder containing the intended store paths of each output. + The builder would typically do something like + + ```bash + ./configure \ + --libdir=$lib/lib \ + --includedir=$headers/include \ + --docdir=$doc/share/doc + ``` + + for an Autoconf-style package. + + You can refer to each output of a + derivation by selecting it as an attribute, e.g. + + ```nix + buildInputs = [ pkg.lib pkg.headers ]; + ``` + + + The first element of `outputs` determines the *default output*. + Thus, you could also write + + ```nix + buildInputs = [ pkg pkg.headers ]; + ``` + + since `pkg` is equivalent to `pkg.lib`. + +- See [Advanced Attributes](./advanced-attributes.md) for more, infrequently used, optional attributes. + + + +- Every other attribute is passed as an environment variable to the builder. + Attribute values are translated to environment variables as follows: + + - Strings and numbers are just passed verbatim. + + - A *path* (e.g., `../foo/sources.tar`) causes the referenced file + to be copied to the store; its location in the store is put in + the environment variable. The idea is that all sources should + reside in the Nix store, since all inputs to a derivation should + reside in the Nix store. + + - A *derivation* causes that derivation to be built prior to the + present derivation; its default output path is put in the + environment variable. + + - Lists of the previous types are also allowed. They are simply + concatenated, separated by spaces. + + - `true` is passed as the string `1`, `false` and `null` are + passed as an empty string. ## Builder execution From d621dd17f23023d4d6a84daba90ef38e848a1375 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 15 Sep 2023 10:31:31 +0200 Subject: [PATCH 307/909] more precise wording Co-authored-by: Robert Hensing --- doc/manual/src/language/derivations.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 56c6818ca..dfefe446f 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -14,14 +14,14 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar - [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) - Symbolic name for the derivation. - It is appended to the [store paths](@docroot@/glossary.md#gloss-store-path) of the resulting store derivation and its [output paths][output path]. + A symbolic name for the derivation. + It is added to the [store derivation]'s [path](@docroot@/glossary.md#gloss-store-path) and its [output paths][output path]. Example: `name = "hello";` - [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) - The system type on which the [`builder`](#attr-builder) executable can be run. + The system type on which the [`builder`](#attr-builder) executable is meant to be run. Nix will only build derivations where the `system` attribute matches the current [`system` configuration option]. It can automatically [build on other platforms](../advanced-topics/distributed-builds.md) by forwarding build requests to other machines. From 026c24e3787d7b1cf81c49fdcb66818fd5dad2e4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 15 Sep 2023 15:59:07 +0200 Subject: [PATCH 308/909] add example for store path using the given name --- doc/manual/src/language/derivations.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index dfefe446f..3ced976f6 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -19,6 +19,7 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar Example: `name = "hello";` + The store derivation's path will be `/nix/store/-hello.drv`, and the output paths will be of the form `/nix/store/-hello[-]` - [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) The system type on which the [`builder`](#attr-builder) executable is meant to be run. From 17884f54d18b6d69dbfefe6c08272ea51b8879f8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 22 Sep 2023 09:53:29 +0200 Subject: [PATCH 309/909] clarification on extra attributes Co-authored-by: Robert Hensing --- doc/manual/src/language/derivations.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 3ced976f6..77eac03b6 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -114,7 +114,11 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar - Every other attribute is passed as an environment variable to the builder. Attribute values are translated to environment variables as follows: - - Strings and numbers are just passed verbatim. + - Strings are passed unchanged. + + - Integral numbers are converted to decimal notation. + + - Floating point numbers are converted to simple decimal or scientific notation with a preset precision. - A *path* (e.g., `../foo/sources.tar`) causes the referenced file to be copied to the store; its location in the store is put in From 5e4734a08b8fb3f80b9617c190a2b5dfaba4a1da Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 00:13:56 +0200 Subject: [PATCH 310/909] reword to avoid saying "build" derivations are about data transformation, so the term "build" does not add any information. there was also some feedback that "build task" is not more helpful than "derivation" if you have no prior experience with Nix or build systems, while existing associations may be misleading. --- doc/manual/src/language/derivations.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 77eac03b6..da3cca4ad 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -1,9 +1,9 @@ # Derivations The most important built-in function is `derivation`, which is used to describe a single derivation: -a build task to run a process on precisely defined input files and repeatably produce output files at uniquely determined file system paths. +a specification for running an executable on precisely defined input files to repeatably produce output files at uniquely determined file system paths. -It takes as input an attribute set, the attributes of which specify the inputs to the build. +It takes as input an attribute set, the attributes of which specify the inputs to the process. It outputs an attribute set, and produces a [store derivation](@docroot@/glossary.md#gloss-store-derivation) as a side effect of evaluation. From 75a231147f2d6acd32321e088a5030a69f188854 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 00:43:41 +0200 Subject: [PATCH 311/909] be more precise about `system` semantics --- doc/manual/src/language/derivations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index da3cca4ad..c13c81e7d 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -24,7 +24,7 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar The system type on which the [`builder`](#attr-builder) executable is meant to be run. - Nix will only build derivations where the `system` attribute matches the current [`system` configuration option]. + A necessary condition for Nix to build derivations locally is that the `system` attribute matches the current [`system` configuration option]. It can automatically [build on other platforms](../advanced-topics/distributed-builds.md) by forwarding build requests to other machines. Examples: From 7de66f19f84edc0d617e00f6e0cc37f2dbc4f5f6 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 00:46:26 +0200 Subject: [PATCH 312/909] example: headers -> dev make the example more realistic, since `headers` is not an output name used in Nixpkgs Co-authored-by: Robert Hensing --- doc/manual/src/language/derivations.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index c13c81e7d..6e6d3298b 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -75,16 +75,16 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar Thus, the library package could specify: ```nix - outputs = [ "lib" "headers" "doc" ]; + outputs = [ "lib" "dev" "doc" ]; ``` - This will cause Nix to pass environment variables `lib`, `headers`, and `doc` to the builder containing the intended store paths of each output. + This will cause Nix to pass environment variables `lib`, `dev`, and `doc` to the builder containing the intended store paths of each output. The builder would typically do something like ```bash ./configure \ --libdir=$lib/lib \ - --includedir=$headers/include \ + --includedir=$dev/include \ --docdir=$doc/share/doc ``` @@ -94,7 +94,7 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar derivation by selecting it as an attribute, e.g. ```nix - buildInputs = [ pkg.lib pkg.headers ]; + buildInputs = [ pkg.lib pkg.dev ]; ``` @@ -102,7 +102,7 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar Thus, you could also write ```nix - buildInputs = [ pkg pkg.headers ]; + buildInputs = [ pkg pkg.dev ]; ``` since `pkg` is equivalent to `pkg.lib`. From 5b0336b3b14551762f37126423ea8b4590ebb560 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 00:58:50 +0200 Subject: [PATCH 313/909] reword example for clarity --- doc/manual/src/language/derivations.md | 28 ++++++++++---------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 6e6d3298b..e0b9d021a 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -71,11 +71,15 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar Examples: Imagine a library package that provides a dynamic library, header files, and documentation. - A program that links against the library doesn’t need the header files and documentation at runtime, and it doesn’t need the documentation at build time. + A program that links against such a library doesn’t need the header files and documentation at runtime, and it doesn’t need the documentation at build time. Thus, the library package could specify: ```nix - outputs = [ "lib" "dev" "doc" ]; + derivation { + # ... + outputs = [ "lib" "dev" "doc" ]; + # ... + } ``` This will cause Nix to pass environment variables `lib`, `dev`, and `doc` to the builder containing the intended store paths of each output. @@ -90,22 +94,12 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar for an Autoconf-style package. - You can refer to each output of a - derivation by selecting it as an attribute, e.g. - - ```nix - buildInputs = [ pkg.lib pkg.dev ]; - ``` - + You can refer to each output of a derivation by selecting it as an attribute, e.g. `myPackage.lib` or `myPackage.doc`. The first element of `outputs` determines the *default output*. - Thus, you could also write + Therefore, in the given example, `myPackage` is equivalent to `myPackage.lib`. - ```nix - buildInputs = [ pkg pkg.dev ]; - ``` - - since `pkg` is equivalent to `pkg.lib`. + - See [Advanced Attributes](./advanced-attributes.md) for more, infrequently used, optional attributes. @@ -115,9 +109,9 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar Attribute values are translated to environment variables as follows: - Strings are passed unchanged. - + - Integral numbers are converted to decimal notation. - + - Floating point numbers are converted to simple decimal or scientific notation with a preset precision. - A *path* (e.g., `../foo/sources.tar`) causes the referenced file From 887cbcd3958abed7cf726f7eb23ba530e021ddfb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:53:50 +0200 Subject: [PATCH 314/909] add contributing guide for documentation --- doc/manual/local.mk | 6 +- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/contributing/contributing.md | 9 +- doc/manual/src/contributing/documentation.md | 181 +++++++++++++++++++ doc/manual/src/contributing/hacking.md | 51 ------ 5 files changed, 195 insertions(+), 53 deletions(-) create mode 100644 doc/manual/src/contributing/documentation.md diff --git a/doc/manual/local.mk b/doc/manual/local.mk index abdfd6a62..ab95af53c 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -173,6 +173,10 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ +# the `! -name 'contributing.md'` filter excludes the one place where +# `@docroot@` is to be preserved for documenting the mechanism +# FIXME: maybe contributing guides should live right next to the code +# instead of in the manual $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(trace-gen) \ tmp="$$(mktemp -d)"; \ @@ -180,7 +184,7 @@ $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/ find "$$tmp" -name '*.md' | while read -r file; do \ $(call process-includes,$$file,$$file); \ done; \ - find "$$tmp" -name '*.md' | while read -r file; do \ + find "$$tmp" -name '*.md' ! -name 'documentation.md' | while read -r file; do \ docroot="$$(realpath --relative-to="$$(dirname "$$file")" $$tmp/manual/src)"; \ sed -i "s,@docroot@,$$docroot,g" "$$file"; \ done; \ diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c23186511..5c3667690 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -105,6 +105,7 @@ - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) - [Testing](contributing/testing.md) + - [Documentation](contributing/documentation.md) - [Experimental Features](contributing/experimental-features.md) - [CLI guideline](contributing/cli-guideline.md) - [C++ style guide](contributing/cxx.md) diff --git a/doc/manual/src/contributing/contributing.md b/doc/manual/src/contributing/contributing.md index 854139a31..4d55c17a4 100644 --- a/doc/manual/src/contributing/contributing.md +++ b/doc/manual/src/contributing/contributing.md @@ -1 +1,8 @@ -# Contributing +# Development + +Nix is developed on GitHub. +Check the [contributing guide](https://github.com/NixOS/nix/blob/master/CONTRIBUTING.md) if you want to get involved. + +This chapter is a collection of guides for making changes to the code and documentation. + +If you're not sure where to start, try to [compile Nix from source](./hacking.md) and consider [making improvements to documentation](./documentation.md). diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/contributing/documentation.md new file mode 100644 index 000000000..f73ab2149 --- /dev/null +++ b/doc/manual/src/contributing/documentation.md @@ -0,0 +1,181 @@ +# Contributing documentation + +Improvements to documentation are very much appreciated, and a good way to start out with contributing to Nix. + +This is how you can help: +- Address [open issues with documentation](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+label%3Adocumentation) +- Review [pull requests concerning documentation](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+label%3Adocumentation) + +Incremental refactorings of the documentation build setup to make it faster or easier to understand and maintain are also welcome. + +## Building the manual + +Build the manual from scratch: + +```console +nix-build $(nix-instantiate)'!doc' +``` + +or + +```console +nix build .#^doc +``` + +and open `./result-doc/share/doc/nix/manual/index.html`. + +To build the manual incrementally, [enter the development shell](./hacking.md) and run: + +```console +make manual-html -j $NIX_BUILD_CORES +``` + +and open `./outputs/out/share/doc/nix/manual/language/index.html`. + +In order to reflect changes to the [Makefile for the manual], clear all generated files before re-building: + +[Makefile for the manual]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk + +```console +rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make manual-html -j $NIX_BUILD_CORES +``` + +## Style guide + +The goal of this style guide is to make it such that +- The manual is easy to search and skim for relevant information +- Documentation sources are easy to edit +- Changes to documentation are easy to review + +You will notice that this is not implemented consistently yet. +Please follow the guide when making additions or changes to existing documentation. +Do not make sweeping changes, unless they are programmatic and can be validated easily. + +### Language + +This manual is [reference documentation](https://diataxis.fr/reference/). +The typical usage pattern is to look up isolated pieces of information. +It should therefore aim to be correct, consistent, complete, and easy to navigate at a glance. + +- Aim for clarity and brevity. + + Please take the time to read the [plain language guidelines](https://www.plainlanguage.gov/guidelines/) for details. + +- Describe the subject factually. + + In particular, do not make value judgements or recommendations. + Check the code or add tests if in doubt. + +- Provide complete, minimal examples, and explain them. + + Readers should be able to try examples verbatim and get the same results as shown in the manual. + Always describe in words what a given example does. + + Non-trivial examples may need additional explanation, especially if they use concepts from outside the given context. + +- Use British English. + + This is a somewhat arbitrary choice to force consistency, and accounts for the fact that a majority of Nix users and developers are from Europe. + +### Links and anchors + +Reference documentation must be readable in arbitrary order. +Readers cannot be expected to have any particular prerequisite knowledge about Nix. +While the table of contents can provide guidance and full-text search can help, they are most likely to find what they need by following sensible cross-references. + +- Link to technical terms + + When mentioning Nix-specific concepts, commands, options, settings, etc., link to appropriate documentation. + Also link to external tools or concepts, especially if their meaning may be ambiguous. + You may also want to link to definitions of less common technical terms. + + Then readers won't have to actively search for definitions and are more likely to discover relevant information on their own. + + > **Note** + > + > `man` and `--help` pages don't display links. + > Use appropriate link texts such that readers of terminal output can infer search terms. + +- Do not break existing URLs between releases. + + There are countless links in the wild pointing to old versions of the manual. + We want people to find up-to-date documentation when following popular advice. + + - When moving files, update [redirects on nixos.org](https://github.com/NixOS/nixos-homepage/blob/master/netlify.toml). + + This is especially important when moving information out of the Nix manual to other resources. + + - When changing anchors, update [client-side redirects](https://github.com/NixOS/nix/blob/master/doc/manual/redirects.js) + + The current setup is cumbersome, and help making better automation is appreciated. + +The build checks for broken internal links with. +This happens late in the process, so [building the whole manual](#building-the-manual) is not suitable for iterating quickly. +[`mdbook-linkcheck`] does not implement checking [URI fragments] yet. + +[`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck +[URI fragments]: https://en.wikipedia.org/wiki/URI_fragment + +### Markdown conventions + +The manual is written in markdown, and rendered with [mdBook](https://github.com/rust-lang/mdBook) for the web and with [lowdown](https://github.com/kristapsdz/lowdown) for `man` pages and `--help` output. + +For supported markdown features, refer to: +- [mdBook documentation](https://rust-lang.github.io/mdBook/format/markdown.html) +- [lowdown documentation](https://kristaps.bsd.lv/lowdown/) + +Please observe these guidelines to ease reviews: + +- Write one sentence per line. + + This makes long sentences immediately visible, and makes it easier to review changes and make direct suggestions. + +- Use reference links – sparingly – to ease source readability. + Put definitions close to their first use. + + Example: + + ``` + A [store object] contains a [file system object] and [references] to other store objects. + + [store object]: @docroot@/glossary.md#gloss-store-object + [file system object]: @docroot@/architecture/file-system-object.md + [references]: @docroot@/glossary.md#gloss-reference + ``` + +- Use admonitions of the following form: + + ``` + > **Note** + > + > This is a note. + ``` + +### The `@docroot@` variable + +`@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own. + +If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. + +If the `@docroot@` literal appears in an error message from the [`mdbook-linkcheck`] tool, the `@docroot@` replacement needs to be applied to the generated source file that mentions it. +See existing `@docroot@` logic in the [Makefile for the manual]. +Regular markdown files used for the manual have a base path of their own and they can use relative paths instead of `@docroot@`. + +## API documentation + +[Doxygen API documentation] is available online. +You can also build and view it yourself: + +[Doxygen API documentation]: https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs + +```console +# nix build .#hydraJobs.internal-api-docs +# xdg-open ./result/share/doc/nix/internal-api/html/index.html +``` + +or inside `nix-shell` or `nix develop`: + +``` +# make internal-api-html +# xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html +``` diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index a2446b53a..db393942c 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -220,54 +220,3 @@ Configure your editor to use the `clangd` from the shell, either by running it i > For some editors (e.g. Visual Studio Code), you may need to install a [special extension](https://open-vsx.org/extension/llvm-vs-code-extensions/vscode-clangd) for the editor to interact with `clangd`. > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). > Editor-specific setup is typically opinionated, so we will not cover it here in more detail. - -### Checking links in the manual - -The build checks for broken internal links. -This happens late in the process, so `nix build` is not suitable for iterating. -To build the manual incrementally, run: - -```console -make manual-html -j $NIX_BUILD_CORES -``` - -In order to reflect changes to the [Makefile], clear all generated files before re-building: - -[Makefile]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk - -```console -rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make html -j $NIX_BUILD_CORES -``` - -[`mdbook-linkcheck`] does not implement checking [URI fragments] yet. - -[`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck -[URI fragments]: https://en.wikipedia.org/wiki/URI_fragment - -#### `@docroot@` variable - -`@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own. - -If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. - -If the `@docroot@` literal appears in an error message from the `mdbook-linkcheck` tool, the `@docroot@` replacement needs to be applied to the generated source file that mentions it. -See existing `@docroot@` logic in the [Makefile]. -Regular markdown files used for the manual have a base path of their own and they can use relative paths instead of `@docroot@`. - -## API documentation - -Doxygen API documentation is [available -online](https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs). You -can also build and view it yourself: - -```console -# nix build .#hydraJobs.internal-api-docs -# xdg-open ./result/share/doc/nix/internal-api/html/index.html -``` - -or inside a `nix develop` shell by running: - -``` -# make internal-api-html -# xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html -``` From 45de35bcf14d1bdad5dca2e24a113d26d1985aed Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 01:24:07 +0200 Subject: [PATCH 315/909] fix broken redirects script --- doc/manual/redirects.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index b43622ed6..b2fad19bb 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -343,7 +343,7 @@ const redirects = { "linux": "uninstall.html#linux", "macos": "uninstall.html#macos", "uninstalling": "uninstall.html", - } + }, "contributing/hacking.html": { "nix-with-flakes": "#building-nix-with-flakes", "classic-nix": "#building-nix", From c6f8247032612b4cc4a9f79213a887aa9ddddfda Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 03:10:12 +0200 Subject: [PATCH 316/909] fix broken reference link --- doc/manual/src/language/derivations.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index e0b9d021a..4a1b70fd1 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -4,7 +4,9 @@ The most important built-in function is `derivation`, which is used to describe a specification for running an executable on precisely defined input files to repeatably produce output files at uniquely determined file system paths. It takes as input an attribute set, the attributes of which specify the inputs to the process. -It outputs an attribute set, and produces a [store derivation](@docroot@/glossary.md#gloss-store-derivation) as a side effect of evaluation. +It outputs an attribute set, and produces a [store derivation] as a side effect of evaluation. + +[store derivation]: @docroot@/glossary.md#gloss-store-derivation From 7e24dc606b1cf7fd4e4dfd7027ba1469a59784d5 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 22 Sep 2023 13:05:55 -0400 Subject: [PATCH 317/909] fix(tests): fix assumption that string.s is a char* --- src/libexpr/tests/libexpr.hh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/tests/libexpr.hh b/src/libexpr/tests/libexpr.hh index b8e65aafe..97b05ac46 100644 --- a/src/libexpr/tests/libexpr.hh +++ b/src/libexpr/tests/libexpr.hh @@ -71,7 +71,7 @@ namespace nix { if (arg.type() != nString) { return false; } - return std::string_view(arg.string.s) == s; + return std::string_view(arg.c_str()) == s; } MATCHER_P(IsIntEq, v, fmt("The string is equal to \"%1%\"", v)) { @@ -106,8 +106,8 @@ namespace nix { if (arg.type() != nPath) { *result_listener << "Expected a path got " << arg.type(); return false; - } else if (std::string_view(arg.string.s) != p) { - *result_listener << "Expected a path that equals \"" << p << "\" but got: " << arg.string.s; + } else if (std::string_view(arg._path) != p) { + *result_listener << "Expected a path that equals \"" << p << "\" but got: " << arg.c_str(); return false; } return true; From b17f200b11b3901c4f975c9901f1e2c6fc54124f Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 03:49:03 +0200 Subject: [PATCH 318/909] Document "Import From Derivation" (#7332) * document "Import From Derivation" Co-authored-by: Robert Hensing Co-authored-by: John Ericson --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/glossary.md | 5 +- .../src/language/import-from-derivation.md | 139 ++++++++++++++++++ src/libexpr/eval-settings.hh | 11 +- 4 files changed, 150 insertions(+), 6 deletions(-) create mode 100644 doc/manual/src/language/import-from-derivation.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c23186511..9e2ad0618 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -33,6 +33,7 @@ - [Operators](language/operators.md) - [Derivations](language/derivations.md) - [Advanced Attributes](language/advanced-attributes.md) + - [Import From Derivation](language/import-from-derivation.md) - [Built-in Constants](language/builtin-constants.md) - [Built-in Functions](language/builtins.md) - [Advanced Topics](advanced-topics/advanced-topics.md) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index b7c340d36..c23466351 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -105,12 +105,15 @@ - [store object]{#gloss-store-object} - A store object consists of a [file system object], [reference]s to other store objects, and other metadata. It can be referred to by a [store path]. [store object]: #gloss-store-object +- [IFD]{#gloss-ifd} + + [Import From Derivation](./language/import-from-derivation.md) + - [input-addressed store object]{#gloss-input-addressed-store-object} A store object produced by building a diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md new file mode 100644 index 000000000..03b3f9d91 --- /dev/null +++ b/doc/manual/src/language/import-from-derivation.md @@ -0,0 +1,139 @@ +# Import From Derivation + +The value of a Nix expression can depend on the contents of a [store object](@docroot@/glossary.md#gloss-store-object). + +Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary.md#gloss-store-path) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): + +- [`import`](./builtins.md#builtins-import)` expr` +- [`builtins.readFile`](./builtins.md#builtins-readFile)` expr` +- [`builtins.readFileType`](./builtins.md#builtins-readFileType)` expr` +- [`builtins.readDir`](./builtins.md#builtins-readDir)` expr` +- [`builtins.pathExists`](./builtins.md#builtins-pathExists)` expr` +- [`builtins.filterSource`](./builtins.md#builtins-filterSource)` f expr` +- [`builtins.path`](./builtins.md#builtins-path)` { path = expr; }` +- [`builtins.hashFile`](./builtins.md#builtins-hashFile)` t expr` +- `builtins.scopedImport x drv` + +When the store path needs to be accessed, evaluation will be paused, the corresponding store object [realised], and then evaluation resumed. + +[realised]: @docroot@/glossary.md#gloss-realise + +This has performance implications: +Evaluation can only finish when all required store objects are realised. +Since the Nix language evaluator is sequential, it only finds store paths to read from one at a time. +While realisation is always parallel, in this case it cannot be done for all required store paths at once, and is therefore much slower than otherwise. + +Realising store objects during evaluation can be disabled by setting [`allow-import-from-derivation`](../command-ref/conf-file.md#conf-allow-import-from-derivation) to `false`. +Without IFD it is ensured that evaluation is complete and Nix can produce a build plan before starting any realisation. + +## Example + +In the following Nix expression, the inner derivation `drv` produces a file with contents `hello`. + +```nix +# IFD.nix +let + drv = derivation { + name = "hello"; + builder = "/bin/sh"; + args = [ "-c" "echo -n hello > $out" ]; + system = builtins.currentSystem; + }; +in "${builtins.readFile drv} world" +``` + +```shellSession +nix-instantiate IFD.nix --eval --read-write-mode +``` + +``` +building '/nix/store/348q1cal6sdgfxs8zqi9v8llrsn4kqkq-hello.drv'... +"hello world" +``` + +The contents of the derivation's output have to be [realised] before they can be read with [`readFile`](./builtins.md#builtins-readFile). +Only then evaluation can continue to produce the final result. + +## Illustration + +As a first approximation, the following data flow graph shows how evaluation and building are interleaved, if the value of a Nix expression depends on realising a [store object]. +Boxes are data structures, arrow labels are transformations. + +``` ++----------------------+ +------------------------+ +| Nix evaluator | | Nix store | +| .----------------. | | | +| | Nix expression | | | | +| '----------------' | | | +| | | | | +| evaluate | | | +| | | | | +| V | | | +| .------------. | | .------------------. | +| | derivation |----|-instantiate-|->| store derivation | | +| '------------' | | '------------------' | +| | | | | +| | | realise | +| | | | | +| | | V | +| .----------------. | | .--------------. | +| | Nix expression |<-|----read-----|----| store object | | +| '----------------' | | '--------------' | +| | | | | +| evaluate | | | +| | | | | +| V | | | +| .------------. | | | +| | value | | | | +| '------------' | | | ++----------------------+ +------------------------+ +``` + +In more detail, the following sequence diagram shows how the expression is evaluated step by step, and where evaluation is blocked to wait for the build output to appear. + +``` +.-------. .-------------. .---------. +|Nix CLI| |Nix evaluator| |Nix store| +'-------' '-------------' '---------' + | | | + |evaluate IFD.nix| | + |--------------->| | + | | | + | evaluate `"${readFile drv} world"` | + | | | + | evaluate `readFile drv` | + | | | + | evaluate `drv` as string | + | | | + | |instantiate /nix/store/...-hello.drv| + | |----------------------------------->| + | : | + | : realise /nix/store/...-hello.drv | + | :----------------------------------->| + | : | + | |--------. + | : | | + | (evaluation blocked) | echo hello > $out + | : | | + | |<-------' + | : /nix/store/...-hello | + | |<-----------------------------------| + | | | + | resume `readFile /nix/store/...-hello` | + | | | + | | readFile /nix/store/...-hello | + | |----------------------------------->| + | | | + | | hello | + | |<-----------------------------------| + | | | + | resume `"${"hello"} world"` | + | | | + | resume `"hello world"` | + | | | + | "hello world" | | + |<---------------| | +.-------. .-------------. .---------. +|Nix CLI| |Nix evaluator| |Nix store| +'-------' '-------------' '---------' +``` diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 19122bc31..19ba679be 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -47,11 +47,12 @@ struct EvalSettings : Config Setting enableImportFromDerivation{ this, true, "allow-import-from-derivation", R"( - By default, Nix allows you to `import` from a derivation, allowing - building at evaluation time. With this option set to false, Nix will - throw an error when evaluating an expression that uses this feature, - allowing users to ensure their evaluation will not require any - builds to take place. + By default, Nix allows [Import from Derivation](@docroot@/language/import-from-derivation.md). + + With this option set to `false`, Nix will throw an error when evaluating an expression that uses this feature, + even when the required store object is readily available. + This ensures that evaluation will not require any builds to take place, + regardless of the state of the store. )"}; Setting allowedUris{this, {}, "allowed-uris", From a757749fcfa810f699fa096f98821efccba86522 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 06:28:17 +0200 Subject: [PATCH 319/909] reword for readability --- doc/manual/src/release-notes/release-notes.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/release-notes.md b/doc/manual/src/release-notes/release-notes.md index 1fa2f5580..cc805e631 100644 --- a/doc/manual/src/release-notes/release-notes.md +++ b/doc/manual/src/release-notes/release-notes.md @@ -6,6 +6,7 @@ Notable changes and additions are announced in the release notes for each versio Bugfixes can be backported on request to previous Nix releases. We typically backport only as far back as the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). -Backports never skip releases (if a feature is backported to `x.y`, it must also be available in `x.(y+1)`; +Backports never skip releases. +If a feature is backported to version `x.y`, it must also be available in version `x.(y+1)`. This ensures that upgrading from an older version with backports is still safe and no backported functionality will go missing. From 5b902ce9d60fca33c0f5bb76fcd31792c0bb03a6 Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Tue, 26 Sep 2023 23:30:32 +0800 Subject: [PATCH 320/909] libexpr: construct ExprPath by move ctor, not copy cotr --- src/libexpr/parser.y | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 792f51fde..509f4fcc4 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -520,7 +520,7 @@ path_start /* add back in the trailing '/' to the first segment */ if ($1.p[$1.l-1] == '/' && $1.l > 1) path += "/"; - $$ = new ExprPath(path); + $$ = new ExprPath(std::move(path)); } | HPATH { if (evalSettings.pureEval) { @@ -530,7 +530,7 @@ path_start ); } Path path(getHome() + std::string($1.p + 1, $1.l - 1)); - $$ = new ExprPath(path); + $$ = new ExprPath(std::move(path)); } ; From 1eeea01931d20d32c917e92d860513b7cb32ba10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christina=20S=C3=B8rensen?= Date: Wed, 27 Sep 2023 02:07:43 +0000 Subject: [PATCH 321/909] Fix repl.md duplicate typo Seems like `legacyPackages.x86_64-linux.emacs.name` is accidentally shown twice. --- src/nix/repl.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/nix/repl.md b/src/nix/repl.md index c5113be61..9dc61fd35 100644 --- a/src/nix/repl.md +++ b/src/nix/repl.md @@ -43,9 +43,6 @@ R""( nix-repl> legacyPackages.x86_64-linux.emacs.name "emacs-27.1" - nix-repl> legacyPackages.x86_64-linux.emacs.name - "emacs-27.1" - nix-repl> :q # nix repl --expr 'import {}' From 399ef8442012308f480ea5841fb6ac732207e03d Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Mon, 25 Sep 2023 21:30:41 -0400 Subject: [PATCH 322/909] refactor: use string accessors Create context, string_view, and c_str, accessors throughout in order to better support improvements to the underlying string representation. --- src/libcmd/repl.cc | 2 +- src/libexpr/eval-cache.cc | 8 ++++---- src/libexpr/eval.cc | 18 +++++++++--------- src/libexpr/flake/flake.cc | 10 +++++----- src/libexpr/get-drvs.cc | 12 ++++++------ src/libexpr/primops.cc | 10 +++++----- src/libexpr/primops/fetchClosure.cc | 2 +- src/libexpr/tests/primops.cc | 4 ++-- src/libexpr/value-to-json.cc | 2 +- src/libexpr/value-to-xml.cc | 6 +++--- src/libexpr/value.hh | 20 +++++++++++++++----- src/nix-env/nix-env.cc | 6 +++--- src/nix/eval.cc | 2 +- 13 files changed, 56 insertions(+), 46 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 944609f22..2e17a29a7 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -922,7 +922,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m case nString: str << ANSI_WARNING; - printLiteralString(str, v.string.s); + printLiteralString(str, v.string_view()); str << ANSI_NORMAL; break; diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 2c9aa5532..391b32a77 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -440,8 +440,8 @@ Value & AttrCursor::forceValue() if (root->db && (!cachedValue || std::get_if(&cachedValue->second))) { if (v.type() == nString) - cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), - string_t{v.string.s, {}}}; + cachedValue = {root->db->setString(getKey(), v.c_str(), v.context()), + string_t{v.c_str(), {}}}; else if (v.type() == nPath) { auto path = v.path().path; cachedValue = {root->db->setString(getKey(), path.abs()), string_t{path.abs(), {}}}; @@ -582,7 +582,7 @@ std::string AttrCursor::getString() if (v.type() != nString && v.type() != nPath) root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); - return v.type() == nString ? v.string.s : v.path().to_string(); + return v.type() == nString ? v.c_str() : v.path().to_string(); } string_t AttrCursor::getStringWithContext() @@ -624,7 +624,7 @@ string_t AttrCursor::getStringWithContext() if (v.type() == nString) { NixStringContext context; copyContext(v, context); - return {v.string.s, std::move(context)}; + return {v.c_str(), std::move(context)}; } else if (v.type() == nPath) return {v.path().to_string(), {}}; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index afa864730..a78c0afb1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -114,7 +114,7 @@ void Value::print(const SymbolTable &symbols, std::ostream &str, printLiteralBool(str, boolean); break; case tString: - printLiteralString(str, string.s); + printLiteralString(str, string_view()); break; case tPath: str << path().to_string(); // !!! escaping? @@ -339,7 +339,7 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) Value nameValue; name.expr->eval(state, env, nameValue); state.forceStringNoCtx(nameValue, noPos, "while evaluating an attribute name"); - return state.symbols.create(nameValue.string.s); + return state.symbols.create(nameValue.string_view()); } } @@ -1343,7 +1343,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) if (nameVal.type() == nNull) continue; state.forceStringNoCtx(nameVal, i.pos, "while evaluating the name of a dynamic attribute"); - auto nameSym = state.symbols.create(nameVal.string.s); + auto nameSym = state.symbols.create(nameVal.string_view()); Bindings::iterator j = v.attrs->find(nameSym); if (j != v.attrs->end()) state.error("dynamic attribute '%1%' already defined at %2%", state.symbols[nameSym], state.positions[j->pos]).atPos(i.pos).withFrame(env, *this).debugThrow(); @@ -2155,7 +2155,7 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string forceValue(v, pos); if (v.type() != nString) error("value is %1% while a string was expected", showType(v)).debugThrow(); - return v.string.s; + return v.string_view(); } catch (Error & e) { e.addTrace(positions[pos], errorCtx); throw; @@ -2182,8 +2182,8 @@ std::string_view EvalState::forceString(Value & v, NixStringContext & context, c std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx) { auto s = forceString(v, pos, errorCtx); - if (v.string.context) { - error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string.s, v.string.context[0]).withTrace(pos, errorCtx).debugThrow(); + if (v.context()) { + error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow(); } return s; } @@ -2196,7 +2196,7 @@ bool EvalState::isDerivation(Value & v) if (i == v.attrs->end()) return false; forceValue(*i->value, i->pos); if (i->value->type() != nString) return false; - return strcmp(i->value->string.s, "derivation") == 0; + return i->value->string_view().compare("derivation") == 0; } @@ -2228,7 +2228,7 @@ BackedStringView EvalState::coerceToString( if (v.type() == nString) { copyContext(v, context); - return std::string_view(v.string.s); + return v.string_view(); } if (v.type() == nPath) { @@ -2426,7 +2426,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v return v1.boolean == v2.boolean; case nString: - return strcmp(v1.string.s, v2.string.s) == 0; + return v1.string_view().compare(v2.string_view()) == 0; case nPath: return strcmp(v1._path, v2._path) == 0; diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 90427e064..a6212c12f 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -113,7 +113,7 @@ static FlakeInput parseFlakeInput(EvalState & state, try { if (attr.name == sUrl) { expectType(state, nString, *attr.value, attr.pos); - url = attr.value->string.s; + url = attr.value->string_view(); attrs.emplace("url", *url); } else if (attr.name == sFlake) { expectType(state, nBool, *attr.value, attr.pos); @@ -122,7 +122,7 @@ static FlakeInput parseFlakeInput(EvalState & state, input.overrides = parseFlakeInputs(state, attr.value, attr.pos, baseDir, lockRootPath); } else if (attr.name == sFollows) { expectType(state, nString, *attr.value, attr.pos); - auto follows(parseInputPath(attr.value->string.s)); + auto follows(parseInputPath(attr.value->c_str())); follows.insert(follows.begin(), lockRootPath.begin(), lockRootPath.end()); input.follows = follows; } else { @@ -131,7 +131,7 @@ static FlakeInput parseFlakeInput(EvalState & state, #pragma GCC diagnostic ignored "-Wswitch-enum" switch (attr.value->type()) { case nString: - attrs.emplace(state.symbols[attr.name], attr.value->string.s); + attrs.emplace(state.symbols[attr.name], attr.value->c_str()); break; case nBool: attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean }); @@ -229,7 +229,7 @@ static Flake getFlake( if (auto description = vInfo.attrs->get(state.sDescription)) { expectType(state, nString, *description->value, description->pos); - flake.description = description->value->string.s; + flake.description = description->value->c_str(); } auto sInputs = state.symbols.create("inputs"); @@ -850,7 +850,7 @@ static void prim_flakeRefToString( Explicit { attr.value->boolean }); } else if (t == nString) { attrs.emplace(state.symbols[attr.name], - std::string(attr.value->str())); + std::string(attr.value->string_view())); } else { state.error( "flake reference attribute sets may only contain integers, Booleans, " diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 506a63677..fe3e6f7ee 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -156,7 +156,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall Outputs result; for (auto elem : outTI->listItems()) { if (elem->type() != nString) throw errMsg; - auto out = outputs.find(elem->string.s); + auto out = outputs.find(elem->c_str()); if (out == outputs.end()) throw errMsg; result.insert(*out); } @@ -230,7 +230,7 @@ std::string DrvInfo::queryMetaString(const std::string & name) { Value * v = queryMeta(name); if (!v || v->type() != nString) return ""; - return v->string.s; + return v->c_str(); } @@ -242,7 +242,7 @@ NixInt DrvInfo::queryMetaInt(const std::string & name, NixInt def) if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ - if (auto n = string2Int(v->string.s)) + if (auto n = string2Int(v->c_str())) return *n; } return def; @@ -256,7 +256,7 @@ NixFloat DrvInfo::queryMetaFloat(const std::string & name, NixFloat def) if (v->type() == nString) { /* Backwards compatibility with before we had support for float meta fields. */ - if (auto n = string2Float(v->string.s)) + if (auto n = string2Float(v->c_str())) return *n; } return def; @@ -271,8 +271,8 @@ bool DrvInfo::queryMetaBool(const std::string & name, bool def) if (v->type() == nString) { /* Backwards compatibility with before we had support for Boolean meta fields. */ - if (strcmp(v->string.s, "true") == 0) return true; - if (strcmp(v->string.s, "false") == 0) return false; + if (v->string_view() == "true") return true; + if (v->string_view() == "false") return false; } return def; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index cd9a05bb2..1cb5cdf30 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -590,7 +590,7 @@ struct CompareValues case nFloat: return v1->fpoint < v2->fpoint; case nString: - return strcmp(v1->string.s, v2->string.s) < 0; + return v1->string_view().compare(v2->string_view()) < 0; case nPath: return strcmp(v1->_path, v2->_path) < 0; case nList: @@ -982,7 +982,7 @@ static void prim_trace(EvalState & state, const PosIdx pos, Value * * args, Valu { state.forceValue(*args[0], pos); if (args[0]->type() == nString) - printError("trace: %1%", args[0]->string.s); + printError("trace: %1%", args[0]->string_view()); else printError("trace: %1%", printValue(state, *args[0])); state.forceValue(*args[1], pos); @@ -1528,7 +1528,7 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, auto path = realisePath(state, pos, arg, { .checkForPureEval = false }); /* SourcePath doesn't know about trailing slash. */ - auto mustBeDir = arg.type() == nString && arg.str().ends_with("/"); + auto mustBeDir = arg.type() == nString && arg.string_view().ends_with("/"); try { auto checked = state.checkSourcePath(path); @@ -2400,7 +2400,7 @@ static void prim_attrNames(EvalState & state, const PosIdx pos, Value * * args, (v.listElems()[n++] = state.allocValue())->mkString(state.symbols[i.name]); std::sort(v.listElems(), v.listElems() + n, - [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); + [](Value * v1, Value * v2) { return v1->string_view().compare(v2->string_view()) < 0; }); } static RegisterPrimOp primop_attrNames({ @@ -2541,7 +2541,7 @@ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args names.reserve(args[1]->listSize()); for (auto elem : args[1]->listItems()) { state.forceStringNoCtx(*elem, pos, "while evaluating the values of the second argument passed to builtins.removeAttrs"); - names.emplace_back(state.symbols.create(elem->string.s), nullptr); + names.emplace_back(state.symbols.create(elem->string_view()), nullptr); } std::sort(names.begin(), names.end()); diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 7fe8203f4..b86ef6b93 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -133,7 +133,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg else if (attrName == "toPath") { state.forceValue(*attr.value, attr.pos); - bool isEmptyString = attr.value->type() == nString && attr.value->string.s == std::string(""); + bool isEmptyString = attr.value->type() == nString && attr.value->string_view() == ""; if (isEmptyString) { toPath = StorePathOrGap {}; } diff --git a/src/libexpr/tests/primops.cc b/src/libexpr/tests/primops.cc index ce3b5d11f..d820b860e 100644 --- a/src/libexpr/tests/primops.cc +++ b/src/libexpr/tests/primops.cc @@ -711,14 +711,14 @@ namespace nix { // FIXME: add a test that verifies the string context is as expected auto v = eval("builtins.replaceStrings [\"oo\" \"a\"] [\"a\" \"i\"] \"foobar\""); ASSERT_EQ(v.type(), nString); - ASSERT_EQ(v.string.s, std::string_view("fabir")); + ASSERT_EQ(v.string_view(), "fabir"); } TEST_F(PrimOpTest, concatStringsSep) { // FIXME: add a test that verifies the string context is as expected auto v = eval("builtins.concatStringsSep \"%\" [\"foo\" \"bar\" \"baz\"]"); ASSERT_EQ(v.type(), nString); - ASSERT_EQ(std::string_view(v.string.s), "foo%bar%baz"); + ASSERT_EQ(v.string_view(), "foo%bar%baz"); } TEST_F(PrimOpTest, split1) { diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index ac3986c87..cbc91f509 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -31,7 +31,7 @@ json printValueAsJSON(EvalState & state, bool strict, case nString: copyContext(v, context); - out = v.string.s; + out = v.c_str(); break; case nPath: diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 2539ad1c1..bd7a4ae30 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -74,7 +74,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, case nString: /* !!! show the context? */ copyContext(v, context); - doc.writeEmptyElement("string", singletonAttrs("value", v.string.s)); + doc.writeEmptyElement("string", singletonAttrs("value", v.c_str())); break; case nPath: @@ -96,14 +96,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) - xmlAttrs["drvPath"] = drvPath = a->value->string.s; + xmlAttrs["drvPath"] = drvPath = a->value->c_str(); } a = v.attrs->find(state.sOutPath); if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) - xmlAttrs["outPath"] = a->value->string.s; + xmlAttrs["outPath"] = a->value->c_str(); } XMLOpenElement _(doc, "derivation", xmlAttrs); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index c44683e50..7a1165cac 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -186,10 +186,9 @@ public: * For canonicity, the store paths should be in sorted order. */ struct { - const char * s; + const char * c_str; const char * * context; // must be in sorted order } string; - const char * _path; Bindings * attrs; struct { @@ -270,7 +269,7 @@ public: inline void mkString(const char * s, const char * * context = 0) { internalType = tString; - string.s = s; + string.c_str = s; string.context = context; } @@ -441,10 +440,21 @@ public: return SourcePath{CanonPath(_path)}; } - std::string_view str() const + std::string_view string_view() const { assert(internalType == tString); - return std::string_view(string.s); + return std::string_view(string.c_str); + } + + const char * const c_str() const + { + assert(internalType == tString); + return string.c_str; + } + + const char * * context() const + { + return string.context; } }; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index b112e8cb3..e455a50fd 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1228,7 +1228,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) else { if (v->type() == nString) { attrs2["type"] = "string"; - attrs2["value"] = v->string.s; + attrs2["value"] = v->c_str(); xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nInt) { attrs2["type"] = "int"; @@ -1248,7 +1248,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) for (auto elem : v->listItems()) { if (elem->type() != nString) continue; XMLAttrs attrs3; - attrs3["value"] = elem->string.s; + attrs3["value"] = elem->c_str(); xml.writeEmptyElement("string", attrs3); } } else if (v->type() == nAttrs) { @@ -1260,7 +1260,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) if(a.value->type() != nString) continue; XMLAttrs attrs3; attrs3["type"] = globals.state->symbols[i.name]; - attrs3["value"] = a.value->string.s; + attrs3["value"] = a.value->c_str(); xml.writeEmptyElement("string", attrs3); } } diff --git a/src/nix/eval.cc b/src/nix/eval.cc index d880bef0a..b34af34e0 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -85,7 +85,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption state->forceValue(v, pos); if (v.type() == nString) // FIXME: disallow strings with contexts? - writeFile(path, v.string.s); + writeFile(path, v.string_view()); else if (v.type() == nAttrs) { if (mkdir(path.c_str(), 0777) == -1) throw SysError("creating directory '%s'", path); From 5bc540a8ca7ada282839ab58225a1a24cc723c4e Mon Sep 17 00:00:00 2001 From: Alex Ameen Date: Wed, 27 Sep 2023 14:49:52 -0500 Subject: [PATCH 323/909] Respect `NOCOLOR` While `nix` has always been respectful towards requests for `NO_COLOR=1`, this change asks represents a new stage of maturity for `nix` - making it also respect quests for `NOCOLOR=1`. This ideally makes the tool more accessible to folks like me, who are exhausted by guessing whether `NO_COLOR` or `NOCOLOR` is the right environment variable to set. <3 --- src/libutil/util.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 5a10c69e2..3b4c181e5 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1515,7 +1515,7 @@ bool shouldANSI() { return isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb" - && !getEnv("NO_COLOR").has_value(); + && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value()); } std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) From dafa38213bda956525fb692eea33f2ec4fed7ff3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 27 Sep 2023 23:10:39 +0100 Subject: [PATCH 324/909] Update doc/manual/src/advanced-topics/post-build-hook.md Co-authored-by: Valentin Gagarin --- doc/manual/src/advanced-topics/post-build-hook.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md index 17b67111c..f7b3b1317 100644 --- a/doc/manual/src/advanced-topics/post-build-hook.md +++ b/doc/manual/src/advanced-topics/post-build-hook.md @@ -69,7 +69,7 @@ exec nix copy --to "s3://example-nix-cache" $OUT_PATHS > store sign`. Nix guarantees the paths will not contain any spaces, > however a store path might contain glob characters. The `set -f` > disables globbing in the shell. -> If you want to upload the .drv file too, the `$DRV_PATH` variable +> If you want to upload the `.drv` file too, the `$DRV_PATH` variable > is also defined for the script and works just like `$OUT_PATHS`. Then make sure the hook program is executable by the `root` user: From 13ed5d710674a9467c8959fed12641b40e38c791 Mon Sep 17 00:00:00 2001 From: Ilan Joselevich Date: Tue, 26 Sep 2023 00:31:00 +0200 Subject: [PATCH 325/909] flakes: adopt repl-flake behavior as default --- doc/manual/src/release-notes/rl-next.md | 2 ++ src/libutil/experimental-features.cc | 2 ++ src/nix/repl.cc | 2 +- src/nix/repl.md | 2 +- tests/repl.sh | 11 +++-------- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 2467e7ccc..a69cad1db 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -3,3 +3,5 @@ - [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. - [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). + +- The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly. diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 6d92222b9..203455b63 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -163,6 +163,8 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::ReplFlake, .name = "repl-flake", .description = R"( + *Enabled with [`flakes`](#xp-feature-flakes) since 2.19* + Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands. )", }, diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 9677c1b48..63fe3044b 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -47,7 +47,7 @@ struct CmdRepl : RawInstallablesCommand void applyDefaultInstallables(std::vector & rawInstallables) override { - if (!experimentalFeatureSettings.isEnabled(Xp::ReplFlake) && !(file) && rawInstallables.size() >= 1) { + if (!experimentalFeatureSettings.isEnabled(Xp::Flakes) && !(file) && rawInstallables.size() >= 1) { warn("future versions of Nix will require using `--file` to load a file"); if (rawInstallables.size() > 1) warn("more than one input file is not currently supported"); diff --git a/src/nix/repl.md b/src/nix/repl.md index 9dc61fd35..32c08e24b 100644 --- a/src/nix/repl.md +++ b/src/nix/repl.md @@ -36,7 +36,7 @@ R""( Loading Installable ''... Added 1 variables. - # nix repl --extra-experimental-features 'flakes repl-flake' nixpkgs + # nix repl --extra-experimental-features 'flakes' nixpkgs Loading Installable 'flake:nixpkgs#'... Added 5 variables. diff --git a/tests/repl.sh b/tests/repl.sh index bb8b60e50..1b779c1f5 100644 --- a/tests/repl.sh +++ b/tests/repl.sh @@ -105,18 +105,13 @@ testReplResponseNoRegex ' testReplResponse ' drvPath ' '".*-simple.drv"' \ -$testDir/simple.nix +--file $testDir/simple.nix testReplResponse ' drvPath ' '".*-simple.drv"' \ --file $testDir/simple.nix --experimental-features 'ca-derivations' -testReplResponse ' -drvPath -' '".*-simple.drv"' \ ---file $testDir/simple.nix --extra-experimental-features 'repl-flake ca-derivations' - mkdir -p flake && cat < flake/flake.nix { outputs = { self }: { @@ -130,7 +125,7 @@ EOF testReplResponse ' foo + baz ' "3" \ - ./flake ./flake\#bar --experimental-features 'flakes repl-flake' + ./flake ./flake\#bar --experimental-features 'flakes' # Test the `:reload` mechansim with flakes: # - Eval `./flake#changingThing` @@ -143,7 +138,7 @@ sleep 1 # Leave the repl the time to eval 'foo' sed -i 's/beforeChange/afterChange/' flake/flake.nix echo ":reload" echo "changingThing" -) | nix repl ./flake --experimental-features 'flakes repl-flake') +) | nix repl ./flake --experimental-features 'flakes') echo "$replResult" | grepQuiet -s beforeChange echo "$replResult" | grepQuiet -s afterChange From add7c99c3b559e531b7426e7a0a987231e0028ed Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Mon, 18 Sep 2023 16:57:18 +0800 Subject: [PATCH 326/909] Include "original" and "locked" in nix flake prefetch --json --- src/nix/flake.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 87dd4da1b..f8df032c8 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1342,6 +1342,8 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON auto res = nlohmann::json::object(); res["storePath"] = store->printStorePath(tree.storePath); res["hash"] = hash.to_string(SRI, true); + res["original"] = fetchers::attrsToJSON(resolvedRef.toAttrs()); + res["locked"] = fetchers::attrsToJSON(lockedRef.toAttrs()); logger->cout(res.dump()); } else { notice("Downloaded '%s' to '%s' (hash '%s').", From cede94dbf7733c2c64bdc4a4f53645cffdd73379 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 28 Sep 2023 20:51:25 -0400 Subject: [PATCH 327/909] `builtins.fetchTree`: Mark experimental the new way This helps ensure uniform docs/error message. --- src/libexpr/primops/fetchTree.cc | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 46d31a4e9..124165896 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -195,7 +195,6 @@ static void fetchTree( static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - experimentalFeatureSettings.require(Xp::Flakes); fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false }); } @@ -237,12 +236,9 @@ static RegisterPrimOp primop_fetchTree({ ```nix builtins.fetchTree "https://example.com/" ``` - - > **Note** - > - > This function requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. )", - .fun = prim_fetchTree + .fun = prim_fetchTree, + .experimentalFeature = Xp::Flakes, }); static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v, From b912f3a9377ce0b7302b8833934bc1af448ea996 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 28 Sep 2023 20:55:41 -0400 Subject: [PATCH 328/909] Move `flakeIdRegex{,S}` from `libutil` to `flakeref.{cc,hh` It isn't used, and doesn't belong in `libutil`. --- src/libexpr/flake/flakeref.cc | 2 ++ src/libexpr/flake/flakeref.hh | 3 +++ src/libutil/url-parts.hh | 3 --- src/libutil/url.cc | 1 - 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index f7ef19b3f..93ad33a33 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -289,4 +289,6 @@ std::tuple parseFlakeRefWithFragment return {std::move(flakeRef), fragment, std::move(extendedOutputsSpec)}; } +std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript); + } diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index a7c9208c0..ffb2e50de 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -6,6 +6,7 @@ #include "fetchers.hh" #include "outputs-spec.hh" +#include #include namespace nix { @@ -91,5 +92,7 @@ std::tuple parseFlakeRefWithFragment bool allowMissing = false, bool isFlake = true); +const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; +extern std::regex flakeIdRegex; } diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 98162b0f7..4712c0c91 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -41,7 +41,4 @@ extern std::regex revRegex; /// A ref or revision, or a ref followed by a revision. const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; -const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; -extern std::regex flakeIdRegex; - } diff --git a/src/libutil/url.cc b/src/libutil/url.cc index e700c8eaf..2970f8d2d 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -8,7 +8,6 @@ namespace nix { std::regex refRegex(refRegexS, std::regex::ECMAScript); std::regex badGitRefRegex(badGitRefRegexS, std::regex::ECMAScript); std::regex revRegex(revRegexS, std::regex::ECMAScript); -std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript); ParsedURL parseURL(const std::string & url) { From c816c67eed6d8508a84f40ca00ac93f5926b44be Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 28 Sep 2023 21:10:17 -0400 Subject: [PATCH 329/909] Reword some comments/API docs to reflect `libfetcher`'s multiple users It's not just flakes, but also `builtins.fetchTree`. Also try to provide some more info in general. --- src/libfetchers/attrs.hh | 6 ++++++ src/libfetchers/fetchers.cc | 3 ++- src/libfetchers/fetchers.hh | 31 ++++++++++++++++++++----------- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index 9f885a793..b9a2c824e 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -13,6 +13,12 @@ namespace nix::fetchers { typedef std::variant> Attr; + +/** + * An `Attrs` can be thought of a JSON object restricted or simplified + * to be "flat", not containing any subcontainers (arrays or objects) + * and also not containing any `null`s. + */ typedef std::map Attrs; Attrs jsonToAttrs(const nlohmann::json & json); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index e683b9f80..7bffb04ff 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -254,7 +254,8 @@ std::optional Input::getRev() const try { hash = Hash::parseAnyPrefixed(*s); } catch (BadHash &e) { - // Default to sha1 for backwards compatibility with existing flakes + // Default to sha1 for backwards compatibility with existing + // usages (e.g. `builtins.fetchTree` calls or flake inputs). hash = Hash::parseAny(*s, htSHA1); } } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 6e10e9513..b27ac4128 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -22,12 +22,11 @@ struct Tree struct InputScheme; /** - * The Input object is generated by a specific fetcher, based on the - * user-supplied input attribute in the flake.nix file, and contains + * The `Input` object is generated by a specific fetcher, based on + * user-supplied information, and contains * the information that the specific fetcher needs to perform the * actual fetch. The Input object is most commonly created via the - * "fromURL()" or "fromAttrs()" static functions which are provided - * the url or attrset specified in the flake file. + * `fromURL()` or `fromAttrs()` static functions. */ struct Input { @@ -44,10 +43,20 @@ struct Input std::optional parent; public: + /** + * Create an `Input` from a URL. + * + * The URL indicate which sort of fetcher, and provides information to that fetcher. + */ static Input fromURL(const std::string & url, bool requireTree = true); static Input fromURL(const ParsedURL & url, bool requireTree = true); + /** + * Create an `Input` from a an `Attrs`. + * + * The URL indicate which sort of fetcher, and provides information to that fetcher. + */ static Input fromAttrs(Attrs && attrs); ParsedURL toURL() const; @@ -116,13 +125,13 @@ public: /** - * The InputScheme represents a type of fetcher. Each fetcher - * registers with nix at startup time. When processing an input for a - * flake, each scheme is given an opportunity to "recognize" that - * input from the url or attributes in the flake file's specification - * and return an Input object to represent the input if it is - * recognized. The Input object contains the information the fetcher - * needs to actually perform the "fetch()" when called. + * The `InputScheme` represents a type of fetcher. Each fetcher + * registers with nix at startup time. When processing an `Input`, + * each scheme is given an opportunity to "recognize" that + * input from the user-provided url or attributes + * and return an `Input` object to represent the input if it is + * recognized. The `Input` object contains the information the fetcher + * needs to actually perform the `fetch()` when called. */ struct InputScheme { From bfe1308d3ff7f3dfd4facf4318adcb459e57ece9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 28 Sep 2023 21:21:58 -0400 Subject: [PATCH 330/909] Add infra for `InputSchemes` to be experimental --- src/libfetchers/fetchers.cc | 7 +++++++ src/libfetchers/fetchers.hh | 5 +++++ src/libutil/args.hh | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 7bffb04ff..e3d3ac562 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -36,6 +36,7 @@ Input Input::fromURL(const ParsedURL & url, bool requireTree) for (auto & inputScheme : *inputSchemes) { auto res = inputScheme->inputFromURL(url, requireTree); if (res) { + experimentalFeatureSettings.require(inputScheme->experimentalFeature()); res->scheme = inputScheme; fixupInput(*res); return std::move(*res); @@ -50,6 +51,7 @@ Input Input::fromAttrs(Attrs && attrs) for (auto & inputScheme : *inputSchemes) { auto res = inputScheme->inputFromAttrs(attrs); if (res) { + experimentalFeatureSettings.require(inputScheme->experimentalFeature()); res->scheme = inputScheme; fixupInput(*res); return std::move(*res); @@ -309,4 +311,9 @@ void InputScheme::clone(const Input & input, const Path & destDir) const throw Error("do not know how to clone input '%s'", input.to_string()); } +std::optional InputScheme::experimentalFeature() +{ + return {}; +} + } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index b27ac4128..b55aabb6f 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -158,6 +158,11 @@ struct InputScheme virtual void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg); virtual std::pair fetch(ref store, const Input & input) = 0; + + /** + * Is this `InputScheme` part of an experimental feature? + */ + virtual std::optional experimentalFeature(); }; void registerInputScheme(std::shared_ptr && fetcher); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index d90129796..1e5bac650 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -236,7 +236,7 @@ struct Command : virtual public Args static constexpr Category catDefault = 0; - virtual std::optional experimentalFeature (); + virtual std::optional experimentalFeature(); virtual Category category() { return catDefault; } }; From 89b39520630726b7c48e8cbc065643134352e43d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 28 Sep 2023 21:35:36 -0400 Subject: [PATCH 331/909] Make the indirect fetcher input scheme part of the Flakes XP feature I don't know much about it, but by the number of times "flake" appears in the code it seems like is part of flakes, at least for now. --- src/libfetchers/indirect.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 4874a43ff..e09d8bfb8 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -98,6 +98,11 @@ struct IndirectInputScheme : InputScheme { throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); } + + std::optional experimentalFeature() override + { + return Xp::Flakes; + } }; static auto rIndirectInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); From 1dd03c62ad8f09b50dd77c18b973d4aee2112f7a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 29 Sep 2023 10:46:42 +0200 Subject: [PATCH 332/909] add hint for troubleshooting tests --- doc/manual/src/contributing/testing.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index 881118505..4a7f1c15c 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -130,6 +130,15 @@ GNU gdb (GDB) 12.1 One can debug the Nix invocation in all the usual ways. For example, enter `run` to start the Nix invocation. +### Troubleshooting + +Sometimes running tests in the development shell may leave artefacts in the local repository. +To remove any traces of that: + +```console +git clean -x --force tests +``` + ### Characterization testing Occasionally, Nix utilizes a technique called [Characterization Testing](https://en.wikipedia.org/wiki/Characterization_test) as part of the functional tests. From 08145a5be5cbc963a7006f65d9417f710b3fb8b5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 29 Sep 2023 16:17:51 +0200 Subject: [PATCH 333/909] contributor guide: emphasize solving a well-specified problem with each pull request this moves the orientation step to the beginning, and adds notes how to make sure that a problem is well-spefified and the according change more likely to get accepted Co-authored-by: Robert Hensing --- CONTRIBUTING.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index facbf0eb0..4a9cff3b2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,25 +24,30 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). ## Making changes to Nix -1. Check for [pull requests](https://github.com/NixOS/nix/pulls) that might already cover the contribution you are about to make. - There are many open pull requests that might already do what you intent to work on. - You can use [labels](https://github.com/NixOS/nix/labels) to filter for relevant topics. - -2. Search for related issues that cover what you're going to work on. It could help to mention there that you will work on the issue. +1. Search for related issues that cover what you're going to work on. + It could help to mention there that you will work on the issue. Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly. - Pull requests addressing issues labeled [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) are especially welcomed by maintainers and will receive prioritised review. + Pull requests addressing issues labeled [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) or [RFC](https://github.com/NixOS/nix/labels/RFC) are especially welcomed by maintainers and will receive prioritised review. + + If there is no relevant issue yet and you're not sure whether your change is likely to be accepted, [open an issue](https://github.com/NixOS/nix/issues/new/choose) yourself. + +2. Check for [pull requests](https://github.com/NixOS/nix/pulls) that might already cover the contribution you are about to make. + There are many open pull requests that might already do what you intend to work on. + You can use [labels](https://github.com/NixOS/nix/labels) to filter for relevant topics. 3. Check the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html) for information on building Nix and running its tests. For contributions to the command line interface, please check the [CLI guidelines](https://nixos.org/manual/nix/unstable/contributing/cli-guideline.html). -4. Make your changes! +4. Make your change! 5. [Create a pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request) for your changes. - * Link related issues in your pull request to inform interested parties and future contributors about your change. + * Clearly explain the problem that you're solving. + + Link related issues to inform interested parties and future contributors about your change. + If your pull request closes one or multiple issues, mention that in the description using `Closes: #`, as it will then happen automatically when your change is merged. * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request). - If your pull request closes one or multiple issues, note that in the description using `Closes: #`, as it will then happen automatically when your change is merged. * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes. 6. Do not expect your pull request to be reviewed immediately. From f8a3893e8d77ce4a6e23719a0b2d88464cb84b9c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 2 Sep 2023 22:47:38 +0200 Subject: [PATCH 334/909] pathExists: isDir when endswith /. --- src/libexpr/primops.cc | 4 +++- tests/lang/eval-okay-pathexists.nix | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index cb1f99acb..5de1b2828 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1528,7 +1528,9 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, auto path = realisePath(state, pos, arg, { .checkForPureEval = false }); /* SourcePath doesn't know about trailing slash. */ - auto mustBeDir = arg.type() == nString && arg.string_view().ends_with("/"); + auto mustBeDir = arg.type() == nString + && (arg.string_view().ends_with("/") + || arg.string_view().ends_with("/.")); try { auto checked = state.checkSourcePath(path); diff --git a/tests/lang/eval-okay-pathexists.nix b/tests/lang/eval-okay-pathexists.nix index e1246e370..c5e7a62de 100644 --- a/tests/lang/eval-okay-pathexists.nix +++ b/tests/lang/eval-okay-pathexists.nix @@ -2,6 +2,27 @@ builtins.pathExists (./lib.nix) && builtins.pathExists (builtins.toPath ./lib.nix) && builtins.pathExists (builtins.toString ./lib.nix) && !builtins.pathExists (builtins.toString ./lib.nix + "/") +&& !builtins.pathExists (builtins.toString ./lib.nix + "/.") +# FIXME +# && !builtins.pathExists (builtins.toString ./lib.nix + "/..") +# && !builtins.pathExists (builtins.toString ./lib.nix + "/a/..") +# && !builtins.pathExists (builtins.toString ./lib.nix + "/../lib.nix") +&& !builtins.pathExists (builtins.toString ./lib.nix + "/./") +&& !builtins.pathExists (builtins.toString ./lib.nix + "/./.") +&& builtins.pathExists (builtins.toString ./.. + "/lang/lib.nix") +&& !builtins.pathExists (builtins.toString ./.. + "lang/lib.nix") +&& builtins.pathExists (builtins.toString ./. + "/../lang/lib.nix") +&& builtins.pathExists (builtins.toString ./. + "/../lang/./lib.nix") +&& builtins.pathExists (builtins.toString ./.) +&& builtins.pathExists (builtins.toString ./. + "/") +&& builtins.pathExists (builtins.toString ./. + "/../lang") +&& builtins.pathExists (builtins.toString ./. + "/../lang/") +&& builtins.pathExists (builtins.toString ./. + "/../lang/.") +&& builtins.pathExists (builtins.toString ./. + "/../lang/./") +&& builtins.pathExists (builtins.toString ./. + "/../lang//./") +&& builtins.pathExists (builtins.toString ./. + "/../lang/..") +&& builtins.pathExists (builtins.toString ./. + "/../lang/../") +&& builtins.pathExists (builtins.toString ./. + "/../lang/..//") && builtins.pathExists (builtins.toPath (builtins.toString ./lib.nix)) && !builtins.pathExists (builtins.toPath (builtins.toString ./bla.nix)) && builtins.pathExists ./lib.nix From bfdd908f7dee5ae04c6a501167ea0fe1f4e08591 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 24 Sep 2023 13:19:04 +0200 Subject: [PATCH 335/909] structured attrs: improve support / usage of NIX_ATTRS_{SH,JSON}_FILE In #4770 I implemented proper `nix-shell(1)` support for derivations using `__structuredAttrs = true;`. Back then we decided to introduce two new environment variables, `NIX_ATTRS_SH_FILE` for `.attrs.sh` and `NIX_ATTRS_JSON_FILE` for `.attrs.json`. This was to avoid having to copy these files to `$NIX_BUILD_TOP` in a `nix-shell(1)` session which effectively meant copying these files to the project dir without cleaning up afterwords[1]. On last NixCon I resumed hacking on `__structuredAttrs = true;` by default for `nixpkgs` with a few other folks and getting back to it, I identified a few problems with the how it's used in `nixpkgs`: * A lot of builders in `nixpkgs` don't care about the env vars and assume that `.attrs.sh` and `.attrs.json` are in `$NIX_BUILD_TOP`. The sole reason why this works is that `nix-shell(1)` sources the contents of `.attrs.sh` and then sources `$stdenv/setup` if it exists. This may not be pretty, but it mostly works. One notable difference when using nixpkgs' stdenv as of now is however that `$__structuredAttrs` is set to `1` on regular builds, but set to an empty string in a shell session. Also, `.attrs.json` cannot be used in shell sessions because it can only be accessed by `$NIX_ATTRS_JSON_FILE` and not by `$NIX_BUILD_TOP/.attrs.json`. I considered changing Nix to be compatible with what nixpkgs effectively does, but then we'd have to either move $NIX_BUILD_TOP for shell sessions to a temporary location (and thus breaking a lot of assumptions) or we'd reintroduce all the problems we solved back then by using these two env vars. This is partly because I didn't document these variables back then (mea culpa), so I decided to drop all mentions of `.attrs.{json,sh}` in the manual and only refer to `$NIX_ATTRS_SH_FILE` and `$NIX_ATTRS_JSON_FILE`. The same applies to all our integration tests. Theoretically we could deprecated using `"$NIX_BUILD_TOP"/.attrs.sh` in the future now. * `nix develop` and `nix print-dev-env` don't support this environment variable at all even though they're supposed to be part of the replacement for `nix-shell` - for the drv debugging part to be precise. This isn't a big deal for the vast majority of derivations, i.e. derivations relying on nixpkgs' `stdenv` wiring things together properly. This is because `nix develop` effectively "clones" the derivation and replaces the builder with a script that dumps all of the environment, shell variables, functions etc, so the state of structured attrs being "sourced" is transmitted into the dev shell and most of the time you don't need to worry about `.attrs.sh` not existing because the shell is correctly configured and the if [ -e .attrs.sh ]; then source .attrs.sh; fi is simply omitted. However, this will break when having a derivation that reads e.g. from `.attrs.json` like with import {}; runCommand "foo" { __structuredAttrs = true; foo.bar = 23; } '' cat $NIX_ATTRS_JSON_FILE # doesn't work because it points to /build/.attrs.json '' To work around this I employed a similar approach as it exists for `nix-shell`: the `NIX_ATTRS_{JSON,SH}_FILE` vars are replaced with temporary locations. The contents of `.attrs.sh` and `.attrs.json` are now written into the JSON by `get-env.sh`, the builder that `nix develop` injects into the derivation it's debugging. So finally the exact file contents are present and exported by `nix develop`. I also made `.attrs.json` a JSON string in the JSON printed by `get-env.sh` on purpose because then it's not necessary to serialize the object structure again. `nix develop` only needs the JSON as string because it's only written into the temporary file. I'm not entirely sure if it makes sense to also use a temporary location for `nix print-dev-env` (rather than just skipping the rewrite in there), but this would probably break certain cases where it's relied upon `$NIX_ATTRS_SH_FILE` to exist (prime example are the `nix print-dev-env` test-cases I wrote in this patch using `tests/shell.nix`, these would fail because the env var exists, but it cannot read from it). [1] https://github.com/NixOS/nix/pull/4770#issuecomment-836799719 --- .../src/language/advanced-attributes.md | 17 ++-- src/nix/develop.cc | 86 +++++++++++++++++-- src/nix/get-env.sh | 20 ++++- tests/build-hook-ca-fixed.nix | 5 +- tests/build-hook.nix | 5 +- tests/config.nix.in | 5 +- tests/failing.nix | 5 +- tests/hermetic.nix | 5 +- tests/structured-attrs.sh | 14 ++- 9 files changed, 141 insertions(+), 21 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 86f0f24b5..7d97b24b6 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -273,18 +273,21 @@ Derivations can declare some infrequently used optional attributes. - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ If the special attribute `__structuredAttrs` is set to `true`, the other derivation - attributes are serialised in JSON format and made available to the - builder via the file `.attrs.json` in the builder’s temporary - directory. This obviates the need for [`passAsFile`](#adv-attr-passAsFile) since JSON files - have no size restrictions, unlike process environments. + attributes are serialised into a file in JSON format. The environment variable + `NIX_ATTRS_JSON_FILE` points to the exact location of that file both in a build + and a [`nix-shell`](../command-ref/nix-shell.md). This obviates the need for + [`passAsFile`](#adv-attr-passAsFile) since JSON files have no size restrictions, + unlike process environments. It also makes it possible to tweak derivation settings in a structured way; see [`outputChecks`](#adv-attr-outputChecks) for example. As a convenience to Bash builders, - Nix writes a script named `.attrs.sh` to the builder’s directory - that initialises shell variables corresponding to all attributes - that are representable in Bash. This includes non-nested + Nix writes a script that initialises shell variables + corresponding to all attributes that are representable in Bash. The + environment variable `NIX_ATTRS_SH_FILE` points to the exact + location of the script, both in a build and a + [`nix-shell`](../command-ref/nix-shell.md). This includes non-nested (associative) arrays. For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`. diff --git a/src/nix/develop.cc b/src/nix/develop.cc index c01ef1a2a..b080a3939 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -8,9 +8,12 @@ #include "derivations.hh" #include "progress-bar.hh" #include "run.hh" +#include "util.hh" +#include #include #include +#include using namespace nix; @@ -51,6 +54,7 @@ struct BuildEnvironment std::map vars; std::map bashFunctions; + std::optional> structuredAttrs; static BuildEnvironment fromJSON(std::string_view in) { @@ -74,6 +78,10 @@ struct BuildEnvironment res.bashFunctions.insert({name, def}); } + if (json.contains("structuredAttrs")) { + res.structuredAttrs = {json["structuredAttrs"][".attrs.json"], json["structuredAttrs"][".attrs.sh"]}; + } + return res; } @@ -102,6 +110,13 @@ struct BuildEnvironment res["bashFunctions"] = bashFunctions; + if (providesStructuredAttrs()) { + auto contents = nlohmann::json::object(); + contents[".attrs.sh"] = getAttrsSH(); + contents[".attrs.json"] = getAttrsJSON(); + res["structuredAttrs"] = std::move(contents); + } + auto json = res.dump(); assert(BuildEnvironment::fromJSON(json) == *this); @@ -109,6 +124,23 @@ struct BuildEnvironment return json; } + bool providesStructuredAttrs() const + { + return structuredAttrs.has_value(); + } + + std::string getAttrsJSON() const + { + assert(providesStructuredAttrs()); + return structuredAttrs->first; + } + + std::string getAttrsSH() const + { + assert(providesStructuredAttrs()); + return structuredAttrs->second; + } + void toBash(std::ostream & out, const std::set & ignoreVars) const { for (auto & [name, value] : vars) { @@ -291,6 +323,7 @@ struct Common : InstallableCommand, MixProfile std::string makeRcScript( ref store, const BuildEnvironment & buildEnvironment, + const Path & tmpDir, const Path & outputsDir = absPath(".") + "/outputs") { // A list of colon-separated environment variables that should be @@ -353,9 +386,48 @@ struct Common : InstallableCommand, MixProfile } } + if (buildEnvironment.providesStructuredAttrs()) { + fixupStructuredAttrs( + "sh", + "NIX_ATTRS_SH_FILE", + buildEnvironment.getAttrsSH(), + rewrites, + buildEnvironment, + tmpDir + ); + fixupStructuredAttrs( + "json", + "NIX_ATTRS_JSON_FILE", + buildEnvironment.getAttrsJSON(), + rewrites, + buildEnvironment, + tmpDir + ); + } + return rewriteStrings(script, rewrites); } + /** + * Replace the value of NIX_ATTRS_*_FILE (`/build/.attrs.*`) with a tmp file + * that's accessible from the interactive shell session. + */ + void fixupStructuredAttrs( + const std::string & ext, + const std::string & envVar, + const std::string & content, + StringMap & rewrites, + const BuildEnvironment & buildEnvironment, + const Path & tmpDir) + { + auto targetFilePath = tmpDir + "/.attrs." + ext; + writeFile(targetFilePath, content); + + auto fileInBuilderEnv = buildEnvironment.vars.find(envVar); + assert(fileInBuilderEnv != buildEnvironment.vars.end()); + rewrites.insert({BuildEnvironment::getString(fileInBuilderEnv->second), targetFilePath}); + } + Strings getDefaultFlakeAttrPaths() override { Strings paths{ @@ -487,7 +559,9 @@ struct CmdDevelop : Common, MixEnvironment auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); - auto script = makeRcScript(store, buildEnvironment); + AutoDelete tmpDir(createTempDir("", "nix-develop"), true); + + auto script = makeRcScript(store, buildEnvironment, (Path) tmpDir); if (verbosity >= lvlDebug) script += "set -x\n"; @@ -615,10 +689,12 @@ struct CmdPrintDevEnv : Common, MixJSON stopProgressBar(); - logger->writeToStdout( - json - ? buildEnvironment.toJSON() - : makeRcScript(store, buildEnvironment)); + if (json) { + logger->writeToStdout(buildEnvironment.toJSON()); + } else { + AutoDelete tmpDir(createTempDir("", "nix-dev-env"), true); + logger->writeToStdout(makeRcScript(store, buildEnvironment, tmpDir)); + } } }; diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index a7a8a01b9..832cc2f11 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -1,5 +1,5 @@ set -e -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then source "$NIX_ATTRS_SH_FILE"; fi export IN_NIX_SHELL=impure export dontAddDisableDepTrack=1 @@ -101,7 +101,21 @@ __dumpEnv() { printf "}" done < <(printf "%s\n" "$__vars") - printf '\n }\n}' + printf '\n }' + + if [ -e "$NIX_ATTRS_SH_FILE" ]; then + printf ',\n "structuredAttrs": {\n ' + __escapeString ".attrs.sh" + printf ': ' + __escapeString "$(<"$NIX_ATTRS_SH_FILE")" + printf ',\n ' + __escapeString ".attrs.json" + printf ': ' + __escapeString "$(<"$NIX_ATTRS_JSON_FILE")" + printf '\n }' + fi + + printf '\n}' } __escapeString() { @@ -117,7 +131,7 @@ __escapeString() { # In case of `__structuredAttrs = true;` the list of outputs is an associative # array with a format like `outname => /nix/store/hash-drvname-outname`, so `__olist` # must contain the array's keys (hence `${!...[@]}`) in this case. -if [ -e .attrs.sh ]; then +if [ -e "$NIX_ATTRS_SH_FILE" ]; then __olist="${!outputs[@]}" else __olist=$outputs diff --git a/tests/build-hook-ca-fixed.nix b/tests/build-hook-ca-fixed.nix index 4cb9e85d1..0ce6d9b12 100644 --- a/tests/build-hook-ca-fixed.nix +++ b/tests/build-hook-ca-fixed.nix @@ -8,7 +8,10 @@ let derivation ({ inherit system; builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; outputHashMode = "recursive"; outputHashAlgo = "sha256"; } // removeAttrs args ["builder" "meta" "passthru"]) diff --git a/tests/build-hook.nix b/tests/build-hook.nix index 7effd7903..99a13aee4 100644 --- a/tests/build-hook.nix +++ b/tests/build-hook.nix @@ -14,7 +14,10 @@ let derivation ({ inherit system; builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; } // removeAttrs args ["builder" "meta" "passthru"] // caArgs) // { meta = args.meta or {}; passthru = args.passthru or {}; }; diff --git a/tests/config.nix.in b/tests/config.nix.in index 7facbdcbc..00dc007e1 100644 --- a/tests/config.nix.in +++ b/tests/config.nix.in @@ -20,7 +20,10 @@ rec { derivation ({ inherit system; builder = shell; - args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; PATH = path; } // caArgs // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; diff --git a/tests/failing.nix b/tests/failing.nix index 2a0350d4d..d25e2d6b6 100644 --- a/tests/failing.nix +++ b/tests/failing.nix @@ -6,7 +6,10 @@ let derivation ({ inherit system; builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; } // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; in diff --git a/tests/hermetic.nix b/tests/hermetic.nix index 4c9d7a51f..0513540f0 100644 --- a/tests/hermetic.nix +++ b/tests/hermetic.nix @@ -14,7 +14,10 @@ let derivation ({ inherit system; builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; } // removeAttrs args ["builder" "meta" "passthru"] // caArgs) // { meta = args.meta or {}; passthru = args.passthru or {}; }; diff --git a/tests/structured-attrs.sh b/tests/structured-attrs.sh index 378dbc735..f2835a761 100644 --- a/tests/structured-attrs.sh +++ b/tests/structured-attrs.sh @@ -15,9 +15,21 @@ nix-build structured-attrs.nix -A all -o $TEST_ROOT/result export NIX_BUILD_SHELL=$SHELL env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \ - --run 'test -e .attrs.json; test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' + --run 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' + +nix develop -f structured-attrs-shell.nix -c bash -c 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' # `nix develop` is a slightly special way of dealing with environment vars, it parses # these from a shell-file exported from a derivation. This is to test especially `outputs` # (which is an associative array in thsi case) being fine. nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"' + +nix print-dev-env -f structured-attrs-shell.nix | grep -q 'NIX_ATTRS_JSON_FILE=' +nix print-dev-env -f structured-attrs-shell.nix | grep -q 'NIX_ATTRS_SH_FILE=' +nix print-dev-env -f shell.nix shellDrv | grepQuietInverse 'NIX_ATTRS_SH_FILE' + +jsonOut="$(nix print-dev-env -f structured-attrs-shell.nix --json)" + +test "$(<<<"$jsonOut" jq '.structuredAttrs|keys|.[]' -r)" = "$(printf ".attrs.json\n.attrs.sh")" + +test "$(<<<"$jsonOut" jq '.variables.out.value' -r)" = "$(<<<"$jsonOut" jq '.structuredAttrs.".attrs.json"' -r | jq -r '.outputs.out')" From 42e3c6d658ebdb60eeef179d91e71ca311f69da8 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 29 Sep 2023 17:14:31 +0200 Subject: [PATCH 336/909] doc: reference NIX_ATTRS_*_FILE vars at the env var reference for drvs --- doc/manual/src/language/derivations.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 4a1b70fd1..d80a7dd48 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -165,6 +165,11 @@ The [`builder`](#attr-builder) is executed as follows: - `NIX_STORE` is set to the path of the top-level Nix store directory (typically, `/nix/store`). + - `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs` + is set to `true` for the dervation. A detailed explanation of this + behavior can be found in the + [section about structured attrs](./advanced-attributes.md#adv-attr-structuredAttrs). + - For each output declared in `outputs`, the corresponding environment variable is set to point to the intended path in the Nix store for that output. Each output path is a concatenation From 7a0886e3ccbab7da645dae6ecd015b3084aadc54 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 30 Sep 2023 22:04:38 +0100 Subject: [PATCH 337/909] tests/structured-attrs.sh: grep -q -> grepQuiet --- tests/structured-attrs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/structured-attrs.sh b/tests/structured-attrs.sh index f2835a761..f11992dcd 100644 --- a/tests/structured-attrs.sh +++ b/tests/structured-attrs.sh @@ -24,8 +24,8 @@ nix develop -f structured-attrs-shell.nix -c bash -c 'test "3" = "$(jq ".my.list # (which is an associative array in thsi case) being fine. nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"' -nix print-dev-env -f structured-attrs-shell.nix | grep -q 'NIX_ATTRS_JSON_FILE=' -nix print-dev-env -f structured-attrs-shell.nix | grep -q 'NIX_ATTRS_SH_FILE=' +nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_JSON_FILE=' +nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_SH_FILE=' nix print-dev-env -f shell.nix shellDrv | grepQuietInverse 'NIX_ATTRS_SH_FILE' jsonOut="$(nix print-dev-env -f structured-attrs-shell.nix --json)" From 78e886bc5fd9e4d85f8503799540c0b71bb270be Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 29 Sep 2023 15:59:11 +0200 Subject: [PATCH 338/909] refine the maintainer's process to unblock discussions more quickly this addresses that we're too often running into open-ended discussions about attempts to solve problems where neither the problem nor the solution is well-understood enough to make decisions in a reasonable amount of time. this also prevents us from doing more work asynchronously. Co-authored-by: Robert Hensing --- maintainers/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/maintainers/README.md b/maintainers/README.md index 0d520cb0c..8eeb47e8b 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -96,8 +96,10 @@ What constitutes a trivial pull request is up to maintainers' judgement. Pull requests and issues that are deemed important and controversial are discussed by the team during discussion meetings. This may be where the merit of the change itself or the implementation strategy is contested by a team member. +Whenever the discussion opens up questions about the process or this team's goals, this may indicate that the change is too large in scope. +In that case it is taken off the board to be reconsidered by the author or broken down into smaller pieces that are less far-reaching and can be reviewed independently. -As a general guideline, the order of items is determined as follows: +As a general guideline, the order of items to discuss is determined as follows: - Prioritise pull requests over issues From 8440afbed756254784d9fea3eaab06649dffd390 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 1 Oct 2023 23:29:45 -0400 Subject: [PATCH 339/909] Revert "Adapt scheduler to work with dynamic derivations" This reverts commit 5e3986f59cb58f48186a49dcec7aa317b4787522. This un-implements RFC 92 but fixes the critical bug #9052 which many people are hitting. This is a decent stop-gap until a minimal reproduction of that bug is found and a proper fix can be made. Mostly fixed #9052, but I would like to leave that issue open until we have a regression test, so I can then properly fix the bug (unbreaking RFC 92) later. --- .../create-derivation-and-realise-goal.cc | 157 ------------------ .../create-derivation-and-realise-goal.hh | 96 ----------- src/libstore/build/derivation-goal.cc | 33 +++- src/libstore/build/derivation-goal.hh | 5 +- src/libstore/build/entry-points.cc | 11 +- src/libstore/build/goal.hh | 10 -- src/libstore/build/worker.cc | 110 +++--------- src/libstore/build/worker.hh | 24 --- src/libstore/derived-path-map.cc | 3 - src/libstore/derived-path-map.hh | 7 +- tests/dyn-drv/build-built-drv.sh | 4 +- tests/dyn-drv/dep-built-drv.sh | 4 +- 12 files changed, 55 insertions(+), 409 deletions(-) delete mode 100644 src/libstore/build/create-derivation-and-realise-goal.cc delete mode 100644 src/libstore/build/create-derivation-and-realise-goal.hh diff --git a/src/libstore/build/create-derivation-and-realise-goal.cc b/src/libstore/build/create-derivation-and-realise-goal.cc deleted file mode 100644 index 60f67956d..000000000 --- a/src/libstore/build/create-derivation-and-realise-goal.cc +++ /dev/null @@ -1,157 +0,0 @@ -#include "create-derivation-and-realise-goal.hh" -#include "worker.hh" - -namespace nix { - -CreateDerivationAndRealiseGoal::CreateDerivationAndRealiseGoal(ref drvReq, - const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = drvReq, .outputs = wantedOutputs }) - , drvReq(drvReq) - , wantedOutputs(wantedOutputs) - , buildMode(buildMode) -{ - state = &CreateDerivationAndRealiseGoal::getDerivation; - name = fmt( - "outer obtaining drv from '%s' and then building outputs %s", - drvReq->to_string(worker.store), - std::visit(overloaded { - [&](const OutputsSpec::All) -> std::string { - return "* (all of them)"; - }, - [&](const OutputsSpec::Names os) { - return concatStringsSep(", ", quoteStrings(os)); - }, - }, wantedOutputs.raw)); - trace("created outer"); - - worker.updateProgress(); -} - - -CreateDerivationAndRealiseGoal::~CreateDerivationAndRealiseGoal() -{ -} - - -static StorePath pathPartOfReq(const SingleDerivedPath & req) -{ - return std::visit(overloaded { - [&](const SingleDerivedPath::Opaque & bo) { - return bo.path; - }, - [&](const SingleDerivedPath::Built & bfd) { - return pathPartOfReq(*bfd.drvPath); - }, - }, req.raw()); -} - - -std::string CreateDerivationAndRealiseGoal::key() -{ - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before "baboon". And - substitution goals and inner derivation goals always happen before - derivation goals (due to "b$"). */ - return "c$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store); -} - - -void CreateDerivationAndRealiseGoal::timedOut(Error && ex) -{ -} - - -void CreateDerivationAndRealiseGoal::work() -{ - (this->*state)(); -} - - -void CreateDerivationAndRealiseGoal::addWantedOutputs(const OutputsSpec & outputs) -{ - /* If we already want all outputs, there is nothing to do. */ - auto newWanted = wantedOutputs.union_(outputs); - bool needRestart = !newWanted.isSubsetOf(wantedOutputs); - wantedOutputs = newWanted; - - if (!needRestart) return; - - if (!optDrvPath) - // haven't started steps where the outputs matter yet - return; - worker.makeDerivationGoal(*optDrvPath, outputs, buildMode); -} - - -void CreateDerivationAndRealiseGoal::getDerivation() -{ - trace("outer init"); - - /* The first thing to do is to make sure that the derivation - exists. If it doesn't, it may be created through a - substitute. */ - if (auto optDrvPath = [this]() -> std::optional { - if (buildMode != bmNormal) return std::nullopt; - - auto drvPath = StorePath::dummy; - try { - drvPath = resolveDerivedPath(worker.store, *drvReq); - } catch (MissingRealisation &) { - return std::nullopt; - } - return worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath) - ? std::optional { drvPath } - : std::nullopt; - }()) { - trace(fmt("already have drv '%s' for '%s', can go straight to building", - worker.store.printStorePath(*optDrvPath), - drvReq->to_string(worker.store))); - - loadAndBuildDerivation(); - } else { - trace("need to obtain drv we want to build"); - - addWaitee(worker.makeGoal(DerivedPath::fromSingle(*drvReq))); - - state = &CreateDerivationAndRealiseGoal::loadAndBuildDerivation; - if (waitees.empty()) work(); - } -} - - -void CreateDerivationAndRealiseGoal::loadAndBuildDerivation() -{ - trace("outer load and build derivation"); - - if (nrFailed != 0) { - amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store))); - return; - } - - StorePath drvPath = resolveDerivedPath(worker.store, *drvReq); - /* Build this step! */ - concreteDrvGoal = worker.makeDerivationGoal(drvPath, wantedOutputs, buildMode); - addWaitee(upcast_goal(concreteDrvGoal)); - state = &CreateDerivationAndRealiseGoal::buildDone; - optDrvPath = std::move(drvPath); - if (waitees.empty()) work(); -} - - -void CreateDerivationAndRealiseGoal::buildDone() -{ - trace("outer build done"); - - buildResult = upcast_goal(concreteDrvGoal)->getBuildResult(DerivedPath::Built { - .drvPath = drvReq, - .outputs = wantedOutputs, - }); - - if (buildResult.success()) - amDone(ecSuccess); - else - amDone(ecFailed, Error("building '%s' failed", drvReq->to_string(worker.store))); -} - - -} diff --git a/src/libstore/build/create-derivation-and-realise-goal.hh b/src/libstore/build/create-derivation-and-realise-goal.hh deleted file mode 100644 index ca936fc95..000000000 --- a/src/libstore/build/create-derivation-and-realise-goal.hh +++ /dev/null @@ -1,96 +0,0 @@ -#pragma once - -#include "parsed-derivations.hh" -#include "lock.hh" -#include "store-api.hh" -#include "pathlocks.hh" -#include "goal.hh" - -namespace nix { - -struct DerivationGoal; - -/** - * This goal type is essentially the serial composition (like function - * composition) of a goal for getting a derivation, and then a - * `DerivationGoal` using the newly-obtained derivation. - * - * In the (currently experimental) general inductive case of derivations - * that are themselves build outputs, that first goal will be *another* - * `CreateDerivationAndRealiseGoal`. In the (much more common) base-case - * where the derivation has no provence and is just referred to by - * (content-addressed) store path, that first goal is a - * `SubstitutionGoal`. - * - * If we already have the derivation (e.g. if the evalutator has created - * the derivation locally and then instructured the store to build it), - * we can skip the first goal entirely as a small optimization. - */ -struct CreateDerivationAndRealiseGoal : public Goal -{ - /** - * How to obtain a store path of the derivation to build. - */ - ref drvReq; - - /** - * The path of the derivation, once obtained. - **/ - std::optional optDrvPath; - - /** - * The goal for the corresponding concrete derivation. - **/ - std::shared_ptr concreteDrvGoal; - - /** - * The specific outputs that we need to build. - */ - OutputsSpec wantedOutputs; - - typedef void (CreateDerivationAndRealiseGoal::*GoalState)(); - GoalState state; - - /** - * The final output paths of the build. - * - * - For input-addressed derivations, always the precomputed paths - * - * - For content-addressed derivations, calcuated from whatever the - * hash ends up being. (Note that fixed outputs derivations that - * produce the "wrong" output still install that data under its - * true content-address.) - */ - OutputPathMap finalOutputs; - - BuildMode buildMode; - - CreateDerivationAndRealiseGoal(ref drvReq, - const OutputsSpec & wantedOutputs, Worker & worker, - BuildMode buildMode = bmNormal); - virtual ~CreateDerivationAndRealiseGoal(); - - void timedOut(Error && ex) override; - - std::string key() override; - - void work() override; - - /** - * Add wanted outputs to an already existing derivation goal. - */ - void addWantedOutputs(const OutputsSpec & outputs); - - /** - * The states. - */ - void getDerivation(); - void loadAndBuildDerivation(); - void buildDone(); - - JobCategory jobCategory() const override { - return JobCategory::Administration; - }; -}; - -} diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 6472ecd99..83c0a3135 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -71,7 +71,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, , wantedOutputs(wantedOutputs) , buildMode(buildMode) { - state = &DerivationGoal::loadDerivation; + state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); @@ -164,6 +164,24 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) } +void DerivationGoal::getDerivation() +{ + trace("init"); + + /* The first thing to do is to make sure that the derivation + exists. If it doesn't, it may be created through a + substitute. */ + if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) { + loadDerivation(); + return; + } + + addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath))); + + state = &DerivationGoal::loadDerivation; +} + + void DerivationGoal::loadDerivation() { trace("loading derivation"); @@ -1498,24 +1516,23 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) if (!useDerivation) return; auto & fullDrv = *dynamic_cast(drv.get()); - std::optional info = tryGetConcreteDrvGoal(waitee); - if (!info) return; - const auto & [dg, drvReq] = *info; + auto * dg = dynamic_cast(&*waitee); + if (!dg) return; - auto * nodeP = fullDrv.inputDrvs.findSlot(drvReq.get()); + auto * nodeP = fullDrv.inputDrvs.findSlot(DerivedPath::Opaque { .path = dg->drvPath }); if (!nodeP) return; auto & outputs = nodeP->value; for (auto & outputName : outputs) { - auto buildResult = dg.get().getBuildResult(DerivedPath::Built { - .drvPath = makeConstantStorePathRef(dg.get().drvPath), + auto buildResult = dg->getBuildResult(DerivedPath::Built { + .drvPath = makeConstantStorePathRef(dg->drvPath), .outputs = OutputsSpec::Names { outputName }, }); if (buildResult.success()) { auto i = buildResult.builtOutputs.find(outputName); if (i != buildResult.builtOutputs.end()) inputDrvOutputs.insert_or_assign( - { dg.get().drvPath, outputName }, + { dg->drvPath, outputName }, i->second.outPath); } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 62b122c27..ddb5ee1e3 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -52,10 +52,6 @@ struct InitialOutput { /** * A goal for building some or all of the outputs of a derivation. - * - * The derivation must already be present, either in the store in a drv - * or in memory. If the derivation itself needs to be gotten first, a - * `CreateDerivationAndRealiseGoal` goal must be used instead. */ struct DerivationGoal : public Goal { @@ -235,6 +231,7 @@ struct DerivationGoal : public Goal /** * The states. */ + void getDerivation(); void loadDerivation(); void haveDerivation(); void outputsSubstitutionTried(); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index f0f0e5519..13ff22f45 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -1,6 +1,5 @@ #include "worker.hh" #include "substitution-goal.hh" -#include "create-derivation-and-realise-goal.hh" #include "derivation-goal.hh" #include "local-store.hh" @@ -16,7 +15,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod worker.run(goals); - StringSet failed; + StorePathSet failed; std::optional ex; for (auto & i : goals) { if (i->ex) { @@ -26,10 +25,10 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod ex = std::move(i->ex); } if (i->exitCode != Goal::ecSuccess) { - if (auto i2 = dynamic_cast(i.get())) - failed.insert(i2->drvReq->to_string(*this)); + if (auto i2 = dynamic_cast(i.get())) + failed.insert(i2->drvPath); else if (auto i2 = dynamic_cast(i.get())) - failed.insert(printStorePath(i2->storePath)); + failed.insert(i2->storePath); } } @@ -38,7 +37,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); - throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed))); + throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed)); } } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 01d3c3491..9af083230 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -49,16 +49,6 @@ enum struct JobCategory { * A substitution an arbitrary store object; it will use network resources. */ Substitution, - /** - * A goal that does no "real" work by itself, and just exists to depend on - * other goals which *do* do real work. These goals therefore are not - * limited. - * - * These goals cannot infinitely create themselves, so there is no risk of - * a "fork bomb" type situation (which would be a problem even though the - * goal do no real work) either. - */ - Administration, }; struct Goal : public std::enable_shared_from_this diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b4a634e7b..37cb86b91 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -2,7 +2,6 @@ #include "worker.hh" #include "substitution-goal.hh" #include "drv-output-substitution-goal.hh" -#include "create-derivation-and-realise-goal.hh" #include "local-derivation-goal.hh" #include "hook-instance.hh" @@ -42,24 +41,6 @@ Worker::~Worker() } -std::shared_ptr Worker::makeCreateDerivationAndRealiseGoal( - ref drvReq, - const OutputsSpec & wantedOutputs, - BuildMode buildMode) -{ - std::weak_ptr & goal_weak = outerDerivationGoals.ensureSlot(*drvReq).value; - std::shared_ptr goal = goal_weak.lock(); - if (!goal) { - goal = std::make_shared(drvReq, wantedOutputs, *this, buildMode); - goal_weak = goal; - wakeUp(goal); - } else { - goal->addWantedOutputs(wantedOutputs); - } - return goal; -} - - std::shared_ptr Worker::makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, @@ -130,7 +111,10 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { [&](const DerivedPath::Built & bfd) -> GoalPtr { - return makeCreateDerivationAndRealiseGoal(bfd.drvPath, bfd.outputs, buildMode); + if (auto bop = std::get_if(&*bfd.drvPath)) + return makeDerivationGoal(bop->path, bfd.outputs, buildMode); + else + throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); }, [&](const DerivedPath::Opaque & bo) -> GoalPtr { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); @@ -139,46 +123,24 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) } -template -static void cullMap(std::map & goalMap, F f) -{ - for (auto i = goalMap.begin(); i != goalMap.end();) - if (!f(i->second)) - i = goalMap.erase(i); - else ++i; -} - - template static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { /* !!! inefficient */ - cullMap(goalMap, [&](const std::weak_ptr & gp) -> bool { - return gp.lock() != goal; - }); -} - -template -static void removeGoal(std::shared_ptr goal, std::map>::ChildNode> & goalMap); - -template -static void removeGoal(std::shared_ptr goal, std::map>::ChildNode> & goalMap) -{ - /* !!! inefficient */ - cullMap(goalMap, [&](DerivedPathMap>::ChildNode & node) -> bool { - if (node.value.lock() == goal) - node.value.reset(); - removeGoal(goal, node.childMap); - return !node.value.expired() || !node.childMap.empty(); - }); + for (auto i = goalMap.begin(); + i != goalMap.end(); ) + if (i->second.lock() == goal) { + auto j = i; ++j; + goalMap.erase(i); + i = j; + } + else ++i; } void Worker::removeGoal(GoalPtr goal) { - if (auto drvGoal = std::dynamic_pointer_cast(goal)) - nix::removeGoal(drvGoal, outerDerivationGoals.map); - else if (auto drvGoal = std::dynamic_pointer_cast(goal)) + if (auto drvGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(drvGoal, derivationGoals); else if (auto subGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(subGoal, substitutionGoals); @@ -236,19 +198,8 @@ void Worker::childStarted(GoalPtr goal, const std::set & fds, child.respectTimeouts = respectTimeouts; children.emplace_back(child); if (inBuildSlot) { - switch (goal->jobCategory()) { - case JobCategory::Substitution: - nrSubstitutions++; - break; - case JobCategory::Build: - nrLocalBuilds++; - break; - case JobCategory::Administration: - /* Intentionally not limited, see docs */ - break; - default: - abort(); - } + if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++; + else nrLocalBuilds++; } } @@ -260,20 +211,12 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers) if (i == children.end()) return; if (i->inBuildSlot) { - switch (goal->jobCategory()) { - case JobCategory::Substitution: + if (goal->jobCategory() == JobCategory::Substitution) { assert(nrSubstitutions > 0); nrSubstitutions--; - break; - case JobCategory::Build: + } else { assert(nrLocalBuilds > 0); nrLocalBuilds--; - break; - case JobCategory::Administration: - /* Intentionally not limited, see docs */ - break; - default: - abort(); } } @@ -324,9 +267,9 @@ void Worker::run(const Goals & _topGoals) for (auto & i : _topGoals) { topGoals.insert(i); - if (auto goal = dynamic_cast(i.get())) { + if (auto goal = dynamic_cast(i.get())) { topPaths.push_back(DerivedPath::Built { - .drvPath = goal->drvReq, + .drvPath = makeConstantStorePathRef(goal->drvPath), .outputs = goal->wantedOutputs, }); } else if (auto goal = dynamic_cast(i.get())) { @@ -589,19 +532,4 @@ GoalPtr upcast_goal(std::shared_ptr subGoal) return subGoal; } -GoalPtr upcast_goal(std::shared_ptr subGoal) -{ - return subGoal; -} - -std::optional, std::reference_wrapper>> tryGetConcreteDrvGoal(GoalPtr waitee) -{ - auto * odg = dynamic_cast(&*waitee); - if (!odg) return std::nullopt; - return {{ - std::cref(*odg->concreteDrvGoal), - std::cref(*odg->drvReq), - }}; -} - } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 6f6d25d7d..23ad87914 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -4,7 +4,6 @@ #include "types.hh" #include "lock.hh" #include "store-api.hh" -#include "derived-path-map.hh" #include "goal.hh" #include "realisation.hh" @@ -14,7 +13,6 @@ namespace nix { /* Forward definition. */ -struct CreateDerivationAndRealiseGoal; struct DerivationGoal; struct PathSubstitutionGoal; class DrvOutputSubstitutionGoal; @@ -33,25 +31,9 @@ class DrvOutputSubstitutionGoal; */ GoalPtr upcast_goal(std::shared_ptr subGoal); GoalPtr upcast_goal(std::shared_ptr subGoal); -GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; -/** - * The current implementation of impure derivations has - * `DerivationGoal`s accumulate realisations from their waitees. - * Unfortunately, `DerivationGoal`s don't directly depend on other - * goals, but instead depend on `CreateDerivationAndRealiseGoal`s. - * - * We try not to share any of the details of any goal type with any - * other, for sake of modularity and quicker rebuilds. This means we - * cannot "just" downcast and fish out the field. So as an escape hatch, - * we have made the function, written in `worker.cc` where all the goal - * types are visible, and use it instead. - */ - -std::optional, std::reference_wrapper>> tryGetConcreteDrvGoal(GoalPtr waitee); - /** * A mapping used to remember for each child process to what goal it * belongs, and file descriptors for receiving log data and output @@ -119,9 +101,6 @@ private: * Maps used to prevent multiple instantiations of a goal for the * same derivation / path. */ - - DerivedPathMap> outerDerivationGoals; - std::map> derivationGoals; std::map> substitutionGoals; std::map> drvOutputSubstitutionGoals; @@ -209,9 +188,6 @@ public: * @ref DerivationGoal "derivation goal" */ private: - std::shared_ptr makeCreateDerivationAndRealiseGoal( - ref drvPath, - const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, std::function()> mkDrvGoal); diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index 437b6a71a..5982c04b3 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -51,11 +51,8 @@ typename DerivedPathMap::ChildNode * DerivedPathMap::findSlot(const Single // instantiations -#include "create-derivation-and-realise-goal.hh" namespace nix { -template struct DerivedPathMap>; - GENERATE_CMP_EXT( template<>, DerivedPathMap>::ChildNode, diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh index 4a2c90733..4d72b301e 100644 --- a/src/libstore/derived-path-map.hh +++ b/src/libstore/derived-path-map.hh @@ -20,11 +20,8 @@ namespace nix { * * @param V A type to instantiate for each output. It should probably * should be an "optional" type so not every interior node has to have a - * value. For example, the scheduler uses - * `DerivedPathMap>` to - * remember which goals correspond to which outputs. `* const Something` - * or `std::optional` would also be good choices for - * "optional" types. + * value. `* const Something` or `std::optional` would be + * good choices for "optional" types. */ template struct DerivedPathMap { diff --git a/tests/dyn-drv/build-built-drv.sh b/tests/dyn-drv/build-built-drv.sh index 94f3550bd..647be9457 100644 --- a/tests/dyn-drv/build-built-drv.sh +++ b/tests/dyn-drv/build-built-drv.sh @@ -18,6 +18,4 @@ clearStore drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv) -out2=$(nix build "${drvDep}^out^out" --no-link) - -test $out1 == $out2 +expectStderr 1 nix build "${drvDep}^out^out" --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" diff --git a/tests/dyn-drv/dep-built-drv.sh b/tests/dyn-drv/dep-built-drv.sh index c3daf2c59..4f6e9b080 100644 --- a/tests/dyn-drv/dep-built-drv.sh +++ b/tests/dyn-drv/dep-built-drv.sh @@ -6,6 +6,6 @@ out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link) clearStore -out2=$(nix-build ./text-hashed-output.nix -A wrapper --no-out-link) +expectStderr 1 nix-build ./text-hashed-output.nix -A wrapper --no-out-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" -diff -r $out1 $out2 +# diff -r $out1 $out2 From 632f24166d9d22e329d9d8626fa7dcd6b9ac83ce Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 26 Sep 2023 14:44:15 -0400 Subject: [PATCH 340/909] Test the rest of the worker protocol serializers Part of the `BuildResult` test is commented out because we have caught a roundtrip bug! A future PR will fix the bug and uncomment that test. --- src/libstore/build-result.cc | 18 ++ src/libstore/build-result.hh | 3 + src/libstore/tests/worker-protocol.cc | 189 ++++++++++++++++++ src/libstore/worker-protocol.hh | 4 +- .../libstore/worker-protocol/build-result.bin | Bin 0 -> 144 bytes .../libstore/worker-protocol/drv-output.bin | Bin 0 -> 176 bytes .../worker-protocol/keyed-build-result.bin | Bin 0 -> 264 bytes .../optional-content-address.bin | Bin 0 -> 64 bytes .../worker-protocol/optional-store-path.bin | Bin 0 -> 72 bytes .../worker-protocol/optional-trusted-flag.bin | Bin 0 -> 24 bytes .../libstore/worker-protocol/realisation.bin | Bin 0 -> 520 bytes .../libstore/worker-protocol/set.bin | Bin 0 -> 152 bytes .../libstore/worker-protocol/vector.bin | Bin 0 -> 152 bytes 13 files changed, 212 insertions(+), 2 deletions(-) create mode 100644 src/libstore/build-result.cc create mode 100644 unit-test-data/libstore/worker-protocol/build-result.bin create mode 100644 unit-test-data/libstore/worker-protocol/drv-output.bin create mode 100644 unit-test-data/libstore/worker-protocol/keyed-build-result.bin create mode 100644 unit-test-data/libstore/worker-protocol/optional-content-address.bin create mode 100644 unit-test-data/libstore/worker-protocol/optional-store-path.bin create mode 100644 unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin create mode 100644 unit-test-data/libstore/worker-protocol/realisation.bin create mode 100644 unit-test-data/libstore/worker-protocol/set.bin create mode 100644 unit-test-data/libstore/worker-protocol/vector.bin diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc new file mode 100644 index 000000000..18f519c5c --- /dev/null +++ b/src/libstore/build-result.cc @@ -0,0 +1,18 @@ +#include "build-result.hh" + +namespace nix { + +GENERATE_CMP_EXT( + , + BuildResult, + me->status, + me->errorMsg, + me->timesBuilt, + me->isNonDeterministic, + me->builtOutputs, + me->startTime, + me->stopTime, + me->cpuUser, + me->cpuSystem); + +} diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index b7a56e791..8840fa7e3 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -3,6 +3,7 @@ #include "realisation.hh" #include "derived-path.hh" +#include "comparator.hh" #include #include @@ -100,6 +101,8 @@ struct BuildResult */ std::optional cpuUser, cpuSystem; + DECLARE_CMP(BuildResult); + bool success() { return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid; diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index 4a6ccf7c0..24a06503c 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -6,6 +6,7 @@ #include "worker-protocol.hh" #include "worker-protocol-impl.hh" #include "derived-path.hh" +#include "build-result.hh" #include "tests/libstore.hh" namespace nix { @@ -136,4 +137,192 @@ CHARACTERIZATION_TEST( }, })) +CHARACTERIZATION_TEST( + drvOutput, + "drv-output", + (std::tuple { + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + })) + +CHARACTERIZATION_TEST( + realisation, + "realisation", + (std::tuple { + Realisation { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + }, + Realisation { + .id = { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + .dependentRealisations = { + { + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + }, + })) + +CHARACTERIZATION_TEST( + buildResult, + "build-result", + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, +#if 0 + // This is commented because this test would fail! + // FIXME uncomment this and fix the underlying bug. + BuildResult { + .status = BuildResult::Built, + .timesBuilt = 1, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + .startTime = 30, + .stopTime = 50, + .cpuUser = std::chrono::milliseconds(500s), + .cpuSystem = std::chrono::milliseconds(604s), + }, +#endif + }; + t; + })) + +CHARACTERIZATION_TEST( + keyedBuildResult, + "keyed-build-result", + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + KeyedBuildResult { + { + .status = KeyedBuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + /* .path = */ DerivedPath::Opaque { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-xxx" }, + }, + }, + KeyedBuildResult { + { + .status = KeyedBuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + /* .path = */ DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::Names { "out" }, + }, + }, + }; + t; + })) + +CHARACTERIZATION_TEST( + optionalTrustedFlag, + "optional-trusted-flag", + (std::tuple, std::optional, std::optional> { + std::nullopt, + std::optional { Trusted }, + std::optional { NotTrusted }, + })) + +CHARACTERIZATION_TEST( + vector, + "vector", + (std::tuple, std::vector, std::vector, std::vector>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + set, + "set", + (std::tuple, std::set, std::set, std::set>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + optionalStorePath, + "optional-store-path", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + }, + })) + +CHARACTERIZATION_TEST( + optionalContentAddress, + "optional-content-address", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + }, + })) + } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 70a5bddb9..b7f42f24d 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -208,10 +208,10 @@ MAKE_WORKER_PROTO(ContentAddress); template<> MAKE_WORKER_PROTO(DerivedPath); template<> -MAKE_WORKER_PROTO(Realisation); -template<> MAKE_WORKER_PROTO(DrvOutput); template<> +MAKE_WORKER_PROTO(Realisation); +template<> MAKE_WORKER_PROTO(BuildResult); template<> MAKE_WORKER_PROTO(KeyedBuildResult); diff --git a/unit-test-data/libstore/worker-protocol/build-result.bin b/unit-test-data/libstore/worker-protocol/build-result.bin new file mode 100644 index 0000000000000000000000000000000000000000..f981fd91f7f32724dc2b9ce1938398aad3d73cdf GIT binary patch literal 144 tcmZQ&fB%U7Sq@}q)U|6h9n2j1!MT?w+C~^gO WO9=?G&ojrYjy`x4ZufnEe#tjMpDPdm literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/keyed-build-result.bin b/unit-test-data/libstore/worker-protocol/keyed-build-result.bin new file mode 100644 index 0000000000000000000000000000000000000000..c5b3c7f3669a906fef3ef450e8fc8cf8d87d1d04 GIT binary patch literal 264 zcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NMMVVzD^wphl+Mdn$V^F1R4C7=go+SH onM0M4Vt-O%kzPtsnPPrv2?GyQKQ4DLL**Hvv>cQ+g3>Sw03m8EMgRZ+ literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/optional-content-address.bin b/unit-test-data/libstore/worker-protocol/optional-content-address.bin new file mode 100644 index 0000000000000000000000000000000000000000..f8cfe65ba27fd78ec7787eed7ebec8dfdead2fba GIT binary patch literal 64 zcmZQzfB#ECTBU06lLKJOBUy literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin b/unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin new file mode 100644 index 0000000000000000000000000000000000000000..51b239409bc815c19b00cb97f62996fb782f24c3 GIT binary patch literal 24 PcmZQzfB;4)%><3#3x z?rW}!>Uc!$VBY>Sr2mUC`<2<)qY;)1KN Po44)luetw6d9AunL$;dR literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/set.bin b/unit-test-data/libstore/worker-protocol/set.bin new file mode 100644 index 0000000000000000000000000000000000000000..ce11ede7fe7aa061a30b211a18cd7a336f879de8 GIT binary patch literal 152 ucmZQzfB;4)4WpQ03@8obCnXkvMPU52{CpHXOdBEdVDg4g4KThDln(&+h63;a literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/vector.bin b/unit-test-data/libstore/worker-protocol/vector.bin new file mode 100644 index 0000000000000000000000000000000000000000..7a37c8cd1093201c56f384747ae9c2aa8aa700f9 GIT binary patch literal 152 ucmZQzfB;4)4WpQ03@8obr{(8^MPU4-#3B?vOdBEdVDg4g4KThDln(&-b^`DK literal 0 HcmV?d00001 From e1af175707007591113b0d6fc56036782b0e347c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 4 Oct 2023 17:36:49 -0400 Subject: [PATCH 341/909] Enable most of the third `BuildResult` worker protocol test This was somewhat of a false alarm. The problem was not that the protocol implementation actually failed to round trip, but that two of the fields were ignored entirely --- not serialized and deserialized at all. For reference, those fields were added in fa68eb367e79297bb1c0451cd92ad18a06edce96. --- src/libstore/tests/worker-protocol.cc | 11 ++++++----- .../libstore/worker-protocol/build-result.bin | Bin 144 -> 744 bytes 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index 24a06503c..fa7cbe121 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -187,7 +187,7 @@ CHARACTERIZATION_TEST( "build-result", ({ using namespace std::literals::chrono_literals; - std::tuple t { + std::tuple t { BuildResult { .status = BuildResult::OutputRejected, .errorMsg = "no idea why", @@ -200,9 +200,6 @@ CHARACTERIZATION_TEST( .startTime = 30, .stopTime = 50, }, -#if 0 - // This is commented because this test would fail! - // FIXME uncomment this and fix the underlying bug. BuildResult { .status = BuildResult::Built, .timesBuilt = 1, @@ -230,10 +227,14 @@ CHARACTERIZATION_TEST( }, .startTime = 30, .stopTime = 50, +#if 0 + // These fields are not yet serialized. + // FIXME Include in next version of protocol or document + // why they are skipped. .cpuUser = std::chrono::milliseconds(500s), .cpuSystem = std::chrono::milliseconds(604s), - }, #endif + }, }; t; })) diff --git a/unit-test-data/libstore/worker-protocol/build-result.bin b/unit-test-data/libstore/worker-protocol/build-result.bin index f981fd91f7f32724dc2b9ce1938398aad3d73cdf..b02c706eab8de4d60eadd6afdcc2c888ab605ad3 100644 GIT binary patch literal 744 zcmc(b&uRiO5XSolPxTp!=B%)rxXHS&(2IC2JXe~O6CwQ^%woR?}>#sX4il$bfs;n zmv%`YOQ~vvTo;t-%xH@l(n4vSK8bRZQHc`kI^B)Ih0TkNGRhXy8rqZN5BnYj(ieFo zAJ+t*u7l`;??iPt&V)lzi91dfGZFf@g4iVAZN4+jUVRU7o>ol_o!fedeM@Pl_mAVf T^ROZOQyyvZZF!s Date: Thu, 7 Sep 2023 00:31:33 +0200 Subject: [PATCH 342/909] always show anchors on setting listings refactor the templates for readability --- doc/manual/generate-manpage.nix | 55 ++++++-------------------- doc/manual/generate-settings.nix | 63 ++++++++++++++++++++++++++++++ doc/manual/generate-store-info.nix | 45 +++++++++++++++++++++ doc/manual/local.mk | 6 +-- doc/manual/utils.nix | 57 --------------------------- 5 files changed, 124 insertions(+), 102 deletions(-) create mode 100644 doc/manual/generate-settings.nix create mode 100644 doc/manual/generate-store-info.nix diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 41725aed6..425eea002 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -2,7 +2,8 @@ let inherit (builtins) attrNames attrValues fromJSON listToAttrs mapAttrs groupBy concatStringsSep concatMap length lessThan replaceStrings sort; - inherit (import ./utils.nix) attrsToList concatStrings optionalString filterAttrs trim squash unique showSettings; + inherit (import ./utils.nix) attrsToList concatStrings optionalString filterAttrs trim squash unique; + showStoreDocs = import ./generate-store-info.nix; in commandDump: @@ -30,7 +31,7 @@ let ${maybeSubcommands} - ${maybeDocumentation} + ${maybeStoreDocs} ${maybeOptions} ''; @@ -40,15 +41,15 @@ let showArgument = arg: "*${arg.label}*" + optionalString (! arg ? arity) "..."; arguments = concatStringsSep " " (map showArgument args); in '' - `${command}` [*option*...] ${arguments} + `${command}` [*option*...] ${arguments} ''; maybeSubcommands = optionalString (details ? commands && details.commands != {}) - '' - where *subcommand* is one of the following: + '' + where *subcommand* is one of the following: - ${subcommands} - ''; + ${subcommands} + ''; subcommands = if length categories > 1 then listCategories @@ -70,10 +71,11 @@ let * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description} ''; - # TODO: move this confusing special case out of here when implementing #8496 - maybeDocumentation = optionalString - (details ? doc) - (replaceStrings ["@stores@"] [storeDocs] details.doc); + # FIXME: this is a hack. + # store parameters should not be part of command documentation to begin + # with, but instead be rendered on separate pages. + maybeStoreDocs = optionalString (details ? doc) + (replaceStrings [ "@stores@" ] [ (showStoreDocs commandInfo.stores) ] details.doc); maybeOptions = let allVisibleOptions = filterAttrs @@ -147,35 +149,4 @@ let " - [${page.command}](command-ref/new-cli/${page.name})"; in concatStringsSep "\n" (map showEntry manpages) + "\n"; - storeDocs = - let - showStore = name: { settings, doc, experimentalFeature }: - let - experimentalFeatureNote = optionalString (experimentalFeature != null) '' - > **Warning** - > This store is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). - - To use this store, you need to make sure the corresponding experimental feature, - [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), - is enabled. - For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): - - ``` - extra-experimental-features = ${experimentalFeature} - ``` - ''; - in '' - ## ${name} - - ${doc} - - ${experimentalFeatureNote} - - **Settings**: - - ${showSettings { useAnchors = false; } settings} - ''; - in concatStrings (attrValues (mapAttrs showStore commandInfo.stores)); - in (listToAttrs manpages) // { "SUMMARY.md" = tableOfContents; } diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix new file mode 100644 index 000000000..450771f73 --- /dev/null +++ b/doc/manual/generate-settings.nix @@ -0,0 +1,63 @@ +let + inherit (builtins) attrValues concatStringsSep isAttrs isBool mapAttrs; + inherit (import ./utils.nix) concatStrings indent optionalString squash; +in + +prefix: settingsInfo: + +let + + showSetting = prefix: setting: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }: + let + result = squash '' + - [`${setting}`](#${prefix}-${setting}) + + ${indent " " body} + ''; + + # separate body to cleanly handle indentation + body = '' + ${description} + + ${experimentalFeatureNote} + + **Default:** ${showDefault documentDefault defaultValue} + + ${showAliases aliases} + ''; + + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > This setting is part of an + > [experimental feature](@docroot@/contributing/experimental-features.md). + + To change this setting, you need to make sure the corresponding experimental feature, + [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), + is enabled. + For example, include the following in [`nix.conf`](#): + + ``` + extra-experimental-features = ${experimentalFeature} + ${setting} = ... + ``` + ''; + + showDefault = documentDefault: defaultValue: + if documentDefault then + # a StringMap value type is specified as a string, but + # this shows the value type. The empty stringmap is `null` in + # JSON, but that converts to `{ }` here. + if defaultValue == "" || defaultValue == [] || isAttrs defaultValue + then "*empty*" + else if isBool defaultValue then + if defaultValue then "`true`" else "`false`" + else "`${toString defaultValue}`" + else "*machine-specific*"; + + showAliases = aliases: + optionalString (aliases != []) + "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}"; + + in result; + +in concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo)) diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix new file mode 100644 index 000000000..7bb2ebad3 --- /dev/null +++ b/doc/manual/generate-store-info.nix @@ -0,0 +1,45 @@ +let + inherit (builtins) attrValues mapAttrs; + inherit (import ./utils.nix) concatStrings optionalString; + showSettings = import ./generate-settings.nix; +in + +storesInfo: + +let + + showStore = name: { settings, doc, experimentalFeature }: + let + + result = '' + ## ${name} + + ${doc} + + ${experimentalFeatureNote} + + ### Settings + + ${showSettings "store-${slug}" settings} + ''; + + # markdown doesn't like spaces in URLs + slug = builtins.replaceStrings [ " " ] [ "-" ] name; + + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > This store is part of an + > [experimental feature](@docroot@/contributing/experimental-features.md). + + To use this store, you need to make sure the corresponding experimental feature, + [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), + is enabled. + For example, include the following in [`nix.conf`](#): + + ``` + extra-experimental-features = ${experimentalFeature} + ``` + ''; + in result; + +in concatStrings (attrValues (mapAttrs showStore storesInfo)) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index abdfd6a62..927854066 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -96,14 +96,14 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/sr @cp $< $@ @$(call process-includes,$@,$@) -$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(bindir)/nix +$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(bindir)/nix @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)' @mv $@.tmp $@ -$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix +$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix "opt-" (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ $(d)/nix.json: $(bindir)/nix diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index 9043dd8cd..849832b2c 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -44,63 +44,6 @@ rec { optionalString = cond: string: if cond then string else ""; - showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }: - let - result = squash '' - - ${if useAnchors - then ''[`${name}`](#conf-${name})'' - else ''`${name}`''} - - ${indent " " body} - ''; - - experimentalFeatureNote = optionalString (experimentalFeature != null) '' - > **Warning** - > This setting is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). - - To change this setting, you need to make sure the corresponding experimental feature, - [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), - is enabled. - For example, include the following in [`nix.conf`](#): - - ``` - extra-experimental-features = ${experimentalFeature} - ${name} = ... - ``` - ''; - - # separate body to cleanly handle indentation - body = '' - ${description} - - ${experimentalFeatureNote} - - **Default:** ${showDefault documentDefault defaultValue} - - ${showAliases aliases} - ''; - - showDefault = documentDefault: defaultValue: - if documentDefault then - # a StringMap value type is specified as a string, but - # this shows the value type. The empty stringmap is `null` in - # JSON, but that converts to `{ }` here. - if defaultValue == "" || defaultValue == [] || isAttrs defaultValue - then "*empty*" - else if isBool defaultValue then - if defaultValue then "`true`" else "`false`" - else "`${toString defaultValue}`" - else "*machine-specific*"; - - showAliases = aliases: - optionalString (aliases != []) - "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}"; - - in result; - indent = prefix: s: concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s)); - - showSettings = args: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting args) settingsInfo)); } From 8232711c9fa0ba831c3554e1d19e6707dd093c39 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 4 Oct 2023 23:07:13 +0200 Subject: [PATCH 343/909] fix wiring of baked-in Nix expressions --- src/nix/local.mk | 2 +- src/nix/main.cc | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/nix/local.mk b/src/nix/local.mk index 20ea29d10..57f8259c4 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -31,7 +31,7 @@ src/nix/develop.cc: src/nix/get-env.sh.gen.hh src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh -src/nix/main.cc: doc/manual/generate-manpage.nix.gen.hh doc/manual/utils.nix.gen.hh +src/nix/main.cc: doc/manual/generate-manpage.nix.gen.hh doc/manual/utils.nix.gen.hh doc/manual/generate-settings.nix.gen.hh doc/manual/generate-store-info.nix.gen.hh src/nix/doc/files/%.md: doc/manual/src/command-ref/files/%.md @mkdir -p $$(dirname $@) diff --git a/src/nix/main.cc b/src/nix/main.cc index d05bac68e..031dc2348 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -214,6 +214,22 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) , CanonPath::root), *vUtils); + auto vSettingsInfo = state.allocValue(); + state.cacheFile( + CanonPath("/generate-settings.nix"), CanonPath("/generate-settings.nix"), + state.parseExprFromString( + #include "generate-settings.nix.gen.hh" + , CanonPath::root), + *vSettingsInfo); + + auto vStoreInfo = state.allocValue(); + state.cacheFile( + CanonPath("/generate-store-info.nix"), CanonPath("/generate-store-info.nix"), + state.parseExprFromString( + #include "generate-store-info.nix.gen.hh" + , CanonPath::root), + *vStoreInfo); + auto vDump = state.allocValue(); vDump->mkString(toplevel.dumpCli()); From 24bda0c7b381e1a017023c6f7cb9661fae8560bd Mon Sep 17 00:00:00 2001 From: edef Date: Wed, 4 Oct 2023 16:50:24 +0000 Subject: [PATCH 344/909] StorePath: reject names starting with '.' This has been the behaviour before Nix 2.4. It was dropped in a rewrite in 759947bf72c134592f0ce23d385e48095bd0a301, allowing the creation of store paths that aren't considered valid by older Nix versions or other Nix tooling. Nix 2.4 didn't ship in NixOS until 22.05, and stdenv.mkDerivation in nixpkgs drops leading periods since April 2022, so it's unlikely anyone is relying on the current lax behaviour. Closes #9091. Change-Id: I4a57bd9899e1b0dba56870ae5a1b680918a18ce9 --- src/libstore/path-regex.hh | 2 +- src/libstore/path.cc | 2 ++ src/libstore/tests/path.cc | 9 +++++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libstore/path-regex.hh b/src/libstore/path-regex.hh index 4f8dc4c1f..a44e6a2eb 100644 --- a/src/libstore/path-regex.hh +++ b/src/libstore/path-regex.hh @@ -3,6 +3,6 @@ namespace nix { -static constexpr std::string_view nameRegexStr = R"([0-9a-zA-Z\+\-\._\?=]+)"; +static constexpr std::string_view nameRegexStr = R"([0-9a-zA-Z\+\-_\?=][0-9a-zA-Z\+\-\._\?=]*)"; } diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 552e83114..3c6b9fc10 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -11,6 +11,8 @@ static void checkName(std::string_view path, std::string_view name) if (name.size() > StorePath::MaxPathLen) throw BadStorePath("store path '%s' has a name longer than %d characters", path, StorePath::MaxPathLen); + if (name[0] == '.') + throw BadStorePath("store path '%s' starts with illegal character '.'", path); // See nameRegexStr for the definition for (auto c : name) if (!((c >= '0' && c <= '9') diff --git a/src/libstore/tests/path.cc b/src/libstore/tests/path.cc index efa35ef2b..5a84d646c 100644 --- a/src/libstore/tests/path.cc +++ b/src/libstore/tests/path.cc @@ -39,6 +39,7 @@ TEST_DONT_PARSE(double_star, "**") TEST_DONT_PARSE(star_first, "*,foo") TEST_DONT_PARSE(star_second, "foo,*") TEST_DONT_PARSE(bang, "foo!o") +TEST_DONT_PARSE(dotfile, ".gitignore") #undef TEST_DONT_PARSE @@ -101,8 +102,12 @@ Gen Arbitrary::arbitrary() pre += '-'; break; case 64: - pre += '.'; - break; + // names aren't permitted to start with a period, + // so just fall through to the next case here + if (c != 0) { + pre += '.'; + break; + } case 65: pre += '_'; break; From e0e47c0a68f7e1b6b0f9e5065e8113b32b12c212 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 5 Oct 2023 01:20:26 +0200 Subject: [PATCH 345/909] accommodate inconsistent output from `lowdown` the `term` output mode leaves inline HTML around verbatim, while `nroff` mode (used for `man` pages) does not. the correct solution would be to pre-render all output with a more benign tool so we have less liabilities in our own code, but this has to do for now. --- doc/manual/generate-manpage.nix | 22 +++++++++++++--------- doc/manual/generate-settings.nix | 9 ++++++--- doc/manual/generate-store-info.nix | 4 ++-- doc/manual/local.mk | 4 ++-- src/nix/main.cc | 3 ++- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 425eea002..d81eac182 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -6,7 +6,7 @@ let showStoreDocs = import ./generate-store-info.nix; in -commandDump: +inlineHTML: commandDump: let @@ -75,7 +75,7 @@ let # store parameters should not be part of command documentation to begin # with, but instead be rendered on separate pages. maybeStoreDocs = optionalString (details ? doc) - (replaceStrings [ "@stores@" ] [ (showStoreDocs commandInfo.stores) ] details.doc); + (replaceStrings [ "@stores@" ] [ (showStoreDocs inlineHTML commandInfo.stores) ] details.doc); maybeOptions = let allVisibleOptions = filterAttrs @@ -84,14 +84,14 @@ let in optionalString (allVisibleOptions != {}) '' # Options - ${showOptions allVisibleOptions} + ${showOptions inlineHTML allVisibleOptions} > **Note** > > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. ''; - showOptions = allOptions: + showOptions = inlineHTML: allOptions: let showCategory = cat: opts: '' ${optionalString (cat != "") "## ${cat}"} @@ -100,17 +100,21 @@ let ''; showOption = name: option: let + result = trim '' + - ${item} + + ${option.description} + ''; + item = if inlineHTML + then ''[`--${name}`](#opt-${name}) ${shortName} ${labels}'' + else "`--${name}` ${shortName} ${labels}"; shortName = optionalString (option ? shortName) ("/ `-${option.shortName}`"); labels = optionalString (option ? labels) (concatStringsSep " " (map (s: "*${s}*") option.labels)); - in trim '' - - [`--${name}`](#opt-${name}) ${shortName} ${labels} - - ${option.description} - ''; + in result; categories = mapAttrs # Convert each group from a list of key-value pairs back to an attrset (_: listToAttrs) diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 450771f73..8736bb793 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -3,18 +3,21 @@ let inherit (import ./utils.nix) concatStrings indent optionalString squash; in -prefix: settingsInfo: +# `inlineHTML` is a hack to accommodate inconsistent output from `lowdown` +{ prefix, inlineHTML ? true }: settingsInfo: let showSetting = prefix: setting: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }: let result = squash '' - - [`${setting}`](#${prefix}-${setting}) + - ${item} ${indent " " body} ''; - + item = if inlineHTML + then ''[`${setting}`](#${prefix}-${setting})'' + else "`${setting}`"; # separate body to cleanly handle indentation body = '' ${description} diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index 7bb2ebad3..36215aadf 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -4,7 +4,7 @@ let showSettings = import ./generate-settings.nix; in -storesInfo: +inlineHTML: storesInfo: let @@ -20,7 +20,7 @@ let ### Settings - ${showSettings "store-${slug}" settings} + ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings} ''; # markdown doesn't like spaces in URLs diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 927854066..2d1db66f3 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -98,12 +98,12 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/sr $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(bindir)/nix @rm -rf $@ $@.tmp - $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)' + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)' @mv $@.tmp $@ $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix "opt-" (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "opt-"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ $(d)/nix.json: $(bindir)/nix diff --git a/src/nix/main.cc b/src/nix/main.cc index 031dc2348..879baa5f5 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -234,7 +234,8 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) vDump->mkString(toplevel.dumpCli()); auto vRes = state.allocValue(); - state.callFunction(*vGenerateManpage, *vDump, *vRes, noPos); + state.callFunction(*vGenerateManpage, state.getBuiltin("false"), *vRes, noPos); + state.callFunction(*vRes, *vDump, *vRes, noPos); auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md")); if (!attr) From 80f734a803d3170eadd51758c71390ff87be22f9 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 5 Oct 2023 09:20:42 +0200 Subject: [PATCH 346/909] more specific links to nix.dev --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1440239bc..65daa22e4 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ for more details. ## Installation and first steps -Visit [nix.dev](https://nix.dev) for installation instructions and beginner tutorials. +Visit [nix.dev](https://nix.dev) for [installation instructions](https://nix.dev/tutorials/install-nix) and [beginner tutorials](https://nix.dev/tutorials/first-steps). Full reference documentation can be found in the [Nix manual](https://nixos.org/nix/manual). From eb68454be61bd6e8baf40136ae69c2b5fec3006a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 4 Oct 2023 17:10:54 +0200 Subject: [PATCH 347/909] Don't run the tests that require building if we're not building A couple of tests require building some libraries that depend on Nix, and assume it to be built locally. Don't run these if we only want to run the install tests. This prevents the CI from rebuilding several times Nix (like in https://github.com/NixOS/nix/actions/runs/6404422275/job/17384964033#step:6:6412), thus removing a fair amount of build time. --- flake.nix | 1 + tests/local.mk | 24 ++++++++++++++++-------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/flake.nix b/flake.nix index e3b9477a5..d74085db7 100644 --- a/flake.nix +++ b/flake.nix @@ -260,6 +260,7 @@ testNixVersions = pkgs: client: daemon: with commonDeps { inherit pkgs; }; with pkgs.lib; pkgs.stdenv.mkDerivation { NIX_DAEMON_PACKAGE = daemon; NIX_CLIENT_PACKAGE = client; + HAVE_LOCAL_NIX_BUILD = false; name = "nix-tests" + optionalString diff --git a/tests/local.mk b/tests/local.mk index 7479b1576..ac418dc0d 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -1,3 +1,7 @@ +# whether to run the tests that assume that we have a local build of +# Nix +HAVE_LOCAL_NIX_BUILD ?= 1 + nix_tests = \ test-infra.sh \ init.sh \ @@ -104,7 +108,6 @@ nix_tests = \ case-hack.sh \ placeholders.sh \ ssh-relay.sh \ - plugins.sh \ build.sh \ build-delete.sh \ output-normalization.sh \ @@ -120,7 +123,6 @@ nix_tests = \ flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ - test-libstoreconsumer.sh \ toString-path.sh \ read-only-store.sh \ nested-sandboxing.sh @@ -129,6 +131,17 @@ ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh endif +ifeq ($(HAVE_LOCAL_NIX_BUILD), 1) + nix_tests += test-libstoreconsumer.sh + + ifeq ($(BUILD_SHARED_LIBS), 1) + nix_tests += plugins.sh + endif +endif + +tests/test-libstoreconsumer.sh.test: tests/test-libstoreconsumer/test-libstoreconsumer +tests/plugins.sh: tests/plugins/libplugintest.$(SO_EXT) + install-tests += $(foreach x, $(nix_tests), $(d)/$(x)) clean-files += \ @@ -137,9 +150,4 @@ clean-files += \ test-deps += \ tests/common/vars-and-functions.sh \ - tests/config.nix \ - tests/test-libstoreconsumer/test-libstoreconsumer - -ifeq ($(BUILD_SHARED_LIBS), 1) - test-deps += tests/plugins/libplugintest.$(SO_EXT) -endif + tests/config.nix From 92e8e1b1bbc59d3a7e9efd5bff146b0a2726c7b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 4 Oct 2023 17:22:45 +0200 Subject: [PATCH 348/909] Poison the build on the test derivation Make sure that we're not accidentally rebuilding Nix here as it's just wasteful and awful for CI times. --- flake.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index d74085db7..4de155362 100644 --- a/flake.nix +++ b/flake.nix @@ -280,9 +280,13 @@ enableParallelBuilding = true; configureFlags = testConfigureFlags; # otherwise configure fails - dontBuild = true; doInstallCheck = true; + buildPhase = '' + # Remove the source files to make sure that we're not accidentally rebuilding Nix + rm src/**/*.cc + ''; + installPhase = '' mkdir -p $out ''; From f95364a803ddf694e4d7a7f6841101b93d9741fe Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 1 Sep 2023 12:20:54 +0200 Subject: [PATCH 349/909] eval: Run a full GC before printing stats This makes the numbers more deterministic, especially when it comes to the final heap size. --- src/libexpr/eval.cc | 212 +++++++++++++++++++++++++------------------- src/libexpr/eval.hh | 18 +++- 2 files changed, 136 insertions(+), 94 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index a78c0afb1..600444955 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2477,10 +2477,37 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v } } +bool EvalState::fullGC() { +#if HAVE_BOEHMGC + GC_gcollect(); + // Check that it ran. We might replace this with a version that uses more + // of the boehm API to get this reliably, at a maintenance cost. + // We use a 1K margin because technically this has a race condtion, but we + // probably won't encounter it in practice, because the CLI isn't concurrent + // like that. + return GC_get_bytes_since_gc() < 1024; +#else + return false; +#endif +} + void EvalState::printStats() { bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0"; + if (showStats) { + // Make the final heap size more deterministic. +#if HAVE_BOEHMGC + if (!fullGC()) { + warn("failed to perform a full GC before reporting stats"); + } +#endif + printStatistics(); + } +} + +void EvalState::printStatistics() +{ struct rusage buf; getrusage(RUSAGE_SELF, &buf); float cpuTime = buf.ru_utime.tv_sec + ((float) buf.ru_utime.tv_usec / 1000000); @@ -2494,106 +2521,105 @@ void EvalState::printStats() GC_word heapSize, totalBytes; GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes); #endif - if (showStats) { - auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-"); - std::fstream fs; - if (outPath != "-") - fs.open(outPath, std::fstream::out); - json topObj = json::object(); - topObj["cpuTime"] = cpuTime; - topObj["envs"] = { - {"number", nrEnvs}, - {"elements", nrValuesInEnvs}, - {"bytes", bEnvs}, - }; - topObj["nrExprs"] = Expr::nrExprs; - topObj["list"] = { - {"elements", nrListElems}, - {"bytes", bLists}, - {"concats", nrListConcats}, - }; - topObj["values"] = { - {"number", nrValues}, - {"bytes", bValues}, - }; - topObj["symbols"] = { - {"number", symbols.size()}, - {"bytes", symbols.totalSize()}, - }; - topObj["sets"] = { - {"number", nrAttrsets}, - {"bytes", bAttrsets}, - {"elements", nrAttrsInAttrsets}, - }; - topObj["sizes"] = { - {"Env", sizeof(Env)}, - {"Value", sizeof(Value)}, - {"Bindings", sizeof(Bindings)}, - {"Attr", sizeof(Attr)}, - }; - topObj["nrOpUpdates"] = nrOpUpdates; - topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied; - topObj["nrThunks"] = nrThunks; - topObj["nrAvoided"] = nrAvoided; - topObj["nrLookups"] = nrLookups; - topObj["nrPrimOpCalls"] = nrPrimOpCalls; - topObj["nrFunctionCalls"] = nrFunctionCalls; + + auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-"); + std::fstream fs; + if (outPath != "-") + fs.open(outPath, std::fstream::out); + json topObj = json::object(); + topObj["cpuTime"] = cpuTime; + topObj["envs"] = { + {"number", nrEnvs}, + {"elements", nrValuesInEnvs}, + {"bytes", bEnvs}, + }; + topObj["nrExprs"] = Expr::nrExprs; + topObj["list"] = { + {"elements", nrListElems}, + {"bytes", bLists}, + {"concats", nrListConcats}, + }; + topObj["values"] = { + {"number", nrValues}, + {"bytes", bValues}, + }; + topObj["symbols"] = { + {"number", symbols.size()}, + {"bytes", symbols.totalSize()}, + }; + topObj["sets"] = { + {"number", nrAttrsets}, + {"bytes", bAttrsets}, + {"elements", nrAttrsInAttrsets}, + }; + topObj["sizes"] = { + {"Env", sizeof(Env)}, + {"Value", sizeof(Value)}, + {"Bindings", sizeof(Bindings)}, + {"Attr", sizeof(Attr)}, + }; + topObj["nrOpUpdates"] = nrOpUpdates; + topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied; + topObj["nrThunks"] = nrThunks; + topObj["nrAvoided"] = nrAvoided; + topObj["nrLookups"] = nrLookups; + topObj["nrPrimOpCalls"] = nrPrimOpCalls; + topObj["nrFunctionCalls"] = nrFunctionCalls; #if HAVE_BOEHMGC - topObj["gc"] = { - {"heapSize", heapSize}, - {"totalBytes", totalBytes}, - }; + topObj["gc"] = { + {"heapSize", heapSize}, + {"totalBytes", totalBytes}, + }; #endif - if (countCalls) { - topObj["primops"] = primOpCalls; - { - auto& list = topObj["functions"]; - list = json::array(); - for (auto & [fun, count] : functionCalls) { - json obj = json::object(); - if (fun->name) - obj["name"] = (std::string_view) symbols[fun->name]; - else - obj["name"] = nullptr; - if (auto pos = positions[fun->pos]) { - if (auto path = std::get_if(&pos.origin)) - obj["file"] = path->to_string(); - obj["line"] = pos.line; - obj["column"] = pos.column; - } - obj["count"] = count; - list.push_back(obj); - } - } - { - auto list = topObj["attributes"]; - list = json::array(); - for (auto & i : attrSelects) { - json obj = json::object(); - if (auto pos = positions[i.first]) { - if (auto path = std::get_if(&pos.origin)) - obj["file"] = path->to_string(); - obj["line"] = pos.line; - obj["column"] = pos.column; - } - obj["count"] = i.second; - list.push_back(obj); + if (countCalls) { + topObj["primops"] = primOpCalls; + { + auto& list = topObj["functions"]; + list = json::array(); + for (auto & [fun, count] : functionCalls) { + json obj = json::object(); + if (fun->name) + obj["name"] = (std::string_view) symbols[fun->name]; + else + obj["name"] = nullptr; + if (auto pos = positions[fun->pos]) { + if (auto path = std::get_if(&pos.origin)) + obj["file"] = path->to_string(); + obj["line"] = pos.line; + obj["column"] = pos.column; } + obj["count"] = count; + list.push_back(obj); } } + { + auto list = topObj["attributes"]; + list = json::array(); + for (auto & i : attrSelects) { + json obj = json::object(); + if (auto pos = positions[i.first]) { + if (auto path = std::get_if(&pos.origin)) + obj["file"] = path->to_string(); + obj["line"] = pos.line; + obj["column"] = pos.column; + } + obj["count"] = i.second; + list.push_back(obj); + } + } + } - if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") { - // XXX: overrides earlier assignment - topObj["symbols"] = json::array(); - auto &list = topObj["symbols"]; - symbols.dump([&](const std::string & s) { list.emplace_back(s); }); - } - if (outPath == "-") { - std::cerr << topObj.dump(2) << std::endl; - } else { - fs << topObj.dump(2) << std::endl; - } + if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") { + // XXX: overrides earlier assignment + topObj["symbols"] = json::array(); + auto &list = topObj["symbols"]; + symbols.dump([&](const std::string & s) { list.emplace_back(s); }); + } + if (outPath == "-") { + std::cerr << topObj.dump(2) << std::endl; + } else { + fs << topObj.dump(2) << std::endl; } } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index fa8fa462b..3bfa34ef6 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -709,10 +709,26 @@ public: void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); /** - * Print statistics. + * Print statistics, if enabled. + * + * Performs a full memory GC before printing the statistics, so that the + * GC statistics are more accurate. */ void printStats(); + /** + * Print statistics, unconditionally, cheaply, without performing a GC first. + */ + void printStatistics(); + + /** + * Perform a full memory garbage collection - not incremental. + * + * @return true if Nix was built with GC and a GC was performed, false if not. + * The return value is currently not thread safe - just the return value. + */ + bool fullGC(); + /** * Realise the given context, and return a mapping from the placeholders * used to construct the associated value to their final store path From 369b076986531f3fbf6933539a0379b5d3fb1970 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 6 Oct 2023 11:46:41 +0200 Subject: [PATCH 350/909] add links and anchors --- doc/manual/src/command-ref/nix-env/install.md | 70 +++++++++---------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/doc/manual/src/command-ref/nix-env/install.md b/doc/manual/src/command-ref/nix-env/install.md index 95648e9e0..c1fff50e8 100644 --- a/doc/manual/src/command-ref/nix-env/install.md +++ b/doc/manual/src/command-ref/nix-env/install.md @@ -14,19 +14,21 @@ # Description -The install operation creates a new user environment, based on the -current generation of the active profile, to which a set of store paths -described by *args* is added. The arguments *args* map to store paths in -a number of possible ways: +The install operation creates a new user environment. +It is based on the current generation of the active [profile](@docroot@/command-ref/files/profiles.md), to which a set of [store paths] described by *args* is added. + +[store paths]: @docroot@/glossary.md#gloss-store-path + +The arguments *args* map to store paths in a number of possible ways: - - By default, *args* is a set of [derivation] names denoting derivations - in the active Nix expression. These are realised, and the resulting - output paths are installed. Currently installed derivations with a - name equal to the name of a derivation being added are removed - unless the option `--preserve-installed` is specified. + - By default, *args* is a set of [derivation] names denoting derivations in the [default Nix expression]. + These are [realised], and the resulting output paths are installed. + Currently installed derivations with a name equal to the name of a derivation being added are removed unless the option `--preserve-installed` is specified. - [derivation]: @docroot@/language/derivations.md + [derivation]: @docroot@/glossary.md#gloss-derivation + [default Nix expression]: @docroot@/command-ref/files/default-nix-expression.md + [realised]: @docroot@/glossary.md#gloss-realise If there are multiple derivations matching a name in *args* that have the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the @@ -43,40 +45,33 @@ a number of possible ways: gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will probably cause a user environment conflict\!). - - If `--attr` (`-A`) is specified, the arguments are *attribute - paths* that select attributes from the top-level Nix - expression. This is faster than using derivation names and - unambiguous. To find out the attribute paths of available - packages, use `nix-env --query --available --attr-path `. + - If [`--attr`](#opt-attr) / `-A` is specified, the arguments are *attribute paths* that select attributes from the [default Nix expression]. + This is faster than using derivation names and unambiguous. + Show the attribute paths of available packages with [`nix-env --query`](./query.md): + + ```console + nix-env --query --available --attr-path` + ``` - If `--from-profile` *path* is given, *args* is a set of names - denoting installed store paths in the profile *path*. This is an + denoting installed [store paths] in the profile *path*. This is an easy way to copy user environment elements from one profile to another. - - If `--from-expression` is given, *args* are Nix - [functions](@docroot@/language/constructs.md#functions) - that are called with the active Nix expression as their single - argument. The derivations returned by those function calls are - installed. This allows derivations to be specified in an - unambiguous way, which is necessary if there are multiple - derivations with the same name. + - If `--from-expression` is given, *args* are [Nix language functions](@docroot@/language/constructs.md#functions) that are called with the [default Nix expression] as their single argument. + The derivations returned by those function calls are installed. + This allows derivations to be specified in an unambiguous way, which is necessary if there are multiple derivations with the same name. - - If *args* are [store derivations](@docroot@/glossary.md#gloss-store-derivation), then these are - [realised](@docroot@/command-ref/nix-store/realise.md), and the resulting output paths - are installed. + - If *args* are [store derivations](@docroot@/glossary.md#gloss-store-derivation), then these are [realised], and the resulting output paths are installed. - - If *args* are store paths that are not store derivations, then these - are [realised](@docroot@/command-ref/nix-store/realise.md) and installed. + - If *args* are [store paths] that are not store derivations, then these are [realised] and installed. - - By default all outputs are installed for each derivation. + - By default all [outputs](@docroot@/language/derivations.md#attr-outputs) are installed for each [derivation]. This can be overridden by adding a `meta.outputsToInstall` attribute on the derivation listing a subset of the output names. - - Example: - The file `example.nix` defines a [derivation] with two outputs `foo` and `bar`, each containing a file. + The file `example.nix` defines a derivation with two outputs `foo` and `bar`, each containing a file. ```nix # example.nix @@ -123,15 +118,17 @@ a number of possible ways: manifest.nix ``` -# Flags +# Options + + - `--prebuilt-only` / `-b` - - `--prebuilt-only` / `-b`\ Use only derivations for which a substitute is registered, i.e., there is a pre-built binary available that can be downloaded in lieu of building the derivation. Thus, no packages will be built from source. - - `--preserve-installed` / `-P`\ + - `--preserve-installed` / `-P` + Do not remove derivations with a name matching one of the derivations being installed. Usually, trying to have two versions of the same package installed in the same generation of a profile will @@ -139,7 +136,8 @@ a number of possible ways: clashes between the two versions. However, this is not the case for all packages. - - `--remove-all` / `-r`\ + - `--remove-all` / `-r` + Remove all previously installed packages first. This is equivalent to running `nix-env --uninstall '.*'` first, except that everything happens in a single transaction. From 68c81c737571794f7246db53fb4774e94fcf4b7e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 5 Oct 2023 12:12:18 -0400 Subject: [PATCH 351/909] Put functional tests in `tests/functional` I think it is bad for these reasons when `tests/` contains a mix of functional and integration tests - Concepts is harder to understand, the documentation makes a good unit vs functional vs integration distinction, but when the integration tests are just two subdirs within `tests/` this is not clear. - Source filtering in the `flake.nix` is more complex. We need to filter out some of the dirs from `tests/`, rather than simply pick the dirs we want and take all of them. This is a good sign the structure of what we are trying to do is not matching the structure of the files. With this change we have a clean: ```shell-session $ git show 'HEAD:tests' tree HEAD:tests functional/ installer/ nixos/ ``` --- .github/labeler.yml | 2 +- .gitignore | 34 ++++++------ CONTRIBUTING.md | 2 +- Makefile | 10 ++-- doc/manual/src/contributing/testing.md | 26 ++++----- flake.nix | 52 ++++++++---------- mk/common-test.sh | 8 ++- src/libstore/gc.cc | 2 +- src/libutil/url-parts.hh | 2 +- tests/{ => functional}/add.sh | 0 tests/{ => functional}/bad.tar.xz | Bin tests/{ => functional}/bash-profile.sh | 2 +- .../{ => functional}/big-derivation-attr.nix | 0 .../binary-cache-build-remote.sh | 0 tests/{ => functional}/binary-cache.sh | 0 tests/{ => functional}/brotli.sh | 0 tests/{ => functional}/build-delete.sh | 0 tests/{ => functional}/build-dry.sh | 0 .../{ => functional}/build-hook-ca-fixed.nix | 0 .../build-hook-ca-floating.nix | 0 tests/{ => functional}/build-hook.nix | 0 .../build-remote-content-addressed-fixed.sh | 0 ...build-remote-content-addressed-floating.sh | 0 .../build-remote-input-addressed.sh | 0 .../build-remote-trustless-after.sh | 0 .../build-remote-trustless-should-fail-0.sh | 0 .../build-remote-trustless-should-pass-0.sh | 0 .../build-remote-trustless-should-pass-1.sh | 0 .../build-remote-trustless-should-pass-2.sh | 0 .../build-remote-trustless-should-pass-3.sh | 0 .../build-remote-trustless.sh | 2 +- tests/{ => functional}/build-remote.sh | 0 tests/{ => functional}/build.sh | 0 tests/{ => functional}/ca-shell.nix | 0 tests/{ => functional}/ca/build-cache.sh | 0 tests/{ => functional}/ca/build-dry.sh | 0 .../ca/build-with-garbage-path.sh | 0 tests/{ => functional}/ca/build.sh | 0 tests/{ => functional}/ca/common.sh | 0 .../{ => functional}/ca/concurrent-builds.sh | 0 tests/{ => functional}/ca/config.nix.in | 0 .../{ => functional}/ca/content-addressed.nix | 0 tests/{ => functional}/ca/derivation-json.sh | 0 .../ca/duplicate-realisation-in-closure.sh | 0 tests/{ => functional}/ca/flake.nix | 0 tests/{ => functional}/ca/gc.sh | 0 .../{ => functional}/ca/import-derivation.sh | 0 tests/{ => functional}/ca/local.mk | 2 +- tests/{ => functional}/ca/new-build-cmd.sh | 0 tests/{ => functional}/ca/nix-copy.sh | 0 tests/{ => functional}/ca/nix-run.sh | 0 tests/{ => functional}/ca/nix-shell.sh | 0 .../{ => functional}/ca/nondeterministic.nix | 0 tests/{ => functional}/ca/post-hook.sh | 0 tests/{ => functional}/ca/racy.nix | 0 tests/{ => functional}/ca/recursive.sh | 0 tests/{ => functional}/ca/repl.sh | 0 tests/{ => functional}/ca/selfref-gc.sh | 0 tests/{ => functional}/ca/signatures.sh | 0 tests/{ => functional}/ca/substitute.sh | 0 tests/{ => functional}/ca/why-depends.sh | 0 tests/{ => functional}/case-hack.sh | 0 tests/{ => functional}/case.nar | Bin tests/{ => functional}/check-refs.nix | 0 tests/{ => functional}/check-refs.sh | 0 tests/{ => functional}/check-reqs.nix | 0 tests/{ => functional}/check-reqs.sh | 0 tests/{ => functional}/check.nix | 0 tests/{ => functional}/check.sh | 0 tests/{ => functional}/common.sh | 0 .../common/vars-and-functions.sh.in | 2 +- tests/{ => functional}/completions.sh | 0 tests/{ => functional}/compression-levels.sh | 0 tests/{ => functional}/compute-levels.sh | 0 tests/{ => functional}/config.nix.in | 0 tests/{ => functional}/config.sh | 0 .../config/nix-with-substituters.conf | 0 tests/{ => functional}/db-migration.sh | 0 .../{ => functional}/dependencies.builder0.sh | 0 tests/{ => functional}/dependencies.nix | 0 tests/{ => functional}/dependencies.sh | 0 tests/{ => functional}/derivation-json.sh | 0 tests/{ => functional}/dummy | 0 tests/{ => functional}/dump-db.sh | 0 .../dyn-drv/build-built-drv.sh | 0 tests/{ => functional}/dyn-drv/common.sh | 0 tests/{ => functional}/dyn-drv/config.nix.in | 0 .../{ => functional}/dyn-drv/dep-built-drv.sh | 0 .../{ => functional}/dyn-drv/eval-outputOf.sh | 0 tests/{ => functional}/dyn-drv/local.mk | 2 +- .../dyn-drv/old-daemon-error-hack.nix | 0 .../dyn-drv/old-daemon-error-hack.sh | 0 .../dyn-drv/recursive-mod-json.nix | 0 .../dyn-drv/recursive-mod-json.sh | 0 .../dyn-drv/text-hashed-output.nix | 0 .../dyn-drv/text-hashed-output.sh | 0 tests/{ => functional}/eval-store.sh | 0 tests/{ => functional}/eval.nix | 0 tests/{ => functional}/eval.sh | 0 .../{ => functional}/experimental-features.sh | 0 tests/{ => functional}/export-graph.nix | 0 tests/{ => functional}/export-graph.sh | 0 tests/{ => functional}/export.sh | 0 tests/{ => functional}/failing.nix | 0 tests/{ => functional}/fetchClosure.sh | 0 tests/{ => functional}/fetchGit.sh | 0 tests/{ => functional}/fetchGitRefs.sh | 0 tests/{ => functional}/fetchGitSubmodules.sh | 0 tests/{ => functional}/fetchMercurial.sh | 0 tests/{ => functional}/fetchPath.sh | 0 tests/{ => functional}/fetchTree-file.sh | 0 tests/{ => functional}/fetchurl.sh | 0 tests/{ => functional}/filter-source.nix | 0 tests/{ => functional}/filter-source.sh | 0 tests/{ => functional}/fixed.builder1.sh | 0 tests/{ => functional}/fixed.builder2.sh | 0 tests/{ => functional}/fixed.nix | 0 tests/{ => functional}/fixed.sh | 0 .../flakes/absolute-attr-paths.sh | 0 .../{ => functional}/flakes/absolute-paths.sh | 0 tests/{ => functional}/flakes/build-paths.sh | 0 tests/{ => functional}/flakes/bundle.sh | 0 tests/{ => functional}/flakes/check.sh | 0 tests/{ => functional}/flakes/circular.sh | 0 tests/{ => functional}/flakes/common.sh | 0 tests/{ => functional}/flakes/config.sh | 0 .../flakes/flake-in-submodule.sh | 0 tests/{ => functional}/flakes/flakes.sh | 0 tests/{ => functional}/flakes/follow-paths.sh | 0 tests/{ => functional}/flakes/init.sh | 0 tests/{ => functional}/flakes/inputs.sh | 0 tests/{ => functional}/flakes/mercurial.sh | 0 tests/{ => functional}/flakes/run.sh | 0 tests/{ => functional}/flakes/search-root.sh | 0 tests/{ => functional}/flakes/show.sh | 0 .../flakes/unlocked-override.sh | 0 tests/{ => functional}/fmt.sh | 0 tests/{ => functional}/fmt.simple.sh | 0 tests/{ => functional}/function-trace.sh | 0 tests/{ => functional}/gc-auto.sh | 0 .../{ => functional}/gc-concurrent.builder.sh | 0 tests/{ => functional}/gc-concurrent.nix | 0 tests/{ => functional}/gc-concurrent.sh | 0 .../gc-concurrent2.builder.sh | 0 tests/{ => functional}/gc-non-blocking.sh | 0 tests/{ => functional}/gc-runtime.nix | 0 tests/{ => functional}/gc-runtime.sh | 0 tests/{ => functional}/gc.sh | 0 tests/{ => functional}/hash-check.nix | 0 tests/{ => functional}/hash.sh | 0 tests/{ => functional}/hermetic.nix | 0 tests/{ => functional}/import-derivation.nix | 0 tests/{ => functional}/import-derivation.sh | 0 tests/{ => functional}/impure-derivations.nix | 0 tests/{ => functional}/impure-derivations.sh | 0 tests/{ => functional}/init.sh | 0 tests/{ => functional}/install-darwin.sh | 0 tests/{ => functional}/lang-test-infra.sh | 0 tests/{ => functional}/lang.sh | 2 +- tests/{ => functional}/lang/binary-data | Bin tests/{ => functional}/lang/data | 0 tests/{ => functional}/lang/dir1/a.nix | 0 tests/{ => functional}/lang/dir2/a.nix | 0 tests/{ => functional}/lang/dir2/b.nix | 0 tests/{ => functional}/lang/dir3/a.nix | 0 tests/{ => functional}/lang/dir3/b.nix | 0 tests/{ => functional}/lang/dir3/c.nix | 0 tests/{ => functional}/lang/dir4/a.nix | 0 tests/{ => functional}/lang/dir4/c.nix | 0 tests/{ => functional}/lang/empty.exp | 0 .../lang/eval-fail-abort.err.exp | 0 .../{ => functional}/lang/eval-fail-abort.nix | 0 .../lang/eval-fail-antiquoted-path.err.exp | 0 .../lang/eval-fail-assert.err.exp | 0 .../lang/eval-fail-assert.nix | 0 .../lang/eval-fail-bad-antiquote-1.err.exp | 0 .../lang/eval-fail-bad-antiquote-2.err.exp | 0 .../lang/eval-fail-bad-antiquote-3.err.exp | 0 ...al-fail-bad-string-interpolation-1.err.exp | 0 .../eval-fail-bad-string-interpolation-1.nix | 0 ...al-fail-bad-string-interpolation-2.err.exp | 0 .../eval-fail-bad-string-interpolation-2.nix | 0 ...al-fail-bad-string-interpolation-3.err.exp | 0 .../eval-fail-bad-string-interpolation-3.nix | 0 .../lang/eval-fail-blackhole.err.exp | 0 .../lang/eval-fail-blackhole.nix | 0 .../lang/eval-fail-deepseq.err.exp | 0 .../lang/eval-fail-deepseq.nix | 0 .../lang/eval-fail-dup-dynamic-attrs.err.exp | 0 .../lang/eval-fail-dup-dynamic-attrs.nix | 0 ...-foldlStrict-strict-op-application.err.exp | 0 ...fail-foldlStrict-strict-op-application.nix | 0 .../eval-fail-fromTOML-timestamps.err.exp | 0 .../lang/eval-fail-fromTOML-timestamps.nix | 0 .../lang/eval-fail-hashfile-missing.err.exp | 0 .../lang/eval-fail-hashfile-missing.nix | 0 .../lang/eval-fail-list.err.exp | 0 .../{ => functional}/lang/eval-fail-list.nix | 0 .../lang/eval-fail-missing-arg.err.exp | 0 .../lang/eval-fail-missing-arg.nix | 0 .../lang/eval-fail-nonexist-path.err.exp | 0 .../lang/eval-fail-nonexist-path.nix | 0 .../lang/eval-fail-path-slash.err.exp | 0 .../lang/eval-fail-path-slash.nix | 0 .../lang/eval-fail-recursion.err.exp | 0 .../lang/eval-fail-recursion.nix | 0 .../lang/eval-fail-remove.err.exp | 0 .../lang/eval-fail-remove.nix | 0 .../lang/eval-fail-scope-5.err.exp | 0 .../lang/eval-fail-scope-5.nix | 0 .../lang/eval-fail-seq.err.exp | 0 tests/{ => functional}/lang/eval-fail-seq.nix | 0 .../lang/eval-fail-set-override.err.exp | 0 .../lang/eval-fail-set-override.nix | 0 .../lang/eval-fail-set.err.exp | 0 tests/{ => functional}/lang/eval-fail-set.nix | 0 .../lang/eval-fail-substring.err.exp | 0 .../lang/eval-fail-substring.nix | 0 .../lang/eval-fail-to-path.err.exp | 0 .../lang/eval-fail-to-path.nix | 0 .../lang/eval-fail-toJSON.err.exp | 0 .../lang/eval-fail-toJSON.nix | 0 .../lang/eval-fail-undeclared-arg.err.exp | 0 .../lang/eval-fail-undeclared-arg.nix | 0 .../lang/eval-okay-any-all.exp | 0 .../lang/eval-okay-any-all.nix | 0 .../lang/eval-okay-arithmetic.exp | 0 .../lang/eval-okay-arithmetic.nix | 0 .../lang/eval-okay-attrnames.exp | 0 .../lang/eval-okay-attrnames.nix | 0 .../{ => functional}/lang/eval-okay-attrs.exp | 0 .../{ => functional}/lang/eval-okay-attrs.nix | 0 .../lang/eval-okay-attrs2.exp | 0 .../lang/eval-okay-attrs2.nix | 0 .../lang/eval-okay-attrs3.exp | 0 .../lang/eval-okay-attrs3.nix | 0 .../lang/eval-okay-attrs4.exp | 0 .../lang/eval-okay-attrs4.nix | 0 .../lang/eval-okay-attrs5.exp | 0 .../lang/eval-okay-attrs5.nix | 0 .../lang/eval-okay-attrs6.exp | 0 .../lang/eval-okay-attrs6.nix | 0 .../lang/eval-okay-autoargs.exp | 0 .../lang/eval-okay-autoargs.flags | 0 .../lang/eval-okay-autoargs.nix | 0 .../lang/eval-okay-backslash-newline-1.exp | 0 .../lang/eval-okay-backslash-newline-1.nix | 0 .../lang/eval-okay-backslash-newline-2.exp | 0 .../lang/eval-okay-backslash-newline-2.nix | 0 .../lang/eval-okay-builtins-add.exp | 0 .../lang/eval-okay-builtins-add.nix | 0 .../lang/eval-okay-builtins.exp | 0 .../lang/eval-okay-builtins.nix | 0 .../lang/eval-okay-callable-attrs.exp | 0 .../lang/eval-okay-callable-attrs.nix | 0 .../lang/eval-okay-catattrs.exp | 0 .../lang/eval-okay-catattrs.nix | 0 .../lang/eval-okay-closure.exp | 0 .../lang/eval-okay-closure.exp.xml | 0 .../lang/eval-okay-closure.nix | 0 .../lang/eval-okay-comments.exp | 0 .../lang/eval-okay-comments.nix | 0 .../lang/eval-okay-concat.exp | 0 .../lang/eval-okay-concat.nix | 0 .../lang/eval-okay-concatmap.exp | 0 .../lang/eval-okay-concatmap.nix | 0 .../lang/eval-okay-concatstringssep.exp | 0 .../lang/eval-okay-concatstringssep.nix | 0 .../lang/eval-okay-context-introspection.exp | 0 .../lang/eval-okay-context-introspection.nix | 0 .../lang/eval-okay-context.exp | 0 .../lang/eval-okay-context.nix | 0 .../lang/eval-okay-curpos.exp | 0 .../lang/eval-okay-curpos.nix | 0 .../lang/eval-okay-deepseq.exp | 0 .../lang/eval-okay-deepseq.nix | 0 .../lang/eval-okay-delayed-with-inherit.exp | 0 .../lang/eval-okay-delayed-with-inherit.nix | 0 .../lang/eval-okay-delayed-with.exp | 0 .../lang/eval-okay-delayed-with.nix | 0 .../lang/eval-okay-dynamic-attrs-2.exp | 0 .../lang/eval-okay-dynamic-attrs-2.nix | 0 .../lang/eval-okay-dynamic-attrs-bare.exp | 0 .../lang/eval-okay-dynamic-attrs-bare.nix | 0 .../lang/eval-okay-dynamic-attrs.exp | 0 .../lang/eval-okay-dynamic-attrs.nix | 0 .../{ => functional}/lang/eval-okay-elem.exp | 0 .../{ => functional}/lang/eval-okay-elem.nix | 0 .../lang/eval-okay-empty-args.exp | 0 .../lang/eval-okay-empty-args.nix | 0 .../lang/eval-okay-eq-derivations.exp | 0 .../lang/eval-okay-eq-derivations.nix | 0 tests/{ => functional}/lang/eval-okay-eq.exp | 0 tests/{ => functional}/lang/eval-okay-eq.nix | 0 .../lang/eval-okay-filter.exp | 0 .../lang/eval-okay-filter.nix | 0 .../lang/eval-okay-flake-ref-to-string.exp | 0 .../lang/eval-okay-flake-ref-to-string.nix | 0 .../lang/eval-okay-flatten.exp | 0 .../lang/eval-okay-flatten.nix | 0 .../{ => functional}/lang/eval-okay-float.exp | 0 .../{ => functional}/lang/eval-okay-float.nix | 0 .../lang/eval-okay-floor-ceil.exp | 0 .../lang/eval-okay-floor-ceil.nix | 0 .../eval-okay-foldlStrict-lazy-elements.exp | 0 .../eval-okay-foldlStrict-lazy-elements.nix | 0 ...y-foldlStrict-lazy-initial-accumulator.exp | 0 ...y-foldlStrict-lazy-initial-accumulator.nix | 0 .../lang/eval-okay-foldlStrict.exp | 0 .../lang/eval-okay-foldlStrict.nix | 0 .../lang/eval-okay-fromTOML-timestamps.exp | 0 .../lang/eval-okay-fromTOML-timestamps.flags | 0 .../lang/eval-okay-fromTOML-timestamps.nix | 0 .../lang/eval-okay-fromTOML.exp | 0 .../lang/eval-okay-fromTOML.nix | 0 .../lang/eval-okay-fromjson-escapes.exp | 0 .../lang/eval-okay-fromjson-escapes.nix | 0 .../lang/eval-okay-fromjson.exp | 0 .../lang/eval-okay-fromjson.nix | 0 .../lang/eval-okay-functionargs.exp | 0 .../lang/eval-okay-functionargs.exp.xml | 0 .../lang/eval-okay-functionargs.nix | 0 .../eval-okay-getattrpos-functionargs.exp | 0 .../eval-okay-getattrpos-functionargs.nix | 0 .../lang/eval-okay-getattrpos-undefined.exp | 0 .../lang/eval-okay-getattrpos-undefined.nix | 0 .../lang/eval-okay-getattrpos.exp | 0 .../lang/eval-okay-getattrpos.nix | 0 .../lang/eval-okay-getenv.exp | 0 .../lang/eval-okay-getenv.nix | 0 .../lang/eval-okay-groupBy.exp | 0 .../lang/eval-okay-groupBy.nix | 0 .../{ => functional}/lang/eval-okay-hash.exp | 0 .../lang/eval-okay-hashfile.exp | 0 .../lang/eval-okay-hashfile.nix | 0 .../lang/eval-okay-hashstring.exp | 0 .../lang/eval-okay-hashstring.nix | 0 tests/{ => functional}/lang/eval-okay-if.exp | 0 tests/{ => functional}/lang/eval-okay-if.nix | 0 .../lang/eval-okay-import.exp | 0 .../lang/eval-okay-import.nix | 0 .../lang/eval-okay-ind-string.exp | 0 .../lang/eval-okay-ind-string.nix | 0 .../lang/eval-okay-intersectAttrs.exp | 0 .../lang/eval-okay-intersectAttrs.nix | 0 tests/{ => functional}/lang/eval-okay-let.exp | 0 tests/{ => functional}/lang/eval-okay-let.nix | 0 .../{ => functional}/lang/eval-okay-list.exp | 0 .../{ => functional}/lang/eval-okay-list.nix | 0 .../lang/eval-okay-listtoattrs.exp | 0 .../lang/eval-okay-listtoattrs.nix | 0 .../{ => functional}/lang/eval-okay-logic.exp | 0 .../{ => functional}/lang/eval-okay-logic.nix | 0 tests/{ => functional}/lang/eval-okay-map.exp | 0 tests/{ => functional}/lang/eval-okay-map.nix | 0 .../lang/eval-okay-mapattrs.exp | 0 .../lang/eval-okay-mapattrs.nix | 0 .../lang/eval-okay-merge-dynamic-attrs.exp | 0 .../lang/eval-okay-merge-dynamic-attrs.nix | 0 .../lang/eval-okay-nested-with.exp | 0 .../lang/eval-okay-nested-with.nix | 0 .../lang/eval-okay-new-let.exp | 0 .../lang/eval-okay-new-let.nix | 0 .../lang/eval-okay-null-dynamic-attrs.exp | 0 .../lang/eval-okay-null-dynamic-attrs.nix | 0 .../lang/eval-okay-overrides.exp | 0 .../lang/eval-okay-overrides.nix | 0 .../lang/eval-okay-parse-flake-ref.exp | 0 .../lang/eval-okay-parse-flake-ref.nix | 0 .../lang/eval-okay-partition.exp | 0 .../lang/eval-okay-partition.nix | 0 .../eval-okay-path-string-interpolation.exp | 0 .../eval-okay-path-string-interpolation.nix | 0 .../{ => functional}/lang/eval-okay-path.exp | 0 .../{ => functional}/lang/eval-okay-path.nix | 0 .../lang/eval-okay-pathexists.exp | 0 .../lang/eval-okay-pathexists.nix | 0 .../lang/eval-okay-patterns.exp | 0 .../lang/eval-okay-patterns.nix | 0 .../lang/eval-okay-print.err.exp | 0 .../{ => functional}/lang/eval-okay-print.exp | 0 .../{ => functional}/lang/eval-okay-print.nix | 0 .../lang/eval-okay-readDir.exp | 0 .../lang/eval-okay-readDir.nix | 0 .../lang/eval-okay-readFileType.exp | 0 .../lang/eval-okay-readFileType.nix | 0 .../lang/eval-okay-readfile.exp | 0 .../lang/eval-okay-readfile.nix | 0 .../lang/eval-okay-redefine-builtin.exp | 0 .../lang/eval-okay-redefine-builtin.nix | 0 .../lang/eval-okay-regex-match.exp | 0 .../lang/eval-okay-regex-match.nix | 0 .../lang/eval-okay-regex-split.exp | 0 .../lang/eval-okay-regex-split.nix | 0 .../lang/eval-okay-regression-20220122.exp | 0 .../lang/eval-okay-regression-20220122.nix | 0 .../lang/eval-okay-regression-20220125.exp | 0 .../lang/eval-okay-regression-20220125.nix | 0 .../lang/eval-okay-remove.exp | 0 .../lang/eval-okay-remove.nix | 0 .../lang/eval-okay-replacestrings.exp | 0 .../lang/eval-okay-replacestrings.nix | 0 .../lang/eval-okay-scope-1.exp | 0 .../lang/eval-okay-scope-1.nix | 0 .../lang/eval-okay-scope-2.exp | 0 .../lang/eval-okay-scope-2.nix | 0 .../lang/eval-okay-scope-3.exp | 0 .../lang/eval-okay-scope-3.nix | 0 .../lang/eval-okay-scope-4.exp | 0 .../lang/eval-okay-scope-4.nix | 0 .../lang/eval-okay-scope-6.exp | 0 .../lang/eval-okay-scope-6.nix | 0 .../lang/eval-okay-scope-7.exp | 0 .../lang/eval-okay-scope-7.nix | 0 .../lang/eval-okay-search-path.exp | 0 .../lang/eval-okay-search-path.flags | 0 .../lang/eval-okay-search-path.nix | 0 tests/{ => functional}/lang/eval-okay-seq.exp | 0 tests/{ => functional}/lang/eval-okay-seq.nix | 0 .../{ => functional}/lang/eval-okay-sort.exp | 0 .../{ => functional}/lang/eval-okay-sort.nix | 0 .../lang/eval-okay-splitversion.exp | 0 .../lang/eval-okay-splitversion.nix | 0 .../lang/eval-okay-string.exp | 0 .../lang/eval-okay-string.nix | 0 .../lang/eval-okay-strings-as-attrs-names.exp | 0 .../lang/eval-okay-strings-as-attrs-names.nix | 0 .../lang/eval-okay-substring.exp | 0 .../lang/eval-okay-substring.nix | 0 .../lang/eval-okay-tail-call-1.exp-disabled | 0 .../lang/eval-okay-tail-call-1.nix | 0 .../lang/eval-okay-tojson.exp | 0 .../lang/eval-okay-tojson.nix | 0 .../{ => functional}/lang/eval-okay-toxml.exp | 0 .../{ => functional}/lang/eval-okay-toxml.nix | 0 .../lang/eval-okay-toxml2.exp | 0 .../lang/eval-okay-toxml2.nix | 0 .../lang/eval-okay-tryeval.exp | 0 .../lang/eval-okay-tryeval.nix | 0 .../{ => functional}/lang/eval-okay-types.exp | 0 .../{ => functional}/lang/eval-okay-types.nix | 0 .../lang/eval-okay-versions.exp | 0 .../lang/eval-okay-versions.nix | 0 .../{ => functional}/lang/eval-okay-with.exp | 0 .../{ => functional}/lang/eval-okay-with.nix | 0 .../lang/eval-okay-xml.exp.xml | 0 tests/{ => functional}/lang/eval-okay-xml.nix | 0 .../lang/eval-okay-zipAttrsWith.exp | 0 .../lang/eval-okay-zipAttrsWith.nix | 0 tests/{ => functional}/lang/framework.sh | 0 tests/{ => functional}/lang/imported.nix | 0 tests/{ => functional}/lang/imported2.nix | 0 tests/{ => functional}/lang/lib.nix | 0 .../lang/parse-fail-dup-attrs-1.err.exp | 0 .../lang/parse-fail-dup-attrs-1.nix | 0 .../lang/parse-fail-dup-attrs-2.err.exp | 0 .../lang/parse-fail-dup-attrs-2.nix | 0 .../lang/parse-fail-dup-attrs-3.err.exp | 0 .../lang/parse-fail-dup-attrs-3.nix | 0 .../lang/parse-fail-dup-attrs-4.err.exp | 0 .../lang/parse-fail-dup-attrs-4.nix | 0 .../lang/parse-fail-dup-attrs-6.err.exp | 0 .../lang/parse-fail-dup-attrs-7.err.exp | 0 .../lang/parse-fail-dup-attrs-7.nix | 0 .../lang/parse-fail-dup-formals.err.exp | 0 .../lang/parse-fail-dup-formals.nix | 0 .../lang/parse-fail-eof-in-string.err.exp | 0 .../lang/parse-fail-eof-in-string.nix | 0 .../parse-fail-mixed-nested-attrs1.err.exp | 0 .../lang/parse-fail-mixed-nested-attrs1.nix | 0 .../parse-fail-mixed-nested-attrs2.err.exp | 0 .../lang/parse-fail-mixed-nested-attrs2.nix | 0 .../lang/parse-fail-patterns-1.err.exp | 0 .../lang/parse-fail-patterns-1.nix | 0 .../parse-fail-regression-20060610.err.exp | 0 .../lang/parse-fail-regression-20060610.nix | 0 .../lang/parse-fail-undef-var-2.err.exp | 0 .../lang/parse-fail-undef-var-2.nix | 0 .../lang/parse-fail-undef-var.err.exp | 0 .../lang/parse-fail-undef-var.nix | 0 .../lang/parse-fail-utf8.err.exp | 0 .../{ => functional}/lang/parse-fail-utf8.nix | 0 tests/{ => functional}/lang/parse-okay-1.exp | 0 tests/{ => functional}/lang/parse-okay-1.nix | 0 .../{ => functional}/lang/parse-okay-crlf.exp | 0 .../{ => functional}/lang/parse-okay-crlf.nix | 0 .../lang/parse-okay-dup-attrs-5.exp | 0 .../lang/parse-okay-dup-attrs-5.nix | 0 .../lang/parse-okay-dup-attrs-6.exp | 0 .../lang/parse-okay-dup-attrs-6.nix | 0 .../lang/parse-okay-mixed-nested-attrs-1.exp | 0 .../lang/parse-okay-mixed-nested-attrs-1.nix | 0 .../lang/parse-okay-mixed-nested-attrs-2.exp | 0 .../lang/parse-okay-mixed-nested-attrs-2.nix | 0 .../lang/parse-okay-mixed-nested-attrs-3.exp | 0 .../lang/parse-okay-mixed-nested-attrs-3.nix | 0 .../lang/parse-okay-regression-20041027.exp | 0 .../lang/parse-okay-regression-20041027.nix | 0 .../lang/parse-okay-regression-751.exp | 0 .../lang/parse-okay-regression-751.nix | 0 .../lang/parse-okay-subversion.exp | 0 .../lang/parse-okay-subversion.nix | 0 .../{ => functional}/lang/parse-okay-url.exp | 0 .../{ => functional}/lang/parse-okay-url.nix | 0 tests/{ => functional}/lang/readDir/bar | 0 .../lang/readDir/foo/git-hates-directories | 0 tests/{ => functional}/lang/readDir/ldir | 0 tests/{ => functional}/lang/readDir/linked | 0 tests/{ => functional}/legacy-ssh-store.sh | 0 .../linux-sandbox-cert-test.nix | 0 tests/{ => functional}/linux-sandbox.sh | 0 tests/{ => functional}/local-store.sh | 0 tests/{ => functional}/local.mk | 13 +++-- tests/{ => functional}/logging.sh | 0 tests/{ => functional}/misc.sh | 0 tests/{ => functional}/multiple-outputs.nix | 0 tests/{ => functional}/multiple-outputs.sh | 0 tests/{ => functional}/nar-access.nix | 0 tests/{ => functional}/nar-access.sh | 0 tests/{ => functional}/nested-sandboxing.sh | 2 +- .../nested-sandboxing/command.sh | 0 .../nested-sandboxing/runner.nix | 0 tests/{ => functional}/nix-build-examples.nix | 0 tests/{ => functional}/nix-build.sh | 0 tests/{ => functional}/nix-channel.sh | 0 .../{ => functional}/nix-collect-garbage-d.sh | 0 tests/{ => functional}/nix-copy-ssh-ng.sh | 0 tests/{ => functional}/nix-copy-ssh.sh | 0 .../{ => functional}/nix-daemon-untrusting.sh | 0 tests/{ => functional}/nix-profile.sh | 0 tests/{ => functional}/nix-shell.sh | 0 tests/{ => functional}/nix_path.sh | 0 tests/{ => functional}/optimise-store.sh | 0 .../{ => functional}/output-normalization.sh | 0 tests/{ => functional}/parallel.builder.sh | 0 tests/{ => functional}/parallel.nix | 0 tests/{ => functional}/parallel.sh | 0 tests/{ => functional}/pass-as-file.sh | 0 tests/{ => functional}/path-from-hash-part.sh | 0 tests/{ => functional}/path.nix | 0 tests/{ => functional}/placeholders.sh | 0 tests/{ => functional}/plugins.sh | 0 tests/{ => functional}/plugins/local.mk | 0 tests/{ => functional}/plugins/plugintest.cc | 0 tests/{ => functional}/post-hook.sh | 0 tests/{ => functional}/pure-eval.nix | 0 tests/{ => functional}/pure-eval.sh | 0 tests/{ => functional}/push-to-store-old.sh | 0 tests/{ => functional}/push-to-store.sh | 0 tests/{ => functional}/read-only-store.sh | 0 tests/{ => functional}/readfile-context.nix | 0 tests/{ => functional}/readfile-context.sh | 0 tests/{ => functional}/recursive.nix | 0 tests/{ => functional}/recursive.sh | 0 tests/{ => functional}/referrers.sh | 0 tests/{ => functional}/remote-store.sh | 0 tests/{ => functional}/repair.sh | 0 tests/{ => functional}/repl.sh | 0 tests/{ => functional}/restricted.nix | 0 tests/{ => functional}/restricted.sh | 6 +- tests/{ => functional}/search.nix | 0 tests/{ => functional}/search.sh | 0 tests/{ => functional}/secure-drv-outputs.nix | 0 tests/{ => functional}/secure-drv-outputs.sh | 0 tests/{ => functional}/selfref-gc.sh | 0 tests/{ => functional}/shell-hello.nix | 0 tests/{ => functional}/shell.nix | 0 tests/{ => functional}/shell.sh | 0 tests/{ => functional}/shell.shebang.rb | 0 tests/{ => functional}/shell.shebang.sh | 0 tests/{ => functional}/signing.sh | 0 tests/{ => functional}/simple-failing.nix | 0 tests/{ => functional}/simple.builder.sh | 0 tests/{ => functional}/simple.nix | 0 tests/{ => functional}/simple.sh | 0 tests/{ => functional}/ssh-relay.sh | 0 tests/{ => functional}/store-ping.sh | 0 .../structured-attrs-shell.nix | 0 tests/{ => functional}/structured-attrs.nix | 0 tests/{ => functional}/structured-attrs.sh | 0 .../substitute-with-invalid-ca.sh | 0 tests/{ => functional}/suggestions.sh | 0 .../{ => functional}/supplementary-groups.sh | 0 tests/{ => functional}/tarball.sh | 0 tests/{ => functional}/test-infra.sh | 0 .../{ => functional}/test-libstoreconsumer.sh | 0 .../test-libstoreconsumer/README.md | 0 .../test-libstoreconsumer/local.mk | 0 .../test-libstoreconsumer/main.cc | 0 tests/{ => functional}/timeout.nix | 0 tests/{ => functional}/timeout.sh | 0 tests/{ => functional}/toString-path.sh | 0 tests/{ => functional}/undefined-variable.nix | 0 tests/{ => functional}/user-envs-migration.sh | 0 tests/{ => functional}/user-envs.builder.sh | 0 tests/{ => functional}/user-envs.nix | 0 tests/{ => functional}/user-envs.sh | 0 tests/{ => functional}/why-depends.sh | 0 tests/{ => functional}/zstd.sh | 0 599 files changed, 84 insertions(+), 87 deletions(-) rename tests/{ => functional}/add.sh (100%) rename tests/{ => functional}/bad.tar.xz (100%) rename tests/{ => functional}/bash-profile.sh (78%) rename tests/{ => functional}/big-derivation-attr.nix (100%) rename tests/{ => functional}/binary-cache-build-remote.sh (100%) rename tests/{ => functional}/binary-cache.sh (100%) rename tests/{ => functional}/brotli.sh (100%) rename tests/{ => functional}/build-delete.sh (100%) rename tests/{ => functional}/build-dry.sh (100%) rename tests/{ => functional}/build-hook-ca-fixed.nix (100%) rename tests/{ => functional}/build-hook-ca-floating.nix (100%) rename tests/{ => functional}/build-hook.nix (100%) rename tests/{ => functional}/build-remote-content-addressed-fixed.sh (100%) rename tests/{ => functional}/build-remote-content-addressed-floating.sh (100%) rename tests/{ => functional}/build-remote-input-addressed.sh (100%) rename tests/{ => functional}/build-remote-trustless-after.sh (100%) rename tests/{ => functional}/build-remote-trustless-should-fail-0.sh (100%) rename tests/{ => functional}/build-remote-trustless-should-pass-0.sh (100%) rename tests/{ => functional}/build-remote-trustless-should-pass-1.sh (100%) rename tests/{ => functional}/build-remote-trustless-should-pass-2.sh (100%) rename tests/{ => functional}/build-remote-trustless-should-pass-3.sh (100%) rename tests/{ => functional}/build-remote-trustless.sh (82%) rename tests/{ => functional}/build-remote.sh (100%) rename tests/{ => functional}/build.sh (100%) rename tests/{ => functional}/ca-shell.nix (100%) rename tests/{ => functional}/ca/build-cache.sh (100%) rename tests/{ => functional}/ca/build-dry.sh (100%) rename tests/{ => functional}/ca/build-with-garbage-path.sh (100%) rename tests/{ => functional}/ca/build.sh (100%) rename tests/{ => functional}/ca/common.sh (100%) rename tests/{ => functional}/ca/concurrent-builds.sh (100%) rename tests/{ => functional}/ca/config.nix.in (100%) rename tests/{ => functional}/ca/content-addressed.nix (100%) rename tests/{ => functional}/ca/derivation-json.sh (100%) rename tests/{ => functional}/ca/duplicate-realisation-in-closure.sh (100%) rename tests/{ => functional}/ca/flake.nix (100%) rename tests/{ => functional}/ca/gc.sh (100%) rename tests/{ => functional}/ca/import-derivation.sh (100%) rename tests/{ => functional}/ca/local.mk (94%) rename tests/{ => functional}/ca/new-build-cmd.sh (100%) rename tests/{ => functional}/ca/nix-copy.sh (100%) rename tests/{ => functional}/ca/nix-run.sh (100%) rename tests/{ => functional}/ca/nix-shell.sh (100%) rename tests/{ => functional}/ca/nondeterministic.nix (100%) rename tests/{ => functional}/ca/post-hook.sh (100%) rename tests/{ => functional}/ca/racy.nix (100%) rename tests/{ => functional}/ca/recursive.sh (100%) rename tests/{ => functional}/ca/repl.sh (100%) rename tests/{ => functional}/ca/selfref-gc.sh (100%) rename tests/{ => functional}/ca/signatures.sh (100%) rename tests/{ => functional}/ca/substitute.sh (100%) rename tests/{ => functional}/ca/why-depends.sh (100%) rename tests/{ => functional}/case-hack.sh (100%) rename tests/{ => functional}/case.nar (100%) rename tests/{ => functional}/check-refs.nix (100%) rename tests/{ => functional}/check-refs.sh (100%) rename tests/{ => functional}/check-reqs.nix (100%) rename tests/{ => functional}/check-reqs.sh (100%) rename tests/{ => functional}/check.nix (100%) rename tests/{ => functional}/check.sh (100%) rename tests/{ => functional}/common.sh (100%) rename tests/{ => functional}/common/vars-and-functions.sh.in (99%) rename tests/{ => functional}/completions.sh (100%) rename tests/{ => functional}/compression-levels.sh (100%) rename tests/{ => functional}/compute-levels.sh (100%) rename tests/{ => functional}/config.nix.in (100%) rename tests/{ => functional}/config.sh (100%) rename tests/{ => functional}/config/nix-with-substituters.conf (100%) rename tests/{ => functional}/db-migration.sh (100%) rename tests/{ => functional}/dependencies.builder0.sh (100%) rename tests/{ => functional}/dependencies.nix (100%) rename tests/{ => functional}/dependencies.sh (100%) rename tests/{ => functional}/derivation-json.sh (100%) rename tests/{ => functional}/dummy (100%) rename tests/{ => functional}/dump-db.sh (100%) rename tests/{ => functional}/dyn-drv/build-built-drv.sh (100%) rename tests/{ => functional}/dyn-drv/common.sh (100%) rename tests/{ => functional}/dyn-drv/config.nix.in (100%) rename tests/{ => functional}/dyn-drv/dep-built-drv.sh (100%) rename tests/{ => functional}/dyn-drv/eval-outputOf.sh (100%) rename tests/{ => functional}/dyn-drv/local.mk (87%) rename tests/{ => functional}/dyn-drv/old-daemon-error-hack.nix (100%) rename tests/{ => functional}/dyn-drv/old-daemon-error-hack.sh (100%) rename tests/{ => functional}/dyn-drv/recursive-mod-json.nix (100%) rename tests/{ => functional}/dyn-drv/recursive-mod-json.sh (100%) rename tests/{ => functional}/dyn-drv/text-hashed-output.nix (100%) rename tests/{ => functional}/dyn-drv/text-hashed-output.sh (100%) rename tests/{ => functional}/eval-store.sh (100%) rename tests/{ => functional}/eval.nix (100%) rename tests/{ => functional}/eval.sh (100%) rename tests/{ => functional}/experimental-features.sh (100%) rename tests/{ => functional}/export-graph.nix (100%) rename tests/{ => functional}/export-graph.sh (100%) rename tests/{ => functional}/export.sh (100%) rename tests/{ => functional}/failing.nix (100%) rename tests/{ => functional}/fetchClosure.sh (100%) rename tests/{ => functional}/fetchGit.sh (100%) rename tests/{ => functional}/fetchGitRefs.sh (100%) rename tests/{ => functional}/fetchGitSubmodules.sh (100%) rename tests/{ => functional}/fetchMercurial.sh (100%) rename tests/{ => functional}/fetchPath.sh (100%) rename tests/{ => functional}/fetchTree-file.sh (100%) rename tests/{ => functional}/fetchurl.sh (100%) rename tests/{ => functional}/filter-source.nix (100%) rename tests/{ => functional}/filter-source.sh (100%) rename tests/{ => functional}/fixed.builder1.sh (100%) rename tests/{ => functional}/fixed.builder2.sh (100%) rename tests/{ => functional}/fixed.nix (100%) rename tests/{ => functional}/fixed.sh (100%) rename tests/{ => functional}/flakes/absolute-attr-paths.sh (100%) rename tests/{ => functional}/flakes/absolute-paths.sh (100%) rename tests/{ => functional}/flakes/build-paths.sh (100%) rename tests/{ => functional}/flakes/bundle.sh (100%) rename tests/{ => functional}/flakes/check.sh (100%) rename tests/{ => functional}/flakes/circular.sh (100%) rename tests/{ => functional}/flakes/common.sh (100%) rename tests/{ => functional}/flakes/config.sh (100%) rename tests/{ => functional}/flakes/flake-in-submodule.sh (100%) rename tests/{ => functional}/flakes/flakes.sh (100%) rename tests/{ => functional}/flakes/follow-paths.sh (100%) rename tests/{ => functional}/flakes/init.sh (100%) rename tests/{ => functional}/flakes/inputs.sh (100%) rename tests/{ => functional}/flakes/mercurial.sh (100%) rename tests/{ => functional}/flakes/run.sh (100%) rename tests/{ => functional}/flakes/search-root.sh (100%) rename tests/{ => functional}/flakes/show.sh (100%) rename tests/{ => functional}/flakes/unlocked-override.sh (100%) rename tests/{ => functional}/fmt.sh (100%) rename tests/{ => functional}/fmt.simple.sh (100%) rename tests/{ => functional}/function-trace.sh (100%) rename tests/{ => functional}/gc-auto.sh (100%) rename tests/{ => functional}/gc-concurrent.builder.sh (100%) rename tests/{ => functional}/gc-concurrent.nix (100%) rename tests/{ => functional}/gc-concurrent.sh (100%) rename tests/{ => functional}/gc-concurrent2.builder.sh (100%) rename tests/{ => functional}/gc-non-blocking.sh (100%) rename tests/{ => functional}/gc-runtime.nix (100%) rename tests/{ => functional}/gc-runtime.sh (100%) rename tests/{ => functional}/gc.sh (100%) rename tests/{ => functional}/hash-check.nix (100%) rename tests/{ => functional}/hash.sh (100%) rename tests/{ => functional}/hermetic.nix (100%) rename tests/{ => functional}/import-derivation.nix (100%) rename tests/{ => functional}/import-derivation.sh (100%) rename tests/{ => functional}/impure-derivations.nix (100%) rename tests/{ => functional}/impure-derivations.sh (100%) rename tests/{ => functional}/init.sh (100%) rename tests/{ => functional}/install-darwin.sh (100%) rename tests/{ => functional}/lang-test-infra.sh (100%) rename tests/{ => functional}/lang.sh (98%) rename tests/{ => functional}/lang/binary-data (100%) rename tests/{ => functional}/lang/data (100%) rename tests/{ => functional}/lang/dir1/a.nix (100%) rename tests/{ => functional}/lang/dir2/a.nix (100%) rename tests/{ => functional}/lang/dir2/b.nix (100%) rename tests/{ => functional}/lang/dir3/a.nix (100%) rename tests/{ => functional}/lang/dir3/b.nix (100%) rename tests/{ => functional}/lang/dir3/c.nix (100%) rename tests/{ => functional}/lang/dir4/a.nix (100%) rename tests/{ => functional}/lang/dir4/c.nix (100%) rename tests/{ => functional}/lang/empty.exp (100%) rename tests/{ => functional}/lang/eval-fail-abort.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-abort.nix (100%) rename tests/{ => functional}/lang/eval-fail-antiquoted-path.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-assert.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-assert.nix (100%) rename tests/{ => functional}/lang/eval-fail-bad-antiquote-1.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-bad-antiquote-2.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-bad-antiquote-3.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-bad-string-interpolation-1.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-bad-string-interpolation-1.nix (100%) rename tests/{ => functional}/lang/eval-fail-bad-string-interpolation-2.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-bad-string-interpolation-2.nix (100%) rename tests/{ => functional}/lang/eval-fail-bad-string-interpolation-3.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-bad-string-interpolation-3.nix (100%) rename tests/{ => functional}/lang/eval-fail-blackhole.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-blackhole.nix (100%) rename tests/{ => functional}/lang/eval-fail-deepseq.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-deepseq.nix (100%) rename tests/{ => functional}/lang/eval-fail-dup-dynamic-attrs.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-dup-dynamic-attrs.nix (100%) rename tests/{ => functional}/lang/eval-fail-foldlStrict-strict-op-application.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-foldlStrict-strict-op-application.nix (100%) rename tests/{ => functional}/lang/eval-fail-fromTOML-timestamps.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-fromTOML-timestamps.nix (100%) rename tests/{ => functional}/lang/eval-fail-hashfile-missing.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-hashfile-missing.nix (100%) rename tests/{ => functional}/lang/eval-fail-list.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-list.nix (100%) rename tests/{ => functional}/lang/eval-fail-missing-arg.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-missing-arg.nix (100%) rename tests/{ => functional}/lang/eval-fail-nonexist-path.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-nonexist-path.nix (100%) rename tests/{ => functional}/lang/eval-fail-path-slash.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-path-slash.nix (100%) rename tests/{ => functional}/lang/eval-fail-recursion.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-recursion.nix (100%) rename tests/{ => functional}/lang/eval-fail-remove.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-remove.nix (100%) rename tests/{ => functional}/lang/eval-fail-scope-5.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-scope-5.nix (100%) rename tests/{ => functional}/lang/eval-fail-seq.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-seq.nix (100%) rename tests/{ => functional}/lang/eval-fail-set-override.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-set-override.nix (100%) rename tests/{ => functional}/lang/eval-fail-set.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-set.nix (100%) rename tests/{ => functional}/lang/eval-fail-substring.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-substring.nix (100%) rename tests/{ => functional}/lang/eval-fail-to-path.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-to-path.nix (100%) rename tests/{ => functional}/lang/eval-fail-toJSON.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-toJSON.nix (100%) rename tests/{ => functional}/lang/eval-fail-undeclared-arg.err.exp (100%) rename tests/{ => functional}/lang/eval-fail-undeclared-arg.nix (100%) rename tests/{ => functional}/lang/eval-okay-any-all.exp (100%) rename tests/{ => functional}/lang/eval-okay-any-all.nix (100%) rename tests/{ => functional}/lang/eval-okay-arithmetic.exp (100%) rename tests/{ => functional}/lang/eval-okay-arithmetic.nix (100%) rename tests/{ => functional}/lang/eval-okay-attrnames.exp (100%) rename tests/{ => functional}/lang/eval-okay-attrnames.nix (100%) rename tests/{ => functional}/lang/eval-okay-attrs.exp (100%) rename tests/{ => functional}/lang/eval-okay-attrs.nix (100%) rename tests/{ => functional}/lang/eval-okay-attrs2.exp (100%) rename tests/{ => functional}/lang/eval-okay-attrs2.nix (100%) rename tests/{ => functional}/lang/eval-okay-attrs3.exp (100%) rename tests/{ => functional}/lang/eval-okay-attrs3.nix (100%) rename tests/{ => functional}/lang/eval-okay-attrs4.exp (100%) rename tests/{ => functional}/lang/eval-okay-attrs4.nix (100%) rename tests/{ => functional}/lang/eval-okay-attrs5.exp (100%) rename tests/{ => functional}/lang/eval-okay-attrs5.nix (100%) rename tests/{ => functional}/lang/eval-okay-attrs6.exp (100%) rename tests/{ => functional}/lang/eval-okay-attrs6.nix (100%) rename tests/{ => functional}/lang/eval-okay-autoargs.exp (100%) rename tests/{ => functional}/lang/eval-okay-autoargs.flags (100%) rename tests/{ => functional}/lang/eval-okay-autoargs.nix (100%) rename tests/{ => functional}/lang/eval-okay-backslash-newline-1.exp (100%) rename tests/{ => functional}/lang/eval-okay-backslash-newline-1.nix (100%) rename tests/{ => functional}/lang/eval-okay-backslash-newline-2.exp (100%) rename tests/{ => functional}/lang/eval-okay-backslash-newline-2.nix (100%) rename tests/{ => functional}/lang/eval-okay-builtins-add.exp (100%) rename tests/{ => functional}/lang/eval-okay-builtins-add.nix (100%) rename tests/{ => functional}/lang/eval-okay-builtins.exp (100%) rename tests/{ => functional}/lang/eval-okay-builtins.nix (100%) rename tests/{ => functional}/lang/eval-okay-callable-attrs.exp (100%) rename tests/{ => functional}/lang/eval-okay-callable-attrs.nix (100%) rename tests/{ => functional}/lang/eval-okay-catattrs.exp (100%) rename tests/{ => functional}/lang/eval-okay-catattrs.nix (100%) rename tests/{ => functional}/lang/eval-okay-closure.exp (100%) rename tests/{ => functional}/lang/eval-okay-closure.exp.xml (100%) rename tests/{ => functional}/lang/eval-okay-closure.nix (100%) rename tests/{ => functional}/lang/eval-okay-comments.exp (100%) rename tests/{ => functional}/lang/eval-okay-comments.nix (100%) rename tests/{ => functional}/lang/eval-okay-concat.exp (100%) rename tests/{ => functional}/lang/eval-okay-concat.nix (100%) rename tests/{ => functional}/lang/eval-okay-concatmap.exp (100%) rename tests/{ => functional}/lang/eval-okay-concatmap.nix (100%) rename tests/{ => functional}/lang/eval-okay-concatstringssep.exp (100%) rename tests/{ => functional}/lang/eval-okay-concatstringssep.nix (100%) rename tests/{ => functional}/lang/eval-okay-context-introspection.exp (100%) rename tests/{ => functional}/lang/eval-okay-context-introspection.nix (100%) rename tests/{ => functional}/lang/eval-okay-context.exp (100%) rename tests/{ => functional}/lang/eval-okay-context.nix (100%) rename tests/{ => functional}/lang/eval-okay-curpos.exp (100%) rename tests/{ => functional}/lang/eval-okay-curpos.nix (100%) rename tests/{ => functional}/lang/eval-okay-deepseq.exp (100%) rename tests/{ => functional}/lang/eval-okay-deepseq.nix (100%) rename tests/{ => functional}/lang/eval-okay-delayed-with-inherit.exp (100%) rename tests/{ => functional}/lang/eval-okay-delayed-with-inherit.nix (100%) rename tests/{ => functional}/lang/eval-okay-delayed-with.exp (100%) rename tests/{ => functional}/lang/eval-okay-delayed-with.nix (100%) rename tests/{ => functional}/lang/eval-okay-dynamic-attrs-2.exp (100%) rename tests/{ => functional}/lang/eval-okay-dynamic-attrs-2.nix (100%) rename tests/{ => functional}/lang/eval-okay-dynamic-attrs-bare.exp (100%) rename tests/{ => functional}/lang/eval-okay-dynamic-attrs-bare.nix (100%) rename tests/{ => functional}/lang/eval-okay-dynamic-attrs.exp (100%) rename tests/{ => functional}/lang/eval-okay-dynamic-attrs.nix (100%) rename tests/{ => functional}/lang/eval-okay-elem.exp (100%) rename tests/{ => functional}/lang/eval-okay-elem.nix (100%) rename tests/{ => functional}/lang/eval-okay-empty-args.exp (100%) rename tests/{ => functional}/lang/eval-okay-empty-args.nix (100%) rename tests/{ => functional}/lang/eval-okay-eq-derivations.exp (100%) rename tests/{ => functional}/lang/eval-okay-eq-derivations.nix (100%) rename tests/{ => functional}/lang/eval-okay-eq.exp (100%) rename tests/{ => functional}/lang/eval-okay-eq.nix (100%) rename tests/{ => functional}/lang/eval-okay-filter.exp (100%) rename tests/{ => functional}/lang/eval-okay-filter.nix (100%) rename tests/{ => functional}/lang/eval-okay-flake-ref-to-string.exp (100%) rename tests/{ => functional}/lang/eval-okay-flake-ref-to-string.nix (100%) rename tests/{ => functional}/lang/eval-okay-flatten.exp (100%) rename tests/{ => functional}/lang/eval-okay-flatten.nix (100%) rename tests/{ => functional}/lang/eval-okay-float.exp (100%) rename tests/{ => functional}/lang/eval-okay-float.nix (100%) rename tests/{ => functional}/lang/eval-okay-floor-ceil.exp (100%) rename tests/{ => functional}/lang/eval-okay-floor-ceil.nix (100%) rename tests/{ => functional}/lang/eval-okay-foldlStrict-lazy-elements.exp (100%) rename tests/{ => functional}/lang/eval-okay-foldlStrict-lazy-elements.nix (100%) rename tests/{ => functional}/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp (100%) rename tests/{ => functional}/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix (100%) rename tests/{ => functional}/lang/eval-okay-foldlStrict.exp (100%) rename tests/{ => functional}/lang/eval-okay-foldlStrict.nix (100%) rename tests/{ => functional}/lang/eval-okay-fromTOML-timestamps.exp (100%) rename tests/{ => functional}/lang/eval-okay-fromTOML-timestamps.flags (100%) rename tests/{ => functional}/lang/eval-okay-fromTOML-timestamps.nix (100%) rename tests/{ => functional}/lang/eval-okay-fromTOML.exp (100%) rename tests/{ => functional}/lang/eval-okay-fromTOML.nix (100%) rename tests/{ => functional}/lang/eval-okay-fromjson-escapes.exp (100%) rename tests/{ => functional}/lang/eval-okay-fromjson-escapes.nix (100%) rename tests/{ => functional}/lang/eval-okay-fromjson.exp (100%) rename tests/{ => functional}/lang/eval-okay-fromjson.nix (100%) rename tests/{ => functional}/lang/eval-okay-functionargs.exp (100%) rename tests/{ => functional}/lang/eval-okay-functionargs.exp.xml (100%) rename tests/{ => functional}/lang/eval-okay-functionargs.nix (100%) rename tests/{ => functional}/lang/eval-okay-getattrpos-functionargs.exp (100%) rename tests/{ => functional}/lang/eval-okay-getattrpos-functionargs.nix (100%) rename tests/{ => functional}/lang/eval-okay-getattrpos-undefined.exp (100%) rename tests/{ => functional}/lang/eval-okay-getattrpos-undefined.nix (100%) rename tests/{ => functional}/lang/eval-okay-getattrpos.exp (100%) rename tests/{ => functional}/lang/eval-okay-getattrpos.nix (100%) rename tests/{ => functional}/lang/eval-okay-getenv.exp (100%) rename tests/{ => functional}/lang/eval-okay-getenv.nix (100%) rename tests/{ => functional}/lang/eval-okay-groupBy.exp (100%) rename tests/{ => functional}/lang/eval-okay-groupBy.nix (100%) rename tests/{ => functional}/lang/eval-okay-hash.exp (100%) rename tests/{ => functional}/lang/eval-okay-hashfile.exp (100%) rename tests/{ => functional}/lang/eval-okay-hashfile.nix (100%) rename tests/{ => functional}/lang/eval-okay-hashstring.exp (100%) rename tests/{ => functional}/lang/eval-okay-hashstring.nix (100%) rename tests/{ => functional}/lang/eval-okay-if.exp (100%) rename tests/{ => functional}/lang/eval-okay-if.nix (100%) rename tests/{ => functional}/lang/eval-okay-import.exp (100%) rename tests/{ => functional}/lang/eval-okay-import.nix (100%) rename tests/{ => functional}/lang/eval-okay-ind-string.exp (100%) rename tests/{ => functional}/lang/eval-okay-ind-string.nix (100%) rename tests/{ => functional}/lang/eval-okay-intersectAttrs.exp (100%) rename tests/{ => functional}/lang/eval-okay-intersectAttrs.nix (100%) rename tests/{ => functional}/lang/eval-okay-let.exp (100%) rename tests/{ => functional}/lang/eval-okay-let.nix (100%) rename tests/{ => functional}/lang/eval-okay-list.exp (100%) rename tests/{ => functional}/lang/eval-okay-list.nix (100%) rename tests/{ => functional}/lang/eval-okay-listtoattrs.exp (100%) rename tests/{ => functional}/lang/eval-okay-listtoattrs.nix (100%) rename tests/{ => functional}/lang/eval-okay-logic.exp (100%) rename tests/{ => functional}/lang/eval-okay-logic.nix (100%) rename tests/{ => functional}/lang/eval-okay-map.exp (100%) rename tests/{ => functional}/lang/eval-okay-map.nix (100%) rename tests/{ => functional}/lang/eval-okay-mapattrs.exp (100%) rename tests/{ => functional}/lang/eval-okay-mapattrs.nix (100%) rename tests/{ => functional}/lang/eval-okay-merge-dynamic-attrs.exp (100%) rename tests/{ => functional}/lang/eval-okay-merge-dynamic-attrs.nix (100%) rename tests/{ => functional}/lang/eval-okay-nested-with.exp (100%) rename tests/{ => functional}/lang/eval-okay-nested-with.nix (100%) rename tests/{ => functional}/lang/eval-okay-new-let.exp (100%) rename tests/{ => functional}/lang/eval-okay-new-let.nix (100%) rename tests/{ => functional}/lang/eval-okay-null-dynamic-attrs.exp (100%) rename tests/{ => functional}/lang/eval-okay-null-dynamic-attrs.nix (100%) rename tests/{ => functional}/lang/eval-okay-overrides.exp (100%) rename tests/{ => functional}/lang/eval-okay-overrides.nix (100%) rename tests/{ => functional}/lang/eval-okay-parse-flake-ref.exp (100%) rename tests/{ => functional}/lang/eval-okay-parse-flake-ref.nix (100%) rename tests/{ => functional}/lang/eval-okay-partition.exp (100%) rename tests/{ => functional}/lang/eval-okay-partition.nix (100%) rename tests/{ => functional}/lang/eval-okay-path-string-interpolation.exp (100%) rename tests/{ => functional}/lang/eval-okay-path-string-interpolation.nix (100%) rename tests/{ => functional}/lang/eval-okay-path.exp (100%) rename tests/{ => functional}/lang/eval-okay-path.nix (100%) rename tests/{ => functional}/lang/eval-okay-pathexists.exp (100%) rename tests/{ => functional}/lang/eval-okay-pathexists.nix (100%) rename tests/{ => functional}/lang/eval-okay-patterns.exp (100%) rename tests/{ => functional}/lang/eval-okay-patterns.nix (100%) rename tests/{ => functional}/lang/eval-okay-print.err.exp (100%) rename tests/{ => functional}/lang/eval-okay-print.exp (100%) rename tests/{ => functional}/lang/eval-okay-print.nix (100%) rename tests/{ => functional}/lang/eval-okay-readDir.exp (100%) rename tests/{ => functional}/lang/eval-okay-readDir.nix (100%) rename tests/{ => functional}/lang/eval-okay-readFileType.exp (100%) rename tests/{ => functional}/lang/eval-okay-readFileType.nix (100%) rename tests/{ => functional}/lang/eval-okay-readfile.exp (100%) rename tests/{ => functional}/lang/eval-okay-readfile.nix (100%) rename tests/{ => functional}/lang/eval-okay-redefine-builtin.exp (100%) rename tests/{ => functional}/lang/eval-okay-redefine-builtin.nix (100%) rename tests/{ => functional}/lang/eval-okay-regex-match.exp (100%) rename tests/{ => functional}/lang/eval-okay-regex-match.nix (100%) rename tests/{ => functional}/lang/eval-okay-regex-split.exp (100%) rename tests/{ => functional}/lang/eval-okay-regex-split.nix (100%) rename tests/{ => functional}/lang/eval-okay-regression-20220122.exp (100%) rename tests/{ => functional}/lang/eval-okay-regression-20220122.nix (100%) rename tests/{ => functional}/lang/eval-okay-regression-20220125.exp (100%) rename tests/{ => functional}/lang/eval-okay-regression-20220125.nix (100%) rename tests/{ => functional}/lang/eval-okay-remove.exp (100%) rename tests/{ => functional}/lang/eval-okay-remove.nix (100%) rename tests/{ => functional}/lang/eval-okay-replacestrings.exp (100%) rename tests/{ => functional}/lang/eval-okay-replacestrings.nix (100%) rename tests/{ => functional}/lang/eval-okay-scope-1.exp (100%) rename tests/{ => functional}/lang/eval-okay-scope-1.nix (100%) rename tests/{ => functional}/lang/eval-okay-scope-2.exp (100%) rename tests/{ => functional}/lang/eval-okay-scope-2.nix (100%) rename tests/{ => functional}/lang/eval-okay-scope-3.exp (100%) rename tests/{ => functional}/lang/eval-okay-scope-3.nix (100%) rename tests/{ => functional}/lang/eval-okay-scope-4.exp (100%) rename tests/{ => functional}/lang/eval-okay-scope-4.nix (100%) rename tests/{ => functional}/lang/eval-okay-scope-6.exp (100%) rename tests/{ => functional}/lang/eval-okay-scope-6.nix (100%) rename tests/{ => functional}/lang/eval-okay-scope-7.exp (100%) rename tests/{ => functional}/lang/eval-okay-scope-7.nix (100%) rename tests/{ => functional}/lang/eval-okay-search-path.exp (100%) rename tests/{ => functional}/lang/eval-okay-search-path.flags (100%) rename tests/{ => functional}/lang/eval-okay-search-path.nix (100%) rename tests/{ => functional}/lang/eval-okay-seq.exp (100%) rename tests/{ => functional}/lang/eval-okay-seq.nix (100%) rename tests/{ => functional}/lang/eval-okay-sort.exp (100%) rename tests/{ => functional}/lang/eval-okay-sort.nix (100%) rename tests/{ => functional}/lang/eval-okay-splitversion.exp (100%) rename tests/{ => functional}/lang/eval-okay-splitversion.nix (100%) rename tests/{ => functional}/lang/eval-okay-string.exp (100%) rename tests/{ => functional}/lang/eval-okay-string.nix (100%) rename tests/{ => functional}/lang/eval-okay-strings-as-attrs-names.exp (100%) rename tests/{ => functional}/lang/eval-okay-strings-as-attrs-names.nix (100%) rename tests/{ => functional}/lang/eval-okay-substring.exp (100%) rename tests/{ => functional}/lang/eval-okay-substring.nix (100%) rename tests/{ => functional}/lang/eval-okay-tail-call-1.exp-disabled (100%) rename tests/{ => functional}/lang/eval-okay-tail-call-1.nix (100%) rename tests/{ => functional}/lang/eval-okay-tojson.exp (100%) rename tests/{ => functional}/lang/eval-okay-tojson.nix (100%) rename tests/{ => functional}/lang/eval-okay-toxml.exp (100%) rename tests/{ => functional}/lang/eval-okay-toxml.nix (100%) rename tests/{ => functional}/lang/eval-okay-toxml2.exp (100%) rename tests/{ => functional}/lang/eval-okay-toxml2.nix (100%) rename tests/{ => functional}/lang/eval-okay-tryeval.exp (100%) rename tests/{ => functional}/lang/eval-okay-tryeval.nix (100%) rename tests/{ => functional}/lang/eval-okay-types.exp (100%) rename tests/{ => functional}/lang/eval-okay-types.nix (100%) rename tests/{ => functional}/lang/eval-okay-versions.exp (100%) rename tests/{ => functional}/lang/eval-okay-versions.nix (100%) rename tests/{ => functional}/lang/eval-okay-with.exp (100%) rename tests/{ => functional}/lang/eval-okay-with.nix (100%) rename tests/{ => functional}/lang/eval-okay-xml.exp.xml (100%) rename tests/{ => functional}/lang/eval-okay-xml.nix (100%) rename tests/{ => functional}/lang/eval-okay-zipAttrsWith.exp (100%) rename tests/{ => functional}/lang/eval-okay-zipAttrsWith.nix (100%) rename tests/{ => functional}/lang/framework.sh (100%) rename tests/{ => functional}/lang/imported.nix (100%) rename tests/{ => functional}/lang/imported2.nix (100%) rename tests/{ => functional}/lang/lib.nix (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-1.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-1.nix (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-2.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-2.nix (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-3.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-3.nix (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-4.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-4.nix (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-6.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-7.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-dup-attrs-7.nix (100%) rename tests/{ => functional}/lang/parse-fail-dup-formals.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-dup-formals.nix (100%) rename tests/{ => functional}/lang/parse-fail-eof-in-string.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-eof-in-string.nix (100%) rename tests/{ => functional}/lang/parse-fail-mixed-nested-attrs1.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-mixed-nested-attrs1.nix (100%) rename tests/{ => functional}/lang/parse-fail-mixed-nested-attrs2.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-mixed-nested-attrs2.nix (100%) rename tests/{ => functional}/lang/parse-fail-patterns-1.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-patterns-1.nix (100%) rename tests/{ => functional}/lang/parse-fail-regression-20060610.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-regression-20060610.nix (100%) rename tests/{ => functional}/lang/parse-fail-undef-var-2.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-undef-var-2.nix (100%) rename tests/{ => functional}/lang/parse-fail-undef-var.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-undef-var.nix (100%) rename tests/{ => functional}/lang/parse-fail-utf8.err.exp (100%) rename tests/{ => functional}/lang/parse-fail-utf8.nix (100%) rename tests/{ => functional}/lang/parse-okay-1.exp (100%) rename tests/{ => functional}/lang/parse-okay-1.nix (100%) rename tests/{ => functional}/lang/parse-okay-crlf.exp (100%) rename tests/{ => functional}/lang/parse-okay-crlf.nix (100%) rename tests/{ => functional}/lang/parse-okay-dup-attrs-5.exp (100%) rename tests/{ => functional}/lang/parse-okay-dup-attrs-5.nix (100%) rename tests/{ => functional}/lang/parse-okay-dup-attrs-6.exp (100%) rename tests/{ => functional}/lang/parse-okay-dup-attrs-6.nix (100%) rename tests/{ => functional}/lang/parse-okay-mixed-nested-attrs-1.exp (100%) rename tests/{ => functional}/lang/parse-okay-mixed-nested-attrs-1.nix (100%) rename tests/{ => functional}/lang/parse-okay-mixed-nested-attrs-2.exp (100%) rename tests/{ => functional}/lang/parse-okay-mixed-nested-attrs-2.nix (100%) rename tests/{ => functional}/lang/parse-okay-mixed-nested-attrs-3.exp (100%) rename tests/{ => functional}/lang/parse-okay-mixed-nested-attrs-3.nix (100%) rename tests/{ => functional}/lang/parse-okay-regression-20041027.exp (100%) rename tests/{ => functional}/lang/parse-okay-regression-20041027.nix (100%) rename tests/{ => functional}/lang/parse-okay-regression-751.exp (100%) rename tests/{ => functional}/lang/parse-okay-regression-751.nix (100%) rename tests/{ => functional}/lang/parse-okay-subversion.exp (100%) rename tests/{ => functional}/lang/parse-okay-subversion.nix (100%) rename tests/{ => functional}/lang/parse-okay-url.exp (100%) rename tests/{ => functional}/lang/parse-okay-url.nix (100%) rename tests/{ => functional}/lang/readDir/bar (100%) rename tests/{ => functional}/lang/readDir/foo/git-hates-directories (100%) rename tests/{ => functional}/lang/readDir/ldir (100%) rename tests/{ => functional}/lang/readDir/linked (100%) rename tests/{ => functional}/legacy-ssh-store.sh (100%) rename tests/{ => functional}/linux-sandbox-cert-test.nix (100%) rename tests/{ => functional}/linux-sandbox.sh (100%) rename tests/{ => functional}/local-store.sh (100%) rename tests/{ => functional}/local.mk (90%) rename tests/{ => functional}/logging.sh (100%) rename tests/{ => functional}/misc.sh (100%) rename tests/{ => functional}/multiple-outputs.nix (100%) rename tests/{ => functional}/multiple-outputs.sh (100%) rename tests/{ => functional}/nar-access.nix (100%) rename tests/{ => functional}/nar-access.sh (100%) rename tests/{ => functional}/nested-sandboxing.sh (75%) rename tests/{ => functional}/nested-sandboxing/command.sh (100%) rename tests/{ => functional}/nested-sandboxing/runner.nix (100%) rename tests/{ => functional}/nix-build-examples.nix (100%) rename tests/{ => functional}/nix-build.sh (100%) rename tests/{ => functional}/nix-channel.sh (100%) rename tests/{ => functional}/nix-collect-garbage-d.sh (100%) rename tests/{ => functional}/nix-copy-ssh-ng.sh (100%) rename tests/{ => functional}/nix-copy-ssh.sh (100%) rename tests/{ => functional}/nix-daemon-untrusting.sh (100%) rename tests/{ => functional}/nix-profile.sh (100%) rename tests/{ => functional}/nix-shell.sh (100%) rename tests/{ => functional}/nix_path.sh (100%) rename tests/{ => functional}/optimise-store.sh (100%) rename tests/{ => functional}/output-normalization.sh (100%) rename tests/{ => functional}/parallel.builder.sh (100%) rename tests/{ => functional}/parallel.nix (100%) rename tests/{ => functional}/parallel.sh (100%) rename tests/{ => functional}/pass-as-file.sh (100%) rename tests/{ => functional}/path-from-hash-part.sh (100%) rename tests/{ => functional}/path.nix (100%) rename tests/{ => functional}/placeholders.sh (100%) rename tests/{ => functional}/plugins.sh (100%) rename tests/{ => functional}/plugins/local.mk (100%) rename tests/{ => functional}/plugins/plugintest.cc (100%) rename tests/{ => functional}/post-hook.sh (100%) rename tests/{ => functional}/pure-eval.nix (100%) rename tests/{ => functional}/pure-eval.sh (100%) rename tests/{ => functional}/push-to-store-old.sh (100%) rename tests/{ => functional}/push-to-store.sh (100%) rename tests/{ => functional}/read-only-store.sh (100%) rename tests/{ => functional}/readfile-context.nix (100%) rename tests/{ => functional}/readfile-context.sh (100%) rename tests/{ => functional}/recursive.nix (100%) rename tests/{ => functional}/recursive.sh (100%) rename tests/{ => functional}/referrers.sh (100%) rename tests/{ => functional}/remote-store.sh (100%) rename tests/{ => functional}/repair.sh (100%) rename tests/{ => functional}/repl.sh (100%) rename tests/{ => functional}/restricted.nix (100%) rename tests/{ => functional}/restricted.sh (95%) rename tests/{ => functional}/search.nix (100%) rename tests/{ => functional}/search.sh (100%) rename tests/{ => functional}/secure-drv-outputs.nix (100%) rename tests/{ => functional}/secure-drv-outputs.sh (100%) rename tests/{ => functional}/selfref-gc.sh (100%) rename tests/{ => functional}/shell-hello.nix (100%) rename tests/{ => functional}/shell.nix (100%) rename tests/{ => functional}/shell.sh (100%) rename tests/{ => functional}/shell.shebang.rb (100%) rename tests/{ => functional}/shell.shebang.sh (100%) rename tests/{ => functional}/signing.sh (100%) rename tests/{ => functional}/simple-failing.nix (100%) rename tests/{ => functional}/simple.builder.sh (100%) rename tests/{ => functional}/simple.nix (100%) rename tests/{ => functional}/simple.sh (100%) rename tests/{ => functional}/ssh-relay.sh (100%) rename tests/{ => functional}/store-ping.sh (100%) rename tests/{ => functional}/structured-attrs-shell.nix (100%) rename tests/{ => functional}/structured-attrs.nix (100%) rename tests/{ => functional}/structured-attrs.sh (100%) rename tests/{ => functional}/substitute-with-invalid-ca.sh (100%) rename tests/{ => functional}/suggestions.sh (100%) rename tests/{ => functional}/supplementary-groups.sh (100%) rename tests/{ => functional}/tarball.sh (100%) rename tests/{ => functional}/test-infra.sh (100%) rename tests/{ => functional}/test-libstoreconsumer.sh (100%) rename tests/{ => functional}/test-libstoreconsumer/README.md (100%) rename tests/{ => functional}/test-libstoreconsumer/local.mk (100%) rename tests/{ => functional}/test-libstoreconsumer/main.cc (100%) rename tests/{ => functional}/timeout.nix (100%) rename tests/{ => functional}/timeout.sh (100%) rename tests/{ => functional}/toString-path.sh (100%) rename tests/{ => functional}/undefined-variable.nix (100%) rename tests/{ => functional}/user-envs-migration.sh (100%) rename tests/{ => functional}/user-envs.builder.sh (100%) rename tests/{ => functional}/user-envs.nix (100%) rename tests/{ => functional}/user-envs.sh (100%) rename tests/{ => functional}/why-depends.sh (100%) rename tests/{ => functional}/zstd.sh (100%) diff --git a/.github/labeler.yml b/.github/labeler.yml index 12120bdb3..7544f07a6 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -20,4 +20,4 @@ # Unit tests - src/*/tests/**/* # Functional and integration tests - - tests/**/* + - tests/functional/**/* diff --git a/.gitignore b/.gitignore index bf4f33564..2d3314015 100644 --- a/.gitignore +++ b/.gitignore @@ -79,24 +79,24 @@ perl/Makefile.config /src/build-remote/build-remote -# /tests/ -/tests/test-tmp -/tests/common/vars-and-functions.sh -/tests/result* -/tests/restricted-innocent -/tests/shell -/tests/shell.drv -/tests/config.nix -/tests/ca/config.nix -/tests/dyn-drv/config.nix -/tests/repl-result-out -/tests/test-libstoreconsumer/test-libstoreconsumer +# /tests/functional/ +/tests/functional/test-tmp +/tests/functional/common/vars-and-functions.sh +/tests/functional/result* +/tests/functional/restricted-innocent +/tests/functional/shell +/tests/functional/shell.drv +/tests/functional/config.nix +/tests/functional/ca/config.nix +/tests/functional/dyn-drv/config.nix +/tests/functional/repl-result-out +/tests/functional/test-libstoreconsumer/test-libstoreconsumer -# /tests/lang/ -/tests/lang/*.out -/tests/lang/*.out.xml -/tests/lang/*.err -/tests/lang/*.ast +# /tests/functional/lang/ +/tests/functional/lang/*.out +/tests/functional/lang/*.out.xml +/tests/functional/lang/*.err +/tests/functional/lang/*.ast /perl/lib/Nix/Config.pm /perl/lib/Nix/Store.cc diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4a9cff3b2..73210b303 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -57,7 +57,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). - [ ] Fixes an [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) issue - [ ] Tests, as appropriate: - - Functional tests – [`tests/**.sh`](./tests) + - Functional tests – [`tests/functional/**.sh`](./tests/functional) - Unit tests – [`src/*/tests`](./src/) - Integration tests – [`tests/nixos/*`](./tests/nixos) - [ ] User documentation in the [manual](..doc/manual/src) diff --git a/Makefile b/Makefile index 2eabeb077..6658e3490 100644 --- a/Makefile +++ b/Makefile @@ -27,11 +27,11 @@ makefiles += \ src/libutil/tests/local.mk \ src/libstore/tests/local.mk \ src/libexpr/tests/local.mk \ - tests/local.mk \ - tests/ca/local.mk \ - tests/dyn-drv/local.mk \ - tests/test-libstoreconsumer/local.mk \ - tests/plugins/local.mk + tests/functional/local.mk \ + tests/functional/ca/local.mk \ + tests/functional/dyn-drv/local.mk \ + tests/functional/test-libstoreconsumer/local.mk \ + tests/functional/plugins/local.mk else makefiles += \ mk/disable-tests.mk diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index 04bb4343d..1a6388e40 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -82,7 +82,7 @@ The characterization tests will mark themselves "skipped" since they regenerated ## Functional tests -The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. +The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. Each test is a bash script. ### Running the whole test suite @@ -91,8 +91,8 @@ The whole test suite can be run with: ```shell-session $ make install && make installcheck -ran test tests/foo.sh... [PASS] -ran test tests/bar.sh... [PASS] +ran test tests/functional/foo.sh... [PASS] +ran test tests/functional/bar.sh... [PASS] ... ``` @@ -100,14 +100,14 @@ ran test tests/bar.sh... [PASS] Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite. Each test group is in a subdirectory of `tests`. -For example, `tests/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs. +For example, `tests/functional/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs. That test group can be run like this: ```shell-session $ make ca.test-group -j50 -ran test tests/ca/nix-run.sh... [PASS] -ran test tests/ca/import-derivation.sh... [PASS] +ran test tests/functional/ca/nix-run.sh... [PASS] +ran test tests/functional/ca/import-derivation.sh... [PASS] ... ``` @@ -126,21 +126,21 @@ install-tests-groups += $(test-group-name) Individual tests can be run with `make`: ```shell-session -$ make tests/${testName}.sh.test -ran test tests/${testName}.sh... [PASS] +$ make tests/functional/${testName}.sh.test +ran test tests/functional/${testName}.sh... [PASS] ``` or without `make`: ```shell-session -$ ./mk/run-test.sh tests/${testName}.sh -ran test tests/${testName}.sh... [PASS] +$ ./mk/run-test.sh tests/functional/${testName}.sh +ran test tests/functional/${testName}.sh... [PASS] ``` To see the complete output, one can also run: ```shell-session -$ ./mk/debug-test.sh tests/${testName}.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh + foo output from foo + bar @@ -175,7 +175,7 @@ edit it like so: Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: ```shell-session -$ ./mk/debug-test.sh tests/${testName}.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh ... + gdb blash blub GNU gdb (GDB) 12.1 @@ -206,7 +206,7 @@ It is frequently useful to regenerate the expected output. To do that, rerun the failed test(s) with `_NIX_TEST_ACCEPT=1`. For example: ```bash -_NIX_TEST_ACCEPT=1 make tests/lang.sh.test +_NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test ``` This convention is shared with the [characterization unit tests](#characterization-testing-1) too. diff --git a/flake.nix b/flake.nix index 4de155362..c331b2651 100644 --- a/flake.nix +++ b/flake.nix @@ -61,36 +61,28 @@ nixSrc = fileset.toSource { root = ./.; - fileset = fileset.intersect baseFiles ( - fileset.difference - (fileset.unions [ - ./.version - ./boehmgc-coroutine-sp-fallback.diff - ./bootstrap.sh - ./configure.ac - ./doc - ./local.mk - ./m4 - ./Makefile - ./Makefile.config.in - ./misc - ./mk - ./precompiled-headers.h - ./src - ./tests - ./unit-test-data - ./COPYING - ./scripts/local.mk - (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) - # TODO: do we really need README.md? It doesn't seem used in the build. - ./README.md - ]) - (fileset.unions [ - # Removed file sets - ./tests/nixos - ./tests/installer - ]) - ); + fileset = fileset.intersect baseFiles (fileset.unions [ + ./.version + ./boehmgc-coroutine-sp-fallback.diff + ./bootstrap.sh + ./configure.ac + ./doc + ./local.mk + ./m4 + ./Makefile + ./Makefile.config.in + ./misc + ./mk + ./precompiled-headers.h + ./src + ./tests/functional + ./unit-test-data + ./COPYING + ./scripts/local.mk + (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) + # TODO: do we really need README.md? It doesn't seem used in the build. + ./README.md + ]); }; # Memoize nixpkgs for different platforms for efficiency. diff --git a/mk/common-test.sh b/mk/common-test.sh index 0a2e4c1c2..7ab25febf 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -1,11 +1,15 @@ +test_dir=tests/functional + +test=$(echo -n "$test" | sed -e "s|^$test_dir/||") + TESTS_ENVIRONMENT=("TEST_NAME=${test%.*}" 'NIX_REMOTE=') : ${BASH:=/usr/bin/env bash} init_test () { - cd tests && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null + cd "$test_dir" && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null } run_test_proper () { - cd $(dirname $test) && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test) + cd "$test_dir/$(dirname $test)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test) } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 26c87391c..516cbef83 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -776,7 +776,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } }; - /* Synchronisation point for testing, see tests/gc-concurrent.sh. */ + /* Synchronisation point for testing, see tests/functional/gc-concurrent.sh. */ if (auto p = getEnv("_NIX_TEST_GC_SYNC")) readFile(*p); diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 4712c0c91..5c5a30dc2 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -30,7 +30,7 @@ extern std::regex refRegex; /// Instead of defining what a good Git Ref is, we define what a bad Git Ref is /// This is because of the definition of a ref in refs.c in https://github.com/git/git -/// See tests/fetchGitRefs.sh for the full definition +/// See tests/functional/fetchGitRefs.sh for the full definition const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; extern std::regex badGitRefRegex; diff --git a/tests/add.sh b/tests/functional/add.sh similarity index 100% rename from tests/add.sh rename to tests/functional/add.sh diff --git a/tests/bad.tar.xz b/tests/functional/bad.tar.xz similarity index 100% rename from tests/bad.tar.xz rename to tests/functional/bad.tar.xz diff --git a/tests/bash-profile.sh b/tests/functional/bash-profile.sh similarity index 78% rename from tests/bash-profile.sh rename to tests/functional/bash-profile.sh index e2e0d1090..3faeaaba1 100644 --- a/tests/bash-profile.sh +++ b/tests/functional/bash-profile.sh @@ -1,6 +1,6 @@ source common.sh -sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh +sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh user=$(whoami) rm -rf $TEST_HOME $TEST_ROOT/profile-var diff --git a/tests/big-derivation-attr.nix b/tests/functional/big-derivation-attr.nix similarity index 100% rename from tests/big-derivation-attr.nix rename to tests/functional/big-derivation-attr.nix diff --git a/tests/binary-cache-build-remote.sh b/tests/functional/binary-cache-build-remote.sh similarity index 100% rename from tests/binary-cache-build-remote.sh rename to tests/functional/binary-cache-build-remote.sh diff --git a/tests/binary-cache.sh b/tests/functional/binary-cache.sh similarity index 100% rename from tests/binary-cache.sh rename to tests/functional/binary-cache.sh diff --git a/tests/brotli.sh b/tests/functional/brotli.sh similarity index 100% rename from tests/brotli.sh rename to tests/functional/brotli.sh diff --git a/tests/build-delete.sh b/tests/functional/build-delete.sh similarity index 100% rename from tests/build-delete.sh rename to tests/functional/build-delete.sh diff --git a/tests/build-dry.sh b/tests/functional/build-dry.sh similarity index 100% rename from tests/build-dry.sh rename to tests/functional/build-dry.sh diff --git a/tests/build-hook-ca-fixed.nix b/tests/functional/build-hook-ca-fixed.nix similarity index 100% rename from tests/build-hook-ca-fixed.nix rename to tests/functional/build-hook-ca-fixed.nix diff --git a/tests/build-hook-ca-floating.nix b/tests/functional/build-hook-ca-floating.nix similarity index 100% rename from tests/build-hook-ca-floating.nix rename to tests/functional/build-hook-ca-floating.nix diff --git a/tests/build-hook.nix b/tests/functional/build-hook.nix similarity index 100% rename from tests/build-hook.nix rename to tests/functional/build-hook.nix diff --git a/tests/build-remote-content-addressed-fixed.sh b/tests/functional/build-remote-content-addressed-fixed.sh similarity index 100% rename from tests/build-remote-content-addressed-fixed.sh rename to tests/functional/build-remote-content-addressed-fixed.sh diff --git a/tests/build-remote-content-addressed-floating.sh b/tests/functional/build-remote-content-addressed-floating.sh similarity index 100% rename from tests/build-remote-content-addressed-floating.sh rename to tests/functional/build-remote-content-addressed-floating.sh diff --git a/tests/build-remote-input-addressed.sh b/tests/functional/build-remote-input-addressed.sh similarity index 100% rename from tests/build-remote-input-addressed.sh rename to tests/functional/build-remote-input-addressed.sh diff --git a/tests/build-remote-trustless-after.sh b/tests/functional/build-remote-trustless-after.sh similarity index 100% rename from tests/build-remote-trustless-after.sh rename to tests/functional/build-remote-trustless-after.sh diff --git a/tests/build-remote-trustless-should-fail-0.sh b/tests/functional/build-remote-trustless-should-fail-0.sh similarity index 100% rename from tests/build-remote-trustless-should-fail-0.sh rename to tests/functional/build-remote-trustless-should-fail-0.sh diff --git a/tests/build-remote-trustless-should-pass-0.sh b/tests/functional/build-remote-trustless-should-pass-0.sh similarity index 100% rename from tests/build-remote-trustless-should-pass-0.sh rename to tests/functional/build-remote-trustless-should-pass-0.sh diff --git a/tests/build-remote-trustless-should-pass-1.sh b/tests/functional/build-remote-trustless-should-pass-1.sh similarity index 100% rename from tests/build-remote-trustless-should-pass-1.sh rename to tests/functional/build-remote-trustless-should-pass-1.sh diff --git a/tests/build-remote-trustless-should-pass-2.sh b/tests/functional/build-remote-trustless-should-pass-2.sh similarity index 100% rename from tests/build-remote-trustless-should-pass-2.sh rename to tests/functional/build-remote-trustless-should-pass-2.sh diff --git a/tests/build-remote-trustless-should-pass-3.sh b/tests/functional/build-remote-trustless-should-pass-3.sh similarity index 100% rename from tests/build-remote-trustless-should-pass-3.sh rename to tests/functional/build-remote-trustless-should-pass-3.sh diff --git a/tests/build-remote-trustless.sh b/tests/functional/build-remote-trustless.sh similarity index 82% rename from tests/build-remote-trustless.sh rename to tests/functional/build-remote-trustless.sh index 9df44e0c5..81e5253bf 100644 --- a/tests/build-remote-trustless.sh +++ b/tests/functional/build-remote-trustless.sh @@ -6,7 +6,7 @@ unset NIX_STATE_DIR remoteDir=$TEST_ROOT/remote -# Note: ssh{-ng}://localhost bypasses ssh. See tests/build-remote.sh for +# Note: ssh{-ng}://localhost bypasses ssh. See tests/functional/build-remote.sh for # more details. nix-build $file -o $TEST_ROOT/result --max-jobs 0 \ --arg busybox $busybox \ diff --git a/tests/build-remote.sh b/tests/functional/build-remote.sh similarity index 100% rename from tests/build-remote.sh rename to tests/functional/build-remote.sh diff --git a/tests/build.sh b/tests/functional/build.sh similarity index 100% rename from tests/build.sh rename to tests/functional/build.sh diff --git a/tests/ca-shell.nix b/tests/functional/ca-shell.nix similarity index 100% rename from tests/ca-shell.nix rename to tests/functional/ca-shell.nix diff --git a/tests/ca/build-cache.sh b/tests/functional/ca/build-cache.sh similarity index 100% rename from tests/ca/build-cache.sh rename to tests/functional/ca/build-cache.sh diff --git a/tests/ca/build-dry.sh b/tests/functional/ca/build-dry.sh similarity index 100% rename from tests/ca/build-dry.sh rename to tests/functional/ca/build-dry.sh diff --git a/tests/ca/build-with-garbage-path.sh b/tests/functional/ca/build-with-garbage-path.sh similarity index 100% rename from tests/ca/build-with-garbage-path.sh rename to tests/functional/ca/build-with-garbage-path.sh diff --git a/tests/ca/build.sh b/tests/functional/ca/build.sh similarity index 100% rename from tests/ca/build.sh rename to tests/functional/ca/build.sh diff --git a/tests/ca/common.sh b/tests/functional/ca/common.sh similarity index 100% rename from tests/ca/common.sh rename to tests/functional/ca/common.sh diff --git a/tests/ca/concurrent-builds.sh b/tests/functional/ca/concurrent-builds.sh similarity index 100% rename from tests/ca/concurrent-builds.sh rename to tests/functional/ca/concurrent-builds.sh diff --git a/tests/ca/config.nix.in b/tests/functional/ca/config.nix.in similarity index 100% rename from tests/ca/config.nix.in rename to tests/functional/ca/config.nix.in diff --git a/tests/ca/content-addressed.nix b/tests/functional/ca/content-addressed.nix similarity index 100% rename from tests/ca/content-addressed.nix rename to tests/functional/ca/content-addressed.nix diff --git a/tests/ca/derivation-json.sh b/tests/functional/ca/derivation-json.sh similarity index 100% rename from tests/ca/derivation-json.sh rename to tests/functional/ca/derivation-json.sh diff --git a/tests/ca/duplicate-realisation-in-closure.sh b/tests/functional/ca/duplicate-realisation-in-closure.sh similarity index 100% rename from tests/ca/duplicate-realisation-in-closure.sh rename to tests/functional/ca/duplicate-realisation-in-closure.sh diff --git a/tests/ca/flake.nix b/tests/functional/ca/flake.nix similarity index 100% rename from tests/ca/flake.nix rename to tests/functional/ca/flake.nix diff --git a/tests/ca/gc.sh b/tests/functional/ca/gc.sh similarity index 100% rename from tests/ca/gc.sh rename to tests/functional/ca/gc.sh diff --git a/tests/ca/import-derivation.sh b/tests/functional/ca/import-derivation.sh similarity index 100% rename from tests/ca/import-derivation.sh rename to tests/functional/ca/import-derivation.sh diff --git a/tests/ca/local.mk b/tests/functional/ca/local.mk similarity index 94% rename from tests/ca/local.mk rename to tests/functional/ca/local.mk index 0852e592e..fd87b8d1f 100644 --- a/tests/ca/local.mk +++ b/tests/functional/ca/local.mk @@ -25,4 +25,4 @@ clean-files += \ $(d)/config.nix test-deps += \ - tests/ca/config.nix + tests/functional/ca/config.nix diff --git a/tests/ca/new-build-cmd.sh b/tests/functional/ca/new-build-cmd.sh similarity index 100% rename from tests/ca/new-build-cmd.sh rename to tests/functional/ca/new-build-cmd.sh diff --git a/tests/ca/nix-copy.sh b/tests/functional/ca/nix-copy.sh similarity index 100% rename from tests/ca/nix-copy.sh rename to tests/functional/ca/nix-copy.sh diff --git a/tests/ca/nix-run.sh b/tests/functional/ca/nix-run.sh similarity index 100% rename from tests/ca/nix-run.sh rename to tests/functional/ca/nix-run.sh diff --git a/tests/ca/nix-shell.sh b/tests/functional/ca/nix-shell.sh similarity index 100% rename from tests/ca/nix-shell.sh rename to tests/functional/ca/nix-shell.sh diff --git a/tests/ca/nondeterministic.nix b/tests/functional/ca/nondeterministic.nix similarity index 100% rename from tests/ca/nondeterministic.nix rename to tests/functional/ca/nondeterministic.nix diff --git a/tests/ca/post-hook.sh b/tests/functional/ca/post-hook.sh similarity index 100% rename from tests/ca/post-hook.sh rename to tests/functional/ca/post-hook.sh diff --git a/tests/ca/racy.nix b/tests/functional/ca/racy.nix similarity index 100% rename from tests/ca/racy.nix rename to tests/functional/ca/racy.nix diff --git a/tests/ca/recursive.sh b/tests/functional/ca/recursive.sh similarity index 100% rename from tests/ca/recursive.sh rename to tests/functional/ca/recursive.sh diff --git a/tests/ca/repl.sh b/tests/functional/ca/repl.sh similarity index 100% rename from tests/ca/repl.sh rename to tests/functional/ca/repl.sh diff --git a/tests/ca/selfref-gc.sh b/tests/functional/ca/selfref-gc.sh similarity index 100% rename from tests/ca/selfref-gc.sh rename to tests/functional/ca/selfref-gc.sh diff --git a/tests/ca/signatures.sh b/tests/functional/ca/signatures.sh similarity index 100% rename from tests/ca/signatures.sh rename to tests/functional/ca/signatures.sh diff --git a/tests/ca/substitute.sh b/tests/functional/ca/substitute.sh similarity index 100% rename from tests/ca/substitute.sh rename to tests/functional/ca/substitute.sh diff --git a/tests/ca/why-depends.sh b/tests/functional/ca/why-depends.sh similarity index 100% rename from tests/ca/why-depends.sh rename to tests/functional/ca/why-depends.sh diff --git a/tests/case-hack.sh b/tests/functional/case-hack.sh similarity index 100% rename from tests/case-hack.sh rename to tests/functional/case-hack.sh diff --git a/tests/case.nar b/tests/functional/case.nar similarity index 100% rename from tests/case.nar rename to tests/functional/case.nar diff --git a/tests/check-refs.nix b/tests/functional/check-refs.nix similarity index 100% rename from tests/check-refs.nix rename to tests/functional/check-refs.nix diff --git a/tests/check-refs.sh b/tests/functional/check-refs.sh similarity index 100% rename from tests/check-refs.sh rename to tests/functional/check-refs.sh diff --git a/tests/check-reqs.nix b/tests/functional/check-reqs.nix similarity index 100% rename from tests/check-reqs.nix rename to tests/functional/check-reqs.nix diff --git a/tests/check-reqs.sh b/tests/functional/check-reqs.sh similarity index 100% rename from tests/check-reqs.sh rename to tests/functional/check-reqs.sh diff --git a/tests/check.nix b/tests/functional/check.nix similarity index 100% rename from tests/check.nix rename to tests/functional/check.nix diff --git a/tests/check.sh b/tests/functional/check.sh similarity index 100% rename from tests/check.sh rename to tests/functional/check.sh diff --git a/tests/common.sh b/tests/functional/common.sh similarity index 100% rename from tests/common.sh rename to tests/functional/common.sh diff --git a/tests/common/vars-and-functions.sh.in b/tests/functional/common/vars-and-functions.sh.in similarity index 99% rename from tests/common/vars-and-functions.sh.in rename to tests/functional/common/vars-and-functions.sh.in index 8f9ec4b1a..967d6be54 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/functional/common/vars-and-functions.sh.in @@ -6,7 +6,7 @@ COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1 export PS4='+(${BASH_SOURCE[0]-$0}:$LINENO) ' -export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default} +export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default/tests\/functional//} export NIX_STORE_DIR if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then # Maybe the build directory is symlinked. diff --git a/tests/completions.sh b/tests/functional/completions.sh similarity index 100% rename from tests/completions.sh rename to tests/functional/completions.sh diff --git a/tests/compression-levels.sh b/tests/functional/compression-levels.sh similarity index 100% rename from tests/compression-levels.sh rename to tests/functional/compression-levels.sh diff --git a/tests/compute-levels.sh b/tests/functional/compute-levels.sh similarity index 100% rename from tests/compute-levels.sh rename to tests/functional/compute-levels.sh diff --git a/tests/config.nix.in b/tests/functional/config.nix.in similarity index 100% rename from tests/config.nix.in rename to tests/functional/config.nix.in diff --git a/tests/config.sh b/tests/functional/config.sh similarity index 100% rename from tests/config.sh rename to tests/functional/config.sh diff --git a/tests/config/nix-with-substituters.conf b/tests/functional/config/nix-with-substituters.conf similarity index 100% rename from tests/config/nix-with-substituters.conf rename to tests/functional/config/nix-with-substituters.conf diff --git a/tests/db-migration.sh b/tests/functional/db-migration.sh similarity index 100% rename from tests/db-migration.sh rename to tests/functional/db-migration.sh diff --git a/tests/dependencies.builder0.sh b/tests/functional/dependencies.builder0.sh similarity index 100% rename from tests/dependencies.builder0.sh rename to tests/functional/dependencies.builder0.sh diff --git a/tests/dependencies.nix b/tests/functional/dependencies.nix similarity index 100% rename from tests/dependencies.nix rename to tests/functional/dependencies.nix diff --git a/tests/dependencies.sh b/tests/functional/dependencies.sh similarity index 100% rename from tests/dependencies.sh rename to tests/functional/dependencies.sh diff --git a/tests/derivation-json.sh b/tests/functional/derivation-json.sh similarity index 100% rename from tests/derivation-json.sh rename to tests/functional/derivation-json.sh diff --git a/tests/dummy b/tests/functional/dummy similarity index 100% rename from tests/dummy rename to tests/functional/dummy diff --git a/tests/dump-db.sh b/tests/functional/dump-db.sh similarity index 100% rename from tests/dump-db.sh rename to tests/functional/dump-db.sh diff --git a/tests/dyn-drv/build-built-drv.sh b/tests/functional/dyn-drv/build-built-drv.sh similarity index 100% rename from tests/dyn-drv/build-built-drv.sh rename to tests/functional/dyn-drv/build-built-drv.sh diff --git a/tests/dyn-drv/common.sh b/tests/functional/dyn-drv/common.sh similarity index 100% rename from tests/dyn-drv/common.sh rename to tests/functional/dyn-drv/common.sh diff --git a/tests/dyn-drv/config.nix.in b/tests/functional/dyn-drv/config.nix.in similarity index 100% rename from tests/dyn-drv/config.nix.in rename to tests/functional/dyn-drv/config.nix.in diff --git a/tests/dyn-drv/dep-built-drv.sh b/tests/functional/dyn-drv/dep-built-drv.sh similarity index 100% rename from tests/dyn-drv/dep-built-drv.sh rename to tests/functional/dyn-drv/dep-built-drv.sh diff --git a/tests/dyn-drv/eval-outputOf.sh b/tests/functional/dyn-drv/eval-outputOf.sh similarity index 100% rename from tests/dyn-drv/eval-outputOf.sh rename to tests/functional/dyn-drv/eval-outputOf.sh diff --git a/tests/dyn-drv/local.mk b/tests/functional/dyn-drv/local.mk similarity index 87% rename from tests/dyn-drv/local.mk rename to tests/functional/dyn-drv/local.mk index 6b435499b..c87534944 100644 --- a/tests/dyn-drv/local.mk +++ b/tests/functional/dyn-drv/local.mk @@ -12,4 +12,4 @@ clean-files += \ $(d)/config.nix test-deps += \ - tests/dyn-drv/config.nix + tests/functional/dyn-drv/config.nix diff --git a/tests/dyn-drv/old-daemon-error-hack.nix b/tests/functional/dyn-drv/old-daemon-error-hack.nix similarity index 100% rename from tests/dyn-drv/old-daemon-error-hack.nix rename to tests/functional/dyn-drv/old-daemon-error-hack.nix diff --git a/tests/dyn-drv/old-daemon-error-hack.sh b/tests/functional/dyn-drv/old-daemon-error-hack.sh similarity index 100% rename from tests/dyn-drv/old-daemon-error-hack.sh rename to tests/functional/dyn-drv/old-daemon-error-hack.sh diff --git a/tests/dyn-drv/recursive-mod-json.nix b/tests/functional/dyn-drv/recursive-mod-json.nix similarity index 100% rename from tests/dyn-drv/recursive-mod-json.nix rename to tests/functional/dyn-drv/recursive-mod-json.nix diff --git a/tests/dyn-drv/recursive-mod-json.sh b/tests/functional/dyn-drv/recursive-mod-json.sh similarity index 100% rename from tests/dyn-drv/recursive-mod-json.sh rename to tests/functional/dyn-drv/recursive-mod-json.sh diff --git a/tests/dyn-drv/text-hashed-output.nix b/tests/functional/dyn-drv/text-hashed-output.nix similarity index 100% rename from tests/dyn-drv/text-hashed-output.nix rename to tests/functional/dyn-drv/text-hashed-output.nix diff --git a/tests/dyn-drv/text-hashed-output.sh b/tests/functional/dyn-drv/text-hashed-output.sh similarity index 100% rename from tests/dyn-drv/text-hashed-output.sh rename to tests/functional/dyn-drv/text-hashed-output.sh diff --git a/tests/eval-store.sh b/tests/functional/eval-store.sh similarity index 100% rename from tests/eval-store.sh rename to tests/functional/eval-store.sh diff --git a/tests/eval.nix b/tests/functional/eval.nix similarity index 100% rename from tests/eval.nix rename to tests/functional/eval.nix diff --git a/tests/eval.sh b/tests/functional/eval.sh similarity index 100% rename from tests/eval.sh rename to tests/functional/eval.sh diff --git a/tests/experimental-features.sh b/tests/functional/experimental-features.sh similarity index 100% rename from tests/experimental-features.sh rename to tests/functional/experimental-features.sh diff --git a/tests/export-graph.nix b/tests/functional/export-graph.nix similarity index 100% rename from tests/export-graph.nix rename to tests/functional/export-graph.nix diff --git a/tests/export-graph.sh b/tests/functional/export-graph.sh similarity index 100% rename from tests/export-graph.sh rename to tests/functional/export-graph.sh diff --git a/tests/export.sh b/tests/functional/export.sh similarity index 100% rename from tests/export.sh rename to tests/functional/export.sh diff --git a/tests/failing.nix b/tests/functional/failing.nix similarity index 100% rename from tests/failing.nix rename to tests/functional/failing.nix diff --git a/tests/fetchClosure.sh b/tests/functional/fetchClosure.sh similarity index 100% rename from tests/fetchClosure.sh rename to tests/functional/fetchClosure.sh diff --git a/tests/fetchGit.sh b/tests/functional/fetchGit.sh similarity index 100% rename from tests/fetchGit.sh rename to tests/functional/fetchGit.sh diff --git a/tests/fetchGitRefs.sh b/tests/functional/fetchGitRefs.sh similarity index 100% rename from tests/fetchGitRefs.sh rename to tests/functional/fetchGitRefs.sh diff --git a/tests/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh similarity index 100% rename from tests/fetchGitSubmodules.sh rename to tests/functional/fetchGitSubmodules.sh diff --git a/tests/fetchMercurial.sh b/tests/functional/fetchMercurial.sh similarity index 100% rename from tests/fetchMercurial.sh rename to tests/functional/fetchMercurial.sh diff --git a/tests/fetchPath.sh b/tests/functional/fetchPath.sh similarity index 100% rename from tests/fetchPath.sh rename to tests/functional/fetchPath.sh diff --git a/tests/fetchTree-file.sh b/tests/functional/fetchTree-file.sh similarity index 100% rename from tests/fetchTree-file.sh rename to tests/functional/fetchTree-file.sh diff --git a/tests/fetchurl.sh b/tests/functional/fetchurl.sh similarity index 100% rename from tests/fetchurl.sh rename to tests/functional/fetchurl.sh diff --git a/tests/filter-source.nix b/tests/functional/filter-source.nix similarity index 100% rename from tests/filter-source.nix rename to tests/functional/filter-source.nix diff --git a/tests/filter-source.sh b/tests/functional/filter-source.sh similarity index 100% rename from tests/filter-source.sh rename to tests/functional/filter-source.sh diff --git a/tests/fixed.builder1.sh b/tests/functional/fixed.builder1.sh similarity index 100% rename from tests/fixed.builder1.sh rename to tests/functional/fixed.builder1.sh diff --git a/tests/fixed.builder2.sh b/tests/functional/fixed.builder2.sh similarity index 100% rename from tests/fixed.builder2.sh rename to tests/functional/fixed.builder2.sh diff --git a/tests/fixed.nix b/tests/functional/fixed.nix similarity index 100% rename from tests/fixed.nix rename to tests/functional/fixed.nix diff --git a/tests/fixed.sh b/tests/functional/fixed.sh similarity index 100% rename from tests/fixed.sh rename to tests/functional/fixed.sh diff --git a/tests/flakes/absolute-attr-paths.sh b/tests/functional/flakes/absolute-attr-paths.sh similarity index 100% rename from tests/flakes/absolute-attr-paths.sh rename to tests/functional/flakes/absolute-attr-paths.sh diff --git a/tests/flakes/absolute-paths.sh b/tests/functional/flakes/absolute-paths.sh similarity index 100% rename from tests/flakes/absolute-paths.sh rename to tests/functional/flakes/absolute-paths.sh diff --git a/tests/flakes/build-paths.sh b/tests/functional/flakes/build-paths.sh similarity index 100% rename from tests/flakes/build-paths.sh rename to tests/functional/flakes/build-paths.sh diff --git a/tests/flakes/bundle.sh b/tests/functional/flakes/bundle.sh similarity index 100% rename from tests/flakes/bundle.sh rename to tests/functional/flakes/bundle.sh diff --git a/tests/flakes/check.sh b/tests/functional/flakes/check.sh similarity index 100% rename from tests/flakes/check.sh rename to tests/functional/flakes/check.sh diff --git a/tests/flakes/circular.sh b/tests/functional/flakes/circular.sh similarity index 100% rename from tests/flakes/circular.sh rename to tests/functional/flakes/circular.sh diff --git a/tests/flakes/common.sh b/tests/functional/flakes/common.sh similarity index 100% rename from tests/flakes/common.sh rename to tests/functional/flakes/common.sh diff --git a/tests/flakes/config.sh b/tests/functional/flakes/config.sh similarity index 100% rename from tests/flakes/config.sh rename to tests/functional/flakes/config.sh diff --git a/tests/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh similarity index 100% rename from tests/flakes/flake-in-submodule.sh rename to tests/functional/flakes/flake-in-submodule.sh diff --git a/tests/flakes/flakes.sh b/tests/functional/flakes/flakes.sh similarity index 100% rename from tests/flakes/flakes.sh rename to tests/functional/flakes/flakes.sh diff --git a/tests/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh similarity index 100% rename from tests/flakes/follow-paths.sh rename to tests/functional/flakes/follow-paths.sh diff --git a/tests/flakes/init.sh b/tests/functional/flakes/init.sh similarity index 100% rename from tests/flakes/init.sh rename to tests/functional/flakes/init.sh diff --git a/tests/flakes/inputs.sh b/tests/functional/flakes/inputs.sh similarity index 100% rename from tests/flakes/inputs.sh rename to tests/functional/flakes/inputs.sh diff --git a/tests/flakes/mercurial.sh b/tests/functional/flakes/mercurial.sh similarity index 100% rename from tests/flakes/mercurial.sh rename to tests/functional/flakes/mercurial.sh diff --git a/tests/flakes/run.sh b/tests/functional/flakes/run.sh similarity index 100% rename from tests/flakes/run.sh rename to tests/functional/flakes/run.sh diff --git a/tests/flakes/search-root.sh b/tests/functional/flakes/search-root.sh similarity index 100% rename from tests/flakes/search-root.sh rename to tests/functional/flakes/search-root.sh diff --git a/tests/flakes/show.sh b/tests/functional/flakes/show.sh similarity index 100% rename from tests/flakes/show.sh rename to tests/functional/flakes/show.sh diff --git a/tests/flakes/unlocked-override.sh b/tests/functional/flakes/unlocked-override.sh similarity index 100% rename from tests/flakes/unlocked-override.sh rename to tests/functional/flakes/unlocked-override.sh diff --git a/tests/fmt.sh b/tests/functional/fmt.sh similarity index 100% rename from tests/fmt.sh rename to tests/functional/fmt.sh diff --git a/tests/fmt.simple.sh b/tests/functional/fmt.simple.sh similarity index 100% rename from tests/fmt.simple.sh rename to tests/functional/fmt.simple.sh diff --git a/tests/function-trace.sh b/tests/functional/function-trace.sh similarity index 100% rename from tests/function-trace.sh rename to tests/functional/function-trace.sh diff --git a/tests/gc-auto.sh b/tests/functional/gc-auto.sh similarity index 100% rename from tests/gc-auto.sh rename to tests/functional/gc-auto.sh diff --git a/tests/gc-concurrent.builder.sh b/tests/functional/gc-concurrent.builder.sh similarity index 100% rename from tests/gc-concurrent.builder.sh rename to tests/functional/gc-concurrent.builder.sh diff --git a/tests/gc-concurrent.nix b/tests/functional/gc-concurrent.nix similarity index 100% rename from tests/gc-concurrent.nix rename to tests/functional/gc-concurrent.nix diff --git a/tests/gc-concurrent.sh b/tests/functional/gc-concurrent.sh similarity index 100% rename from tests/gc-concurrent.sh rename to tests/functional/gc-concurrent.sh diff --git a/tests/gc-concurrent2.builder.sh b/tests/functional/gc-concurrent2.builder.sh similarity index 100% rename from tests/gc-concurrent2.builder.sh rename to tests/functional/gc-concurrent2.builder.sh diff --git a/tests/gc-non-blocking.sh b/tests/functional/gc-non-blocking.sh similarity index 100% rename from tests/gc-non-blocking.sh rename to tests/functional/gc-non-blocking.sh diff --git a/tests/gc-runtime.nix b/tests/functional/gc-runtime.nix similarity index 100% rename from tests/gc-runtime.nix rename to tests/functional/gc-runtime.nix diff --git a/tests/gc-runtime.sh b/tests/functional/gc-runtime.sh similarity index 100% rename from tests/gc-runtime.sh rename to tests/functional/gc-runtime.sh diff --git a/tests/gc.sh b/tests/functional/gc.sh similarity index 100% rename from tests/gc.sh rename to tests/functional/gc.sh diff --git a/tests/hash-check.nix b/tests/functional/hash-check.nix similarity index 100% rename from tests/hash-check.nix rename to tests/functional/hash-check.nix diff --git a/tests/hash.sh b/tests/functional/hash.sh similarity index 100% rename from tests/hash.sh rename to tests/functional/hash.sh diff --git a/tests/hermetic.nix b/tests/functional/hermetic.nix similarity index 100% rename from tests/hermetic.nix rename to tests/functional/hermetic.nix diff --git a/tests/import-derivation.nix b/tests/functional/import-derivation.nix similarity index 100% rename from tests/import-derivation.nix rename to tests/functional/import-derivation.nix diff --git a/tests/import-derivation.sh b/tests/functional/import-derivation.sh similarity index 100% rename from tests/import-derivation.sh rename to tests/functional/import-derivation.sh diff --git a/tests/impure-derivations.nix b/tests/functional/impure-derivations.nix similarity index 100% rename from tests/impure-derivations.nix rename to tests/functional/impure-derivations.nix diff --git a/tests/impure-derivations.sh b/tests/functional/impure-derivations.sh similarity index 100% rename from tests/impure-derivations.sh rename to tests/functional/impure-derivations.sh diff --git a/tests/init.sh b/tests/functional/init.sh similarity index 100% rename from tests/init.sh rename to tests/functional/init.sh diff --git a/tests/install-darwin.sh b/tests/functional/install-darwin.sh similarity index 100% rename from tests/install-darwin.sh rename to tests/functional/install-darwin.sh diff --git a/tests/lang-test-infra.sh b/tests/functional/lang-test-infra.sh similarity index 100% rename from tests/lang-test-infra.sh rename to tests/functional/lang-test-infra.sh diff --git a/tests/lang.sh b/tests/functional/lang.sh similarity index 98% rename from tests/lang.sh rename to tests/functional/lang.sh index 75dbbc38e..f4760eced 100755 --- a/tests/lang.sh +++ b/tests/functional/lang.sh @@ -134,7 +134,7 @@ else echo '' echo 'You can rerun this test with:' echo '' - echo ' _NIX_TEST_ACCEPT=1 make tests/lang.sh.test' + echo ' _NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test' echo '' echo 'to regenerate the files containing the expected output,' echo 'and then view the git diff to decide whether a change is' diff --git a/tests/lang/binary-data b/tests/functional/lang/binary-data similarity index 100% rename from tests/lang/binary-data rename to tests/functional/lang/binary-data diff --git a/tests/lang/data b/tests/functional/lang/data similarity index 100% rename from tests/lang/data rename to tests/functional/lang/data diff --git a/tests/lang/dir1/a.nix b/tests/functional/lang/dir1/a.nix similarity index 100% rename from tests/lang/dir1/a.nix rename to tests/functional/lang/dir1/a.nix diff --git a/tests/lang/dir2/a.nix b/tests/functional/lang/dir2/a.nix similarity index 100% rename from tests/lang/dir2/a.nix rename to tests/functional/lang/dir2/a.nix diff --git a/tests/lang/dir2/b.nix b/tests/functional/lang/dir2/b.nix similarity index 100% rename from tests/lang/dir2/b.nix rename to tests/functional/lang/dir2/b.nix diff --git a/tests/lang/dir3/a.nix b/tests/functional/lang/dir3/a.nix similarity index 100% rename from tests/lang/dir3/a.nix rename to tests/functional/lang/dir3/a.nix diff --git a/tests/lang/dir3/b.nix b/tests/functional/lang/dir3/b.nix similarity index 100% rename from tests/lang/dir3/b.nix rename to tests/functional/lang/dir3/b.nix diff --git a/tests/lang/dir3/c.nix b/tests/functional/lang/dir3/c.nix similarity index 100% rename from tests/lang/dir3/c.nix rename to tests/functional/lang/dir3/c.nix diff --git a/tests/lang/dir4/a.nix b/tests/functional/lang/dir4/a.nix similarity index 100% rename from tests/lang/dir4/a.nix rename to tests/functional/lang/dir4/a.nix diff --git a/tests/lang/dir4/c.nix b/tests/functional/lang/dir4/c.nix similarity index 100% rename from tests/lang/dir4/c.nix rename to tests/functional/lang/dir4/c.nix diff --git a/tests/lang/empty.exp b/tests/functional/lang/empty.exp similarity index 100% rename from tests/lang/empty.exp rename to tests/functional/lang/empty.exp diff --git a/tests/lang/eval-fail-abort.err.exp b/tests/functional/lang/eval-fail-abort.err.exp similarity index 100% rename from tests/lang/eval-fail-abort.err.exp rename to tests/functional/lang/eval-fail-abort.err.exp diff --git a/tests/lang/eval-fail-abort.nix b/tests/functional/lang/eval-fail-abort.nix similarity index 100% rename from tests/lang/eval-fail-abort.nix rename to tests/functional/lang/eval-fail-abort.nix diff --git a/tests/lang/eval-fail-antiquoted-path.err.exp b/tests/functional/lang/eval-fail-antiquoted-path.err.exp similarity index 100% rename from tests/lang/eval-fail-antiquoted-path.err.exp rename to tests/functional/lang/eval-fail-antiquoted-path.err.exp diff --git a/tests/lang/eval-fail-assert.err.exp b/tests/functional/lang/eval-fail-assert.err.exp similarity index 100% rename from tests/lang/eval-fail-assert.err.exp rename to tests/functional/lang/eval-fail-assert.err.exp diff --git a/tests/lang/eval-fail-assert.nix b/tests/functional/lang/eval-fail-assert.nix similarity index 100% rename from tests/lang/eval-fail-assert.nix rename to tests/functional/lang/eval-fail-assert.nix diff --git a/tests/lang/eval-fail-bad-antiquote-1.err.exp b/tests/functional/lang/eval-fail-bad-antiquote-1.err.exp similarity index 100% rename from tests/lang/eval-fail-bad-antiquote-1.err.exp rename to tests/functional/lang/eval-fail-bad-antiquote-1.err.exp diff --git a/tests/lang/eval-fail-bad-antiquote-2.err.exp b/tests/functional/lang/eval-fail-bad-antiquote-2.err.exp similarity index 100% rename from tests/lang/eval-fail-bad-antiquote-2.err.exp rename to tests/functional/lang/eval-fail-bad-antiquote-2.err.exp diff --git a/tests/lang/eval-fail-bad-antiquote-3.err.exp b/tests/functional/lang/eval-fail-bad-antiquote-3.err.exp similarity index 100% rename from tests/lang/eval-fail-bad-antiquote-3.err.exp rename to tests/functional/lang/eval-fail-bad-antiquote-3.err.exp diff --git a/tests/lang/eval-fail-bad-string-interpolation-1.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp similarity index 100% rename from tests/lang/eval-fail-bad-string-interpolation-1.err.exp rename to tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp diff --git a/tests/lang/eval-fail-bad-string-interpolation-1.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-1.nix similarity index 100% rename from tests/lang/eval-fail-bad-string-interpolation-1.nix rename to tests/functional/lang/eval-fail-bad-string-interpolation-1.nix diff --git a/tests/lang/eval-fail-bad-string-interpolation-2.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp similarity index 100% rename from tests/lang/eval-fail-bad-string-interpolation-2.err.exp rename to tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp diff --git a/tests/lang/eval-fail-bad-string-interpolation-2.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-2.nix similarity index 100% rename from tests/lang/eval-fail-bad-string-interpolation-2.nix rename to tests/functional/lang/eval-fail-bad-string-interpolation-2.nix diff --git a/tests/lang/eval-fail-bad-string-interpolation-3.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp similarity index 100% rename from tests/lang/eval-fail-bad-string-interpolation-3.err.exp rename to tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp diff --git a/tests/lang/eval-fail-bad-string-interpolation-3.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-3.nix similarity index 100% rename from tests/lang/eval-fail-bad-string-interpolation-3.nix rename to tests/functional/lang/eval-fail-bad-string-interpolation-3.nix diff --git a/tests/lang/eval-fail-blackhole.err.exp b/tests/functional/lang/eval-fail-blackhole.err.exp similarity index 100% rename from tests/lang/eval-fail-blackhole.err.exp rename to tests/functional/lang/eval-fail-blackhole.err.exp diff --git a/tests/lang/eval-fail-blackhole.nix b/tests/functional/lang/eval-fail-blackhole.nix similarity index 100% rename from tests/lang/eval-fail-blackhole.nix rename to tests/functional/lang/eval-fail-blackhole.nix diff --git a/tests/lang/eval-fail-deepseq.err.exp b/tests/functional/lang/eval-fail-deepseq.err.exp similarity index 100% rename from tests/lang/eval-fail-deepseq.err.exp rename to tests/functional/lang/eval-fail-deepseq.err.exp diff --git a/tests/lang/eval-fail-deepseq.nix b/tests/functional/lang/eval-fail-deepseq.nix similarity index 100% rename from tests/lang/eval-fail-deepseq.nix rename to tests/functional/lang/eval-fail-deepseq.nix diff --git a/tests/lang/eval-fail-dup-dynamic-attrs.err.exp b/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp similarity index 100% rename from tests/lang/eval-fail-dup-dynamic-attrs.err.exp rename to tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp diff --git a/tests/lang/eval-fail-dup-dynamic-attrs.nix b/tests/functional/lang/eval-fail-dup-dynamic-attrs.nix similarity index 100% rename from tests/lang/eval-fail-dup-dynamic-attrs.nix rename to tests/functional/lang/eval-fail-dup-dynamic-attrs.nix diff --git a/tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp similarity index 100% rename from tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp rename to tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp diff --git a/tests/lang/eval-fail-foldlStrict-strict-op-application.nix b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.nix similarity index 100% rename from tests/lang/eval-fail-foldlStrict-strict-op-application.nix rename to tests/functional/lang/eval-fail-foldlStrict-strict-op-application.nix diff --git a/tests/lang/eval-fail-fromTOML-timestamps.err.exp b/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp similarity index 100% rename from tests/lang/eval-fail-fromTOML-timestamps.err.exp rename to tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp diff --git a/tests/lang/eval-fail-fromTOML-timestamps.nix b/tests/functional/lang/eval-fail-fromTOML-timestamps.nix similarity index 100% rename from tests/lang/eval-fail-fromTOML-timestamps.nix rename to tests/functional/lang/eval-fail-fromTOML-timestamps.nix diff --git a/tests/lang/eval-fail-hashfile-missing.err.exp b/tests/functional/lang/eval-fail-hashfile-missing.err.exp similarity index 100% rename from tests/lang/eval-fail-hashfile-missing.err.exp rename to tests/functional/lang/eval-fail-hashfile-missing.err.exp diff --git a/tests/lang/eval-fail-hashfile-missing.nix b/tests/functional/lang/eval-fail-hashfile-missing.nix similarity index 100% rename from tests/lang/eval-fail-hashfile-missing.nix rename to tests/functional/lang/eval-fail-hashfile-missing.nix diff --git a/tests/lang/eval-fail-list.err.exp b/tests/functional/lang/eval-fail-list.err.exp similarity index 100% rename from tests/lang/eval-fail-list.err.exp rename to tests/functional/lang/eval-fail-list.err.exp diff --git a/tests/lang/eval-fail-list.nix b/tests/functional/lang/eval-fail-list.nix similarity index 100% rename from tests/lang/eval-fail-list.nix rename to tests/functional/lang/eval-fail-list.nix diff --git a/tests/lang/eval-fail-missing-arg.err.exp b/tests/functional/lang/eval-fail-missing-arg.err.exp similarity index 100% rename from tests/lang/eval-fail-missing-arg.err.exp rename to tests/functional/lang/eval-fail-missing-arg.err.exp diff --git a/tests/lang/eval-fail-missing-arg.nix b/tests/functional/lang/eval-fail-missing-arg.nix similarity index 100% rename from tests/lang/eval-fail-missing-arg.nix rename to tests/functional/lang/eval-fail-missing-arg.nix diff --git a/tests/lang/eval-fail-nonexist-path.err.exp b/tests/functional/lang/eval-fail-nonexist-path.err.exp similarity index 100% rename from tests/lang/eval-fail-nonexist-path.err.exp rename to tests/functional/lang/eval-fail-nonexist-path.err.exp diff --git a/tests/lang/eval-fail-nonexist-path.nix b/tests/functional/lang/eval-fail-nonexist-path.nix similarity index 100% rename from tests/lang/eval-fail-nonexist-path.nix rename to tests/functional/lang/eval-fail-nonexist-path.nix diff --git a/tests/lang/eval-fail-path-slash.err.exp b/tests/functional/lang/eval-fail-path-slash.err.exp similarity index 100% rename from tests/lang/eval-fail-path-slash.err.exp rename to tests/functional/lang/eval-fail-path-slash.err.exp diff --git a/tests/lang/eval-fail-path-slash.nix b/tests/functional/lang/eval-fail-path-slash.nix similarity index 100% rename from tests/lang/eval-fail-path-slash.nix rename to tests/functional/lang/eval-fail-path-slash.nix diff --git a/tests/lang/eval-fail-recursion.err.exp b/tests/functional/lang/eval-fail-recursion.err.exp similarity index 100% rename from tests/lang/eval-fail-recursion.err.exp rename to tests/functional/lang/eval-fail-recursion.err.exp diff --git a/tests/lang/eval-fail-recursion.nix b/tests/functional/lang/eval-fail-recursion.nix similarity index 100% rename from tests/lang/eval-fail-recursion.nix rename to tests/functional/lang/eval-fail-recursion.nix diff --git a/tests/lang/eval-fail-remove.err.exp b/tests/functional/lang/eval-fail-remove.err.exp similarity index 100% rename from tests/lang/eval-fail-remove.err.exp rename to tests/functional/lang/eval-fail-remove.err.exp diff --git a/tests/lang/eval-fail-remove.nix b/tests/functional/lang/eval-fail-remove.nix similarity index 100% rename from tests/lang/eval-fail-remove.nix rename to tests/functional/lang/eval-fail-remove.nix diff --git a/tests/lang/eval-fail-scope-5.err.exp b/tests/functional/lang/eval-fail-scope-5.err.exp similarity index 100% rename from tests/lang/eval-fail-scope-5.err.exp rename to tests/functional/lang/eval-fail-scope-5.err.exp diff --git a/tests/lang/eval-fail-scope-5.nix b/tests/functional/lang/eval-fail-scope-5.nix similarity index 100% rename from tests/lang/eval-fail-scope-5.nix rename to tests/functional/lang/eval-fail-scope-5.nix diff --git a/tests/lang/eval-fail-seq.err.exp b/tests/functional/lang/eval-fail-seq.err.exp similarity index 100% rename from tests/lang/eval-fail-seq.err.exp rename to tests/functional/lang/eval-fail-seq.err.exp diff --git a/tests/lang/eval-fail-seq.nix b/tests/functional/lang/eval-fail-seq.nix similarity index 100% rename from tests/lang/eval-fail-seq.nix rename to tests/functional/lang/eval-fail-seq.nix diff --git a/tests/lang/eval-fail-set-override.err.exp b/tests/functional/lang/eval-fail-set-override.err.exp similarity index 100% rename from tests/lang/eval-fail-set-override.err.exp rename to tests/functional/lang/eval-fail-set-override.err.exp diff --git a/tests/lang/eval-fail-set-override.nix b/tests/functional/lang/eval-fail-set-override.nix similarity index 100% rename from tests/lang/eval-fail-set-override.nix rename to tests/functional/lang/eval-fail-set-override.nix diff --git a/tests/lang/eval-fail-set.err.exp b/tests/functional/lang/eval-fail-set.err.exp similarity index 100% rename from tests/lang/eval-fail-set.err.exp rename to tests/functional/lang/eval-fail-set.err.exp diff --git a/tests/lang/eval-fail-set.nix b/tests/functional/lang/eval-fail-set.nix similarity index 100% rename from tests/lang/eval-fail-set.nix rename to tests/functional/lang/eval-fail-set.nix diff --git a/tests/lang/eval-fail-substring.err.exp b/tests/functional/lang/eval-fail-substring.err.exp similarity index 100% rename from tests/lang/eval-fail-substring.err.exp rename to tests/functional/lang/eval-fail-substring.err.exp diff --git a/tests/lang/eval-fail-substring.nix b/tests/functional/lang/eval-fail-substring.nix similarity index 100% rename from tests/lang/eval-fail-substring.nix rename to tests/functional/lang/eval-fail-substring.nix diff --git a/tests/lang/eval-fail-to-path.err.exp b/tests/functional/lang/eval-fail-to-path.err.exp similarity index 100% rename from tests/lang/eval-fail-to-path.err.exp rename to tests/functional/lang/eval-fail-to-path.err.exp diff --git a/tests/lang/eval-fail-to-path.nix b/tests/functional/lang/eval-fail-to-path.nix similarity index 100% rename from tests/lang/eval-fail-to-path.nix rename to tests/functional/lang/eval-fail-to-path.nix diff --git a/tests/lang/eval-fail-toJSON.err.exp b/tests/functional/lang/eval-fail-toJSON.err.exp similarity index 100% rename from tests/lang/eval-fail-toJSON.err.exp rename to tests/functional/lang/eval-fail-toJSON.err.exp diff --git a/tests/lang/eval-fail-toJSON.nix b/tests/functional/lang/eval-fail-toJSON.nix similarity index 100% rename from tests/lang/eval-fail-toJSON.nix rename to tests/functional/lang/eval-fail-toJSON.nix diff --git a/tests/lang/eval-fail-undeclared-arg.err.exp b/tests/functional/lang/eval-fail-undeclared-arg.err.exp similarity index 100% rename from tests/lang/eval-fail-undeclared-arg.err.exp rename to tests/functional/lang/eval-fail-undeclared-arg.err.exp diff --git a/tests/lang/eval-fail-undeclared-arg.nix b/tests/functional/lang/eval-fail-undeclared-arg.nix similarity index 100% rename from tests/lang/eval-fail-undeclared-arg.nix rename to tests/functional/lang/eval-fail-undeclared-arg.nix diff --git a/tests/lang/eval-okay-any-all.exp b/tests/functional/lang/eval-okay-any-all.exp similarity index 100% rename from tests/lang/eval-okay-any-all.exp rename to tests/functional/lang/eval-okay-any-all.exp diff --git a/tests/lang/eval-okay-any-all.nix b/tests/functional/lang/eval-okay-any-all.nix similarity index 100% rename from tests/lang/eval-okay-any-all.nix rename to tests/functional/lang/eval-okay-any-all.nix diff --git a/tests/lang/eval-okay-arithmetic.exp b/tests/functional/lang/eval-okay-arithmetic.exp similarity index 100% rename from tests/lang/eval-okay-arithmetic.exp rename to tests/functional/lang/eval-okay-arithmetic.exp diff --git a/tests/lang/eval-okay-arithmetic.nix b/tests/functional/lang/eval-okay-arithmetic.nix similarity index 100% rename from tests/lang/eval-okay-arithmetic.nix rename to tests/functional/lang/eval-okay-arithmetic.nix diff --git a/tests/lang/eval-okay-attrnames.exp b/tests/functional/lang/eval-okay-attrnames.exp similarity index 100% rename from tests/lang/eval-okay-attrnames.exp rename to tests/functional/lang/eval-okay-attrnames.exp diff --git a/tests/lang/eval-okay-attrnames.nix b/tests/functional/lang/eval-okay-attrnames.nix similarity index 100% rename from tests/lang/eval-okay-attrnames.nix rename to tests/functional/lang/eval-okay-attrnames.nix diff --git a/tests/lang/eval-okay-attrs.exp b/tests/functional/lang/eval-okay-attrs.exp similarity index 100% rename from tests/lang/eval-okay-attrs.exp rename to tests/functional/lang/eval-okay-attrs.exp diff --git a/tests/lang/eval-okay-attrs.nix b/tests/functional/lang/eval-okay-attrs.nix similarity index 100% rename from tests/lang/eval-okay-attrs.nix rename to tests/functional/lang/eval-okay-attrs.nix diff --git a/tests/lang/eval-okay-attrs2.exp b/tests/functional/lang/eval-okay-attrs2.exp similarity index 100% rename from tests/lang/eval-okay-attrs2.exp rename to tests/functional/lang/eval-okay-attrs2.exp diff --git a/tests/lang/eval-okay-attrs2.nix b/tests/functional/lang/eval-okay-attrs2.nix similarity index 100% rename from tests/lang/eval-okay-attrs2.nix rename to tests/functional/lang/eval-okay-attrs2.nix diff --git a/tests/lang/eval-okay-attrs3.exp b/tests/functional/lang/eval-okay-attrs3.exp similarity index 100% rename from tests/lang/eval-okay-attrs3.exp rename to tests/functional/lang/eval-okay-attrs3.exp diff --git a/tests/lang/eval-okay-attrs3.nix b/tests/functional/lang/eval-okay-attrs3.nix similarity index 100% rename from tests/lang/eval-okay-attrs3.nix rename to tests/functional/lang/eval-okay-attrs3.nix diff --git a/tests/lang/eval-okay-attrs4.exp b/tests/functional/lang/eval-okay-attrs4.exp similarity index 100% rename from tests/lang/eval-okay-attrs4.exp rename to tests/functional/lang/eval-okay-attrs4.exp diff --git a/tests/lang/eval-okay-attrs4.nix b/tests/functional/lang/eval-okay-attrs4.nix similarity index 100% rename from tests/lang/eval-okay-attrs4.nix rename to tests/functional/lang/eval-okay-attrs4.nix diff --git a/tests/lang/eval-okay-attrs5.exp b/tests/functional/lang/eval-okay-attrs5.exp similarity index 100% rename from tests/lang/eval-okay-attrs5.exp rename to tests/functional/lang/eval-okay-attrs5.exp diff --git a/tests/lang/eval-okay-attrs5.nix b/tests/functional/lang/eval-okay-attrs5.nix similarity index 100% rename from tests/lang/eval-okay-attrs5.nix rename to tests/functional/lang/eval-okay-attrs5.nix diff --git a/tests/lang/eval-okay-attrs6.exp b/tests/functional/lang/eval-okay-attrs6.exp similarity index 100% rename from tests/lang/eval-okay-attrs6.exp rename to tests/functional/lang/eval-okay-attrs6.exp diff --git a/tests/lang/eval-okay-attrs6.nix b/tests/functional/lang/eval-okay-attrs6.nix similarity index 100% rename from tests/lang/eval-okay-attrs6.nix rename to tests/functional/lang/eval-okay-attrs6.nix diff --git a/tests/lang/eval-okay-autoargs.exp b/tests/functional/lang/eval-okay-autoargs.exp similarity index 100% rename from tests/lang/eval-okay-autoargs.exp rename to tests/functional/lang/eval-okay-autoargs.exp diff --git a/tests/lang/eval-okay-autoargs.flags b/tests/functional/lang/eval-okay-autoargs.flags similarity index 100% rename from tests/lang/eval-okay-autoargs.flags rename to tests/functional/lang/eval-okay-autoargs.flags diff --git a/tests/lang/eval-okay-autoargs.nix b/tests/functional/lang/eval-okay-autoargs.nix similarity index 100% rename from tests/lang/eval-okay-autoargs.nix rename to tests/functional/lang/eval-okay-autoargs.nix diff --git a/tests/lang/eval-okay-backslash-newline-1.exp b/tests/functional/lang/eval-okay-backslash-newline-1.exp similarity index 100% rename from tests/lang/eval-okay-backslash-newline-1.exp rename to tests/functional/lang/eval-okay-backslash-newline-1.exp diff --git a/tests/lang/eval-okay-backslash-newline-1.nix b/tests/functional/lang/eval-okay-backslash-newline-1.nix similarity index 100% rename from tests/lang/eval-okay-backslash-newline-1.nix rename to tests/functional/lang/eval-okay-backslash-newline-1.nix diff --git a/tests/lang/eval-okay-backslash-newline-2.exp b/tests/functional/lang/eval-okay-backslash-newline-2.exp similarity index 100% rename from tests/lang/eval-okay-backslash-newline-2.exp rename to tests/functional/lang/eval-okay-backslash-newline-2.exp diff --git a/tests/lang/eval-okay-backslash-newline-2.nix b/tests/functional/lang/eval-okay-backslash-newline-2.nix similarity index 100% rename from tests/lang/eval-okay-backslash-newline-2.nix rename to tests/functional/lang/eval-okay-backslash-newline-2.nix diff --git a/tests/lang/eval-okay-builtins-add.exp b/tests/functional/lang/eval-okay-builtins-add.exp similarity index 100% rename from tests/lang/eval-okay-builtins-add.exp rename to tests/functional/lang/eval-okay-builtins-add.exp diff --git a/tests/lang/eval-okay-builtins-add.nix b/tests/functional/lang/eval-okay-builtins-add.nix similarity index 100% rename from tests/lang/eval-okay-builtins-add.nix rename to tests/functional/lang/eval-okay-builtins-add.nix diff --git a/tests/lang/eval-okay-builtins.exp b/tests/functional/lang/eval-okay-builtins.exp similarity index 100% rename from tests/lang/eval-okay-builtins.exp rename to tests/functional/lang/eval-okay-builtins.exp diff --git a/tests/lang/eval-okay-builtins.nix b/tests/functional/lang/eval-okay-builtins.nix similarity index 100% rename from tests/lang/eval-okay-builtins.nix rename to tests/functional/lang/eval-okay-builtins.nix diff --git a/tests/lang/eval-okay-callable-attrs.exp b/tests/functional/lang/eval-okay-callable-attrs.exp similarity index 100% rename from tests/lang/eval-okay-callable-attrs.exp rename to tests/functional/lang/eval-okay-callable-attrs.exp diff --git a/tests/lang/eval-okay-callable-attrs.nix b/tests/functional/lang/eval-okay-callable-attrs.nix similarity index 100% rename from tests/lang/eval-okay-callable-attrs.nix rename to tests/functional/lang/eval-okay-callable-attrs.nix diff --git a/tests/lang/eval-okay-catattrs.exp b/tests/functional/lang/eval-okay-catattrs.exp similarity index 100% rename from tests/lang/eval-okay-catattrs.exp rename to tests/functional/lang/eval-okay-catattrs.exp diff --git a/tests/lang/eval-okay-catattrs.nix b/tests/functional/lang/eval-okay-catattrs.nix similarity index 100% rename from tests/lang/eval-okay-catattrs.nix rename to tests/functional/lang/eval-okay-catattrs.nix diff --git a/tests/lang/eval-okay-closure.exp b/tests/functional/lang/eval-okay-closure.exp similarity index 100% rename from tests/lang/eval-okay-closure.exp rename to tests/functional/lang/eval-okay-closure.exp diff --git a/tests/lang/eval-okay-closure.exp.xml b/tests/functional/lang/eval-okay-closure.exp.xml similarity index 100% rename from tests/lang/eval-okay-closure.exp.xml rename to tests/functional/lang/eval-okay-closure.exp.xml diff --git a/tests/lang/eval-okay-closure.nix b/tests/functional/lang/eval-okay-closure.nix similarity index 100% rename from tests/lang/eval-okay-closure.nix rename to tests/functional/lang/eval-okay-closure.nix diff --git a/tests/lang/eval-okay-comments.exp b/tests/functional/lang/eval-okay-comments.exp similarity index 100% rename from tests/lang/eval-okay-comments.exp rename to tests/functional/lang/eval-okay-comments.exp diff --git a/tests/lang/eval-okay-comments.nix b/tests/functional/lang/eval-okay-comments.nix similarity index 100% rename from tests/lang/eval-okay-comments.nix rename to tests/functional/lang/eval-okay-comments.nix diff --git a/tests/lang/eval-okay-concat.exp b/tests/functional/lang/eval-okay-concat.exp similarity index 100% rename from tests/lang/eval-okay-concat.exp rename to tests/functional/lang/eval-okay-concat.exp diff --git a/tests/lang/eval-okay-concat.nix b/tests/functional/lang/eval-okay-concat.nix similarity index 100% rename from tests/lang/eval-okay-concat.nix rename to tests/functional/lang/eval-okay-concat.nix diff --git a/tests/lang/eval-okay-concatmap.exp b/tests/functional/lang/eval-okay-concatmap.exp similarity index 100% rename from tests/lang/eval-okay-concatmap.exp rename to tests/functional/lang/eval-okay-concatmap.exp diff --git a/tests/lang/eval-okay-concatmap.nix b/tests/functional/lang/eval-okay-concatmap.nix similarity index 100% rename from tests/lang/eval-okay-concatmap.nix rename to tests/functional/lang/eval-okay-concatmap.nix diff --git a/tests/lang/eval-okay-concatstringssep.exp b/tests/functional/lang/eval-okay-concatstringssep.exp similarity index 100% rename from tests/lang/eval-okay-concatstringssep.exp rename to tests/functional/lang/eval-okay-concatstringssep.exp diff --git a/tests/lang/eval-okay-concatstringssep.nix b/tests/functional/lang/eval-okay-concatstringssep.nix similarity index 100% rename from tests/lang/eval-okay-concatstringssep.nix rename to tests/functional/lang/eval-okay-concatstringssep.nix diff --git a/tests/lang/eval-okay-context-introspection.exp b/tests/functional/lang/eval-okay-context-introspection.exp similarity index 100% rename from tests/lang/eval-okay-context-introspection.exp rename to tests/functional/lang/eval-okay-context-introspection.exp diff --git a/tests/lang/eval-okay-context-introspection.nix b/tests/functional/lang/eval-okay-context-introspection.nix similarity index 100% rename from tests/lang/eval-okay-context-introspection.nix rename to tests/functional/lang/eval-okay-context-introspection.nix diff --git a/tests/lang/eval-okay-context.exp b/tests/functional/lang/eval-okay-context.exp similarity index 100% rename from tests/lang/eval-okay-context.exp rename to tests/functional/lang/eval-okay-context.exp diff --git a/tests/lang/eval-okay-context.nix b/tests/functional/lang/eval-okay-context.nix similarity index 100% rename from tests/lang/eval-okay-context.nix rename to tests/functional/lang/eval-okay-context.nix diff --git a/tests/lang/eval-okay-curpos.exp b/tests/functional/lang/eval-okay-curpos.exp similarity index 100% rename from tests/lang/eval-okay-curpos.exp rename to tests/functional/lang/eval-okay-curpos.exp diff --git a/tests/lang/eval-okay-curpos.nix b/tests/functional/lang/eval-okay-curpos.nix similarity index 100% rename from tests/lang/eval-okay-curpos.nix rename to tests/functional/lang/eval-okay-curpos.nix diff --git a/tests/lang/eval-okay-deepseq.exp b/tests/functional/lang/eval-okay-deepseq.exp similarity index 100% rename from tests/lang/eval-okay-deepseq.exp rename to tests/functional/lang/eval-okay-deepseq.exp diff --git a/tests/lang/eval-okay-deepseq.nix b/tests/functional/lang/eval-okay-deepseq.nix similarity index 100% rename from tests/lang/eval-okay-deepseq.nix rename to tests/functional/lang/eval-okay-deepseq.nix diff --git a/tests/lang/eval-okay-delayed-with-inherit.exp b/tests/functional/lang/eval-okay-delayed-with-inherit.exp similarity index 100% rename from tests/lang/eval-okay-delayed-with-inherit.exp rename to tests/functional/lang/eval-okay-delayed-with-inherit.exp diff --git a/tests/lang/eval-okay-delayed-with-inherit.nix b/tests/functional/lang/eval-okay-delayed-with-inherit.nix similarity index 100% rename from tests/lang/eval-okay-delayed-with-inherit.nix rename to tests/functional/lang/eval-okay-delayed-with-inherit.nix diff --git a/tests/lang/eval-okay-delayed-with.exp b/tests/functional/lang/eval-okay-delayed-with.exp similarity index 100% rename from tests/lang/eval-okay-delayed-with.exp rename to tests/functional/lang/eval-okay-delayed-with.exp diff --git a/tests/lang/eval-okay-delayed-with.nix b/tests/functional/lang/eval-okay-delayed-with.nix similarity index 100% rename from tests/lang/eval-okay-delayed-with.nix rename to tests/functional/lang/eval-okay-delayed-with.nix diff --git a/tests/lang/eval-okay-dynamic-attrs-2.exp b/tests/functional/lang/eval-okay-dynamic-attrs-2.exp similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs-2.exp rename to tests/functional/lang/eval-okay-dynamic-attrs-2.exp diff --git a/tests/lang/eval-okay-dynamic-attrs-2.nix b/tests/functional/lang/eval-okay-dynamic-attrs-2.nix similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs-2.nix rename to tests/functional/lang/eval-okay-dynamic-attrs-2.nix diff --git a/tests/lang/eval-okay-dynamic-attrs-bare.exp b/tests/functional/lang/eval-okay-dynamic-attrs-bare.exp similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs-bare.exp rename to tests/functional/lang/eval-okay-dynamic-attrs-bare.exp diff --git a/tests/lang/eval-okay-dynamic-attrs-bare.nix b/tests/functional/lang/eval-okay-dynamic-attrs-bare.nix similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs-bare.nix rename to tests/functional/lang/eval-okay-dynamic-attrs-bare.nix diff --git a/tests/lang/eval-okay-dynamic-attrs.exp b/tests/functional/lang/eval-okay-dynamic-attrs.exp similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs.exp rename to tests/functional/lang/eval-okay-dynamic-attrs.exp diff --git a/tests/lang/eval-okay-dynamic-attrs.nix b/tests/functional/lang/eval-okay-dynamic-attrs.nix similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs.nix rename to tests/functional/lang/eval-okay-dynamic-attrs.nix diff --git a/tests/lang/eval-okay-elem.exp b/tests/functional/lang/eval-okay-elem.exp similarity index 100% rename from tests/lang/eval-okay-elem.exp rename to tests/functional/lang/eval-okay-elem.exp diff --git a/tests/lang/eval-okay-elem.nix b/tests/functional/lang/eval-okay-elem.nix similarity index 100% rename from tests/lang/eval-okay-elem.nix rename to tests/functional/lang/eval-okay-elem.nix diff --git a/tests/lang/eval-okay-empty-args.exp b/tests/functional/lang/eval-okay-empty-args.exp similarity index 100% rename from tests/lang/eval-okay-empty-args.exp rename to tests/functional/lang/eval-okay-empty-args.exp diff --git a/tests/lang/eval-okay-empty-args.nix b/tests/functional/lang/eval-okay-empty-args.nix similarity index 100% rename from tests/lang/eval-okay-empty-args.nix rename to tests/functional/lang/eval-okay-empty-args.nix diff --git a/tests/lang/eval-okay-eq-derivations.exp b/tests/functional/lang/eval-okay-eq-derivations.exp similarity index 100% rename from tests/lang/eval-okay-eq-derivations.exp rename to tests/functional/lang/eval-okay-eq-derivations.exp diff --git a/tests/lang/eval-okay-eq-derivations.nix b/tests/functional/lang/eval-okay-eq-derivations.nix similarity index 100% rename from tests/lang/eval-okay-eq-derivations.nix rename to tests/functional/lang/eval-okay-eq-derivations.nix diff --git a/tests/lang/eval-okay-eq.exp b/tests/functional/lang/eval-okay-eq.exp similarity index 100% rename from tests/lang/eval-okay-eq.exp rename to tests/functional/lang/eval-okay-eq.exp diff --git a/tests/lang/eval-okay-eq.nix b/tests/functional/lang/eval-okay-eq.nix similarity index 100% rename from tests/lang/eval-okay-eq.nix rename to tests/functional/lang/eval-okay-eq.nix diff --git a/tests/lang/eval-okay-filter.exp b/tests/functional/lang/eval-okay-filter.exp similarity index 100% rename from tests/lang/eval-okay-filter.exp rename to tests/functional/lang/eval-okay-filter.exp diff --git a/tests/lang/eval-okay-filter.nix b/tests/functional/lang/eval-okay-filter.nix similarity index 100% rename from tests/lang/eval-okay-filter.nix rename to tests/functional/lang/eval-okay-filter.nix diff --git a/tests/lang/eval-okay-flake-ref-to-string.exp b/tests/functional/lang/eval-okay-flake-ref-to-string.exp similarity index 100% rename from tests/lang/eval-okay-flake-ref-to-string.exp rename to tests/functional/lang/eval-okay-flake-ref-to-string.exp diff --git a/tests/lang/eval-okay-flake-ref-to-string.nix b/tests/functional/lang/eval-okay-flake-ref-to-string.nix similarity index 100% rename from tests/lang/eval-okay-flake-ref-to-string.nix rename to tests/functional/lang/eval-okay-flake-ref-to-string.nix diff --git a/tests/lang/eval-okay-flatten.exp b/tests/functional/lang/eval-okay-flatten.exp similarity index 100% rename from tests/lang/eval-okay-flatten.exp rename to tests/functional/lang/eval-okay-flatten.exp diff --git a/tests/lang/eval-okay-flatten.nix b/tests/functional/lang/eval-okay-flatten.nix similarity index 100% rename from tests/lang/eval-okay-flatten.nix rename to tests/functional/lang/eval-okay-flatten.nix diff --git a/tests/lang/eval-okay-float.exp b/tests/functional/lang/eval-okay-float.exp similarity index 100% rename from tests/lang/eval-okay-float.exp rename to tests/functional/lang/eval-okay-float.exp diff --git a/tests/lang/eval-okay-float.nix b/tests/functional/lang/eval-okay-float.nix similarity index 100% rename from tests/lang/eval-okay-float.nix rename to tests/functional/lang/eval-okay-float.nix diff --git a/tests/lang/eval-okay-floor-ceil.exp b/tests/functional/lang/eval-okay-floor-ceil.exp similarity index 100% rename from tests/lang/eval-okay-floor-ceil.exp rename to tests/functional/lang/eval-okay-floor-ceil.exp diff --git a/tests/lang/eval-okay-floor-ceil.nix b/tests/functional/lang/eval-okay-floor-ceil.nix similarity index 100% rename from tests/lang/eval-okay-floor-ceil.nix rename to tests/functional/lang/eval-okay-floor-ceil.nix diff --git a/tests/lang/eval-okay-foldlStrict-lazy-elements.exp b/tests/functional/lang/eval-okay-foldlStrict-lazy-elements.exp similarity index 100% rename from tests/lang/eval-okay-foldlStrict-lazy-elements.exp rename to tests/functional/lang/eval-okay-foldlStrict-lazy-elements.exp diff --git a/tests/lang/eval-okay-foldlStrict-lazy-elements.nix b/tests/functional/lang/eval-okay-foldlStrict-lazy-elements.nix similarity index 100% rename from tests/lang/eval-okay-foldlStrict-lazy-elements.nix rename to tests/functional/lang/eval-okay-foldlStrict-lazy-elements.nix diff --git a/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp b/tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp similarity index 100% rename from tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp rename to tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp diff --git a/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix b/tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix similarity index 100% rename from tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix rename to tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix diff --git a/tests/lang/eval-okay-foldlStrict.exp b/tests/functional/lang/eval-okay-foldlStrict.exp similarity index 100% rename from tests/lang/eval-okay-foldlStrict.exp rename to tests/functional/lang/eval-okay-foldlStrict.exp diff --git a/tests/lang/eval-okay-foldlStrict.nix b/tests/functional/lang/eval-okay-foldlStrict.nix similarity index 100% rename from tests/lang/eval-okay-foldlStrict.nix rename to tests/functional/lang/eval-okay-foldlStrict.nix diff --git a/tests/lang/eval-okay-fromTOML-timestamps.exp b/tests/functional/lang/eval-okay-fromTOML-timestamps.exp similarity index 100% rename from tests/lang/eval-okay-fromTOML-timestamps.exp rename to tests/functional/lang/eval-okay-fromTOML-timestamps.exp diff --git a/tests/lang/eval-okay-fromTOML-timestamps.flags b/tests/functional/lang/eval-okay-fromTOML-timestamps.flags similarity index 100% rename from tests/lang/eval-okay-fromTOML-timestamps.flags rename to tests/functional/lang/eval-okay-fromTOML-timestamps.flags diff --git a/tests/lang/eval-okay-fromTOML-timestamps.nix b/tests/functional/lang/eval-okay-fromTOML-timestamps.nix similarity index 100% rename from tests/lang/eval-okay-fromTOML-timestamps.nix rename to tests/functional/lang/eval-okay-fromTOML-timestamps.nix diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/functional/lang/eval-okay-fromTOML.exp similarity index 100% rename from tests/lang/eval-okay-fromTOML.exp rename to tests/functional/lang/eval-okay-fromTOML.exp diff --git a/tests/lang/eval-okay-fromTOML.nix b/tests/functional/lang/eval-okay-fromTOML.nix similarity index 100% rename from tests/lang/eval-okay-fromTOML.nix rename to tests/functional/lang/eval-okay-fromTOML.nix diff --git a/tests/lang/eval-okay-fromjson-escapes.exp b/tests/functional/lang/eval-okay-fromjson-escapes.exp similarity index 100% rename from tests/lang/eval-okay-fromjson-escapes.exp rename to tests/functional/lang/eval-okay-fromjson-escapes.exp diff --git a/tests/lang/eval-okay-fromjson-escapes.nix b/tests/functional/lang/eval-okay-fromjson-escapes.nix similarity index 100% rename from tests/lang/eval-okay-fromjson-escapes.nix rename to tests/functional/lang/eval-okay-fromjson-escapes.nix diff --git a/tests/lang/eval-okay-fromjson.exp b/tests/functional/lang/eval-okay-fromjson.exp similarity index 100% rename from tests/lang/eval-okay-fromjson.exp rename to tests/functional/lang/eval-okay-fromjson.exp diff --git a/tests/lang/eval-okay-fromjson.nix b/tests/functional/lang/eval-okay-fromjson.nix similarity index 100% rename from tests/lang/eval-okay-fromjson.nix rename to tests/functional/lang/eval-okay-fromjson.nix diff --git a/tests/lang/eval-okay-functionargs.exp b/tests/functional/lang/eval-okay-functionargs.exp similarity index 100% rename from tests/lang/eval-okay-functionargs.exp rename to tests/functional/lang/eval-okay-functionargs.exp diff --git a/tests/lang/eval-okay-functionargs.exp.xml b/tests/functional/lang/eval-okay-functionargs.exp.xml similarity index 100% rename from tests/lang/eval-okay-functionargs.exp.xml rename to tests/functional/lang/eval-okay-functionargs.exp.xml diff --git a/tests/lang/eval-okay-functionargs.nix b/tests/functional/lang/eval-okay-functionargs.nix similarity index 100% rename from tests/lang/eval-okay-functionargs.nix rename to tests/functional/lang/eval-okay-functionargs.nix diff --git a/tests/lang/eval-okay-getattrpos-functionargs.exp b/tests/functional/lang/eval-okay-getattrpos-functionargs.exp similarity index 100% rename from tests/lang/eval-okay-getattrpos-functionargs.exp rename to tests/functional/lang/eval-okay-getattrpos-functionargs.exp diff --git a/tests/lang/eval-okay-getattrpos-functionargs.nix b/tests/functional/lang/eval-okay-getattrpos-functionargs.nix similarity index 100% rename from tests/lang/eval-okay-getattrpos-functionargs.nix rename to tests/functional/lang/eval-okay-getattrpos-functionargs.nix diff --git a/tests/lang/eval-okay-getattrpos-undefined.exp b/tests/functional/lang/eval-okay-getattrpos-undefined.exp similarity index 100% rename from tests/lang/eval-okay-getattrpos-undefined.exp rename to tests/functional/lang/eval-okay-getattrpos-undefined.exp diff --git a/tests/lang/eval-okay-getattrpos-undefined.nix b/tests/functional/lang/eval-okay-getattrpos-undefined.nix similarity index 100% rename from tests/lang/eval-okay-getattrpos-undefined.nix rename to tests/functional/lang/eval-okay-getattrpos-undefined.nix diff --git a/tests/lang/eval-okay-getattrpos.exp b/tests/functional/lang/eval-okay-getattrpos.exp similarity index 100% rename from tests/lang/eval-okay-getattrpos.exp rename to tests/functional/lang/eval-okay-getattrpos.exp diff --git a/tests/lang/eval-okay-getattrpos.nix b/tests/functional/lang/eval-okay-getattrpos.nix similarity index 100% rename from tests/lang/eval-okay-getattrpos.nix rename to tests/functional/lang/eval-okay-getattrpos.nix diff --git a/tests/lang/eval-okay-getenv.exp b/tests/functional/lang/eval-okay-getenv.exp similarity index 100% rename from tests/lang/eval-okay-getenv.exp rename to tests/functional/lang/eval-okay-getenv.exp diff --git a/tests/lang/eval-okay-getenv.nix b/tests/functional/lang/eval-okay-getenv.nix similarity index 100% rename from tests/lang/eval-okay-getenv.nix rename to tests/functional/lang/eval-okay-getenv.nix diff --git a/tests/lang/eval-okay-groupBy.exp b/tests/functional/lang/eval-okay-groupBy.exp similarity index 100% rename from tests/lang/eval-okay-groupBy.exp rename to tests/functional/lang/eval-okay-groupBy.exp diff --git a/tests/lang/eval-okay-groupBy.nix b/tests/functional/lang/eval-okay-groupBy.nix similarity index 100% rename from tests/lang/eval-okay-groupBy.nix rename to tests/functional/lang/eval-okay-groupBy.nix diff --git a/tests/lang/eval-okay-hash.exp b/tests/functional/lang/eval-okay-hash.exp similarity index 100% rename from tests/lang/eval-okay-hash.exp rename to tests/functional/lang/eval-okay-hash.exp diff --git a/tests/lang/eval-okay-hashfile.exp b/tests/functional/lang/eval-okay-hashfile.exp similarity index 100% rename from tests/lang/eval-okay-hashfile.exp rename to tests/functional/lang/eval-okay-hashfile.exp diff --git a/tests/lang/eval-okay-hashfile.nix b/tests/functional/lang/eval-okay-hashfile.nix similarity index 100% rename from tests/lang/eval-okay-hashfile.nix rename to tests/functional/lang/eval-okay-hashfile.nix diff --git a/tests/lang/eval-okay-hashstring.exp b/tests/functional/lang/eval-okay-hashstring.exp similarity index 100% rename from tests/lang/eval-okay-hashstring.exp rename to tests/functional/lang/eval-okay-hashstring.exp diff --git a/tests/lang/eval-okay-hashstring.nix b/tests/functional/lang/eval-okay-hashstring.nix similarity index 100% rename from tests/lang/eval-okay-hashstring.nix rename to tests/functional/lang/eval-okay-hashstring.nix diff --git a/tests/lang/eval-okay-if.exp b/tests/functional/lang/eval-okay-if.exp similarity index 100% rename from tests/lang/eval-okay-if.exp rename to tests/functional/lang/eval-okay-if.exp diff --git a/tests/lang/eval-okay-if.nix b/tests/functional/lang/eval-okay-if.nix similarity index 100% rename from tests/lang/eval-okay-if.nix rename to tests/functional/lang/eval-okay-if.nix diff --git a/tests/lang/eval-okay-import.exp b/tests/functional/lang/eval-okay-import.exp similarity index 100% rename from tests/lang/eval-okay-import.exp rename to tests/functional/lang/eval-okay-import.exp diff --git a/tests/lang/eval-okay-import.nix b/tests/functional/lang/eval-okay-import.nix similarity index 100% rename from tests/lang/eval-okay-import.nix rename to tests/functional/lang/eval-okay-import.nix diff --git a/tests/lang/eval-okay-ind-string.exp b/tests/functional/lang/eval-okay-ind-string.exp similarity index 100% rename from tests/lang/eval-okay-ind-string.exp rename to tests/functional/lang/eval-okay-ind-string.exp diff --git a/tests/lang/eval-okay-ind-string.nix b/tests/functional/lang/eval-okay-ind-string.nix similarity index 100% rename from tests/lang/eval-okay-ind-string.nix rename to tests/functional/lang/eval-okay-ind-string.nix diff --git a/tests/lang/eval-okay-intersectAttrs.exp b/tests/functional/lang/eval-okay-intersectAttrs.exp similarity index 100% rename from tests/lang/eval-okay-intersectAttrs.exp rename to tests/functional/lang/eval-okay-intersectAttrs.exp diff --git a/tests/lang/eval-okay-intersectAttrs.nix b/tests/functional/lang/eval-okay-intersectAttrs.nix similarity index 100% rename from tests/lang/eval-okay-intersectAttrs.nix rename to tests/functional/lang/eval-okay-intersectAttrs.nix diff --git a/tests/lang/eval-okay-let.exp b/tests/functional/lang/eval-okay-let.exp similarity index 100% rename from tests/lang/eval-okay-let.exp rename to tests/functional/lang/eval-okay-let.exp diff --git a/tests/lang/eval-okay-let.nix b/tests/functional/lang/eval-okay-let.nix similarity index 100% rename from tests/lang/eval-okay-let.nix rename to tests/functional/lang/eval-okay-let.nix diff --git a/tests/lang/eval-okay-list.exp b/tests/functional/lang/eval-okay-list.exp similarity index 100% rename from tests/lang/eval-okay-list.exp rename to tests/functional/lang/eval-okay-list.exp diff --git a/tests/lang/eval-okay-list.nix b/tests/functional/lang/eval-okay-list.nix similarity index 100% rename from tests/lang/eval-okay-list.nix rename to tests/functional/lang/eval-okay-list.nix diff --git a/tests/lang/eval-okay-listtoattrs.exp b/tests/functional/lang/eval-okay-listtoattrs.exp similarity index 100% rename from tests/lang/eval-okay-listtoattrs.exp rename to tests/functional/lang/eval-okay-listtoattrs.exp diff --git a/tests/lang/eval-okay-listtoattrs.nix b/tests/functional/lang/eval-okay-listtoattrs.nix similarity index 100% rename from tests/lang/eval-okay-listtoattrs.nix rename to tests/functional/lang/eval-okay-listtoattrs.nix diff --git a/tests/lang/eval-okay-logic.exp b/tests/functional/lang/eval-okay-logic.exp similarity index 100% rename from tests/lang/eval-okay-logic.exp rename to tests/functional/lang/eval-okay-logic.exp diff --git a/tests/lang/eval-okay-logic.nix b/tests/functional/lang/eval-okay-logic.nix similarity index 100% rename from tests/lang/eval-okay-logic.nix rename to tests/functional/lang/eval-okay-logic.nix diff --git a/tests/lang/eval-okay-map.exp b/tests/functional/lang/eval-okay-map.exp similarity index 100% rename from tests/lang/eval-okay-map.exp rename to tests/functional/lang/eval-okay-map.exp diff --git a/tests/lang/eval-okay-map.nix b/tests/functional/lang/eval-okay-map.nix similarity index 100% rename from tests/lang/eval-okay-map.nix rename to tests/functional/lang/eval-okay-map.nix diff --git a/tests/lang/eval-okay-mapattrs.exp b/tests/functional/lang/eval-okay-mapattrs.exp similarity index 100% rename from tests/lang/eval-okay-mapattrs.exp rename to tests/functional/lang/eval-okay-mapattrs.exp diff --git a/tests/lang/eval-okay-mapattrs.nix b/tests/functional/lang/eval-okay-mapattrs.nix similarity index 100% rename from tests/lang/eval-okay-mapattrs.nix rename to tests/functional/lang/eval-okay-mapattrs.nix diff --git a/tests/lang/eval-okay-merge-dynamic-attrs.exp b/tests/functional/lang/eval-okay-merge-dynamic-attrs.exp similarity index 100% rename from tests/lang/eval-okay-merge-dynamic-attrs.exp rename to tests/functional/lang/eval-okay-merge-dynamic-attrs.exp diff --git a/tests/lang/eval-okay-merge-dynamic-attrs.nix b/tests/functional/lang/eval-okay-merge-dynamic-attrs.nix similarity index 100% rename from tests/lang/eval-okay-merge-dynamic-attrs.nix rename to tests/functional/lang/eval-okay-merge-dynamic-attrs.nix diff --git a/tests/lang/eval-okay-nested-with.exp b/tests/functional/lang/eval-okay-nested-with.exp similarity index 100% rename from tests/lang/eval-okay-nested-with.exp rename to tests/functional/lang/eval-okay-nested-with.exp diff --git a/tests/lang/eval-okay-nested-with.nix b/tests/functional/lang/eval-okay-nested-with.nix similarity index 100% rename from tests/lang/eval-okay-nested-with.nix rename to tests/functional/lang/eval-okay-nested-with.nix diff --git a/tests/lang/eval-okay-new-let.exp b/tests/functional/lang/eval-okay-new-let.exp similarity index 100% rename from tests/lang/eval-okay-new-let.exp rename to tests/functional/lang/eval-okay-new-let.exp diff --git a/tests/lang/eval-okay-new-let.nix b/tests/functional/lang/eval-okay-new-let.nix similarity index 100% rename from tests/lang/eval-okay-new-let.nix rename to tests/functional/lang/eval-okay-new-let.nix diff --git a/tests/lang/eval-okay-null-dynamic-attrs.exp b/tests/functional/lang/eval-okay-null-dynamic-attrs.exp similarity index 100% rename from tests/lang/eval-okay-null-dynamic-attrs.exp rename to tests/functional/lang/eval-okay-null-dynamic-attrs.exp diff --git a/tests/lang/eval-okay-null-dynamic-attrs.nix b/tests/functional/lang/eval-okay-null-dynamic-attrs.nix similarity index 100% rename from tests/lang/eval-okay-null-dynamic-attrs.nix rename to tests/functional/lang/eval-okay-null-dynamic-attrs.nix diff --git a/tests/lang/eval-okay-overrides.exp b/tests/functional/lang/eval-okay-overrides.exp similarity index 100% rename from tests/lang/eval-okay-overrides.exp rename to tests/functional/lang/eval-okay-overrides.exp diff --git a/tests/lang/eval-okay-overrides.nix b/tests/functional/lang/eval-okay-overrides.nix similarity index 100% rename from tests/lang/eval-okay-overrides.nix rename to tests/functional/lang/eval-okay-overrides.nix diff --git a/tests/lang/eval-okay-parse-flake-ref.exp b/tests/functional/lang/eval-okay-parse-flake-ref.exp similarity index 100% rename from tests/lang/eval-okay-parse-flake-ref.exp rename to tests/functional/lang/eval-okay-parse-flake-ref.exp diff --git a/tests/lang/eval-okay-parse-flake-ref.nix b/tests/functional/lang/eval-okay-parse-flake-ref.nix similarity index 100% rename from tests/lang/eval-okay-parse-flake-ref.nix rename to tests/functional/lang/eval-okay-parse-flake-ref.nix diff --git a/tests/lang/eval-okay-partition.exp b/tests/functional/lang/eval-okay-partition.exp similarity index 100% rename from tests/lang/eval-okay-partition.exp rename to tests/functional/lang/eval-okay-partition.exp diff --git a/tests/lang/eval-okay-partition.nix b/tests/functional/lang/eval-okay-partition.nix similarity index 100% rename from tests/lang/eval-okay-partition.nix rename to tests/functional/lang/eval-okay-partition.nix diff --git a/tests/lang/eval-okay-path-string-interpolation.exp b/tests/functional/lang/eval-okay-path-string-interpolation.exp similarity index 100% rename from tests/lang/eval-okay-path-string-interpolation.exp rename to tests/functional/lang/eval-okay-path-string-interpolation.exp diff --git a/tests/lang/eval-okay-path-string-interpolation.nix b/tests/functional/lang/eval-okay-path-string-interpolation.nix similarity index 100% rename from tests/lang/eval-okay-path-string-interpolation.nix rename to tests/functional/lang/eval-okay-path-string-interpolation.nix diff --git a/tests/lang/eval-okay-path.exp b/tests/functional/lang/eval-okay-path.exp similarity index 100% rename from tests/lang/eval-okay-path.exp rename to tests/functional/lang/eval-okay-path.exp diff --git a/tests/lang/eval-okay-path.nix b/tests/functional/lang/eval-okay-path.nix similarity index 100% rename from tests/lang/eval-okay-path.nix rename to tests/functional/lang/eval-okay-path.nix diff --git a/tests/lang/eval-okay-pathexists.exp b/tests/functional/lang/eval-okay-pathexists.exp similarity index 100% rename from tests/lang/eval-okay-pathexists.exp rename to tests/functional/lang/eval-okay-pathexists.exp diff --git a/tests/lang/eval-okay-pathexists.nix b/tests/functional/lang/eval-okay-pathexists.nix similarity index 100% rename from tests/lang/eval-okay-pathexists.nix rename to tests/functional/lang/eval-okay-pathexists.nix diff --git a/tests/lang/eval-okay-patterns.exp b/tests/functional/lang/eval-okay-patterns.exp similarity index 100% rename from tests/lang/eval-okay-patterns.exp rename to tests/functional/lang/eval-okay-patterns.exp diff --git a/tests/lang/eval-okay-patterns.nix b/tests/functional/lang/eval-okay-patterns.nix similarity index 100% rename from tests/lang/eval-okay-patterns.nix rename to tests/functional/lang/eval-okay-patterns.nix diff --git a/tests/lang/eval-okay-print.err.exp b/tests/functional/lang/eval-okay-print.err.exp similarity index 100% rename from tests/lang/eval-okay-print.err.exp rename to tests/functional/lang/eval-okay-print.err.exp diff --git a/tests/lang/eval-okay-print.exp b/tests/functional/lang/eval-okay-print.exp similarity index 100% rename from tests/lang/eval-okay-print.exp rename to tests/functional/lang/eval-okay-print.exp diff --git a/tests/lang/eval-okay-print.nix b/tests/functional/lang/eval-okay-print.nix similarity index 100% rename from tests/lang/eval-okay-print.nix rename to tests/functional/lang/eval-okay-print.nix diff --git a/tests/lang/eval-okay-readDir.exp b/tests/functional/lang/eval-okay-readDir.exp similarity index 100% rename from tests/lang/eval-okay-readDir.exp rename to tests/functional/lang/eval-okay-readDir.exp diff --git a/tests/lang/eval-okay-readDir.nix b/tests/functional/lang/eval-okay-readDir.nix similarity index 100% rename from tests/lang/eval-okay-readDir.nix rename to tests/functional/lang/eval-okay-readDir.nix diff --git a/tests/lang/eval-okay-readFileType.exp b/tests/functional/lang/eval-okay-readFileType.exp similarity index 100% rename from tests/lang/eval-okay-readFileType.exp rename to tests/functional/lang/eval-okay-readFileType.exp diff --git a/tests/lang/eval-okay-readFileType.nix b/tests/functional/lang/eval-okay-readFileType.nix similarity index 100% rename from tests/lang/eval-okay-readFileType.nix rename to tests/functional/lang/eval-okay-readFileType.nix diff --git a/tests/lang/eval-okay-readfile.exp b/tests/functional/lang/eval-okay-readfile.exp similarity index 100% rename from tests/lang/eval-okay-readfile.exp rename to tests/functional/lang/eval-okay-readfile.exp diff --git a/tests/lang/eval-okay-readfile.nix b/tests/functional/lang/eval-okay-readfile.nix similarity index 100% rename from tests/lang/eval-okay-readfile.nix rename to tests/functional/lang/eval-okay-readfile.nix diff --git a/tests/lang/eval-okay-redefine-builtin.exp b/tests/functional/lang/eval-okay-redefine-builtin.exp similarity index 100% rename from tests/lang/eval-okay-redefine-builtin.exp rename to tests/functional/lang/eval-okay-redefine-builtin.exp diff --git a/tests/lang/eval-okay-redefine-builtin.nix b/tests/functional/lang/eval-okay-redefine-builtin.nix similarity index 100% rename from tests/lang/eval-okay-redefine-builtin.nix rename to tests/functional/lang/eval-okay-redefine-builtin.nix diff --git a/tests/lang/eval-okay-regex-match.exp b/tests/functional/lang/eval-okay-regex-match.exp similarity index 100% rename from tests/lang/eval-okay-regex-match.exp rename to tests/functional/lang/eval-okay-regex-match.exp diff --git a/tests/lang/eval-okay-regex-match.nix b/tests/functional/lang/eval-okay-regex-match.nix similarity index 100% rename from tests/lang/eval-okay-regex-match.nix rename to tests/functional/lang/eval-okay-regex-match.nix diff --git a/tests/lang/eval-okay-regex-split.exp b/tests/functional/lang/eval-okay-regex-split.exp similarity index 100% rename from tests/lang/eval-okay-regex-split.exp rename to tests/functional/lang/eval-okay-regex-split.exp diff --git a/tests/lang/eval-okay-regex-split.nix b/tests/functional/lang/eval-okay-regex-split.nix similarity index 100% rename from tests/lang/eval-okay-regex-split.nix rename to tests/functional/lang/eval-okay-regex-split.nix diff --git a/tests/lang/eval-okay-regression-20220122.exp b/tests/functional/lang/eval-okay-regression-20220122.exp similarity index 100% rename from tests/lang/eval-okay-regression-20220122.exp rename to tests/functional/lang/eval-okay-regression-20220122.exp diff --git a/tests/lang/eval-okay-regression-20220122.nix b/tests/functional/lang/eval-okay-regression-20220122.nix similarity index 100% rename from tests/lang/eval-okay-regression-20220122.nix rename to tests/functional/lang/eval-okay-regression-20220122.nix diff --git a/tests/lang/eval-okay-regression-20220125.exp b/tests/functional/lang/eval-okay-regression-20220125.exp similarity index 100% rename from tests/lang/eval-okay-regression-20220125.exp rename to tests/functional/lang/eval-okay-regression-20220125.exp diff --git a/tests/lang/eval-okay-regression-20220125.nix b/tests/functional/lang/eval-okay-regression-20220125.nix similarity index 100% rename from tests/lang/eval-okay-regression-20220125.nix rename to tests/functional/lang/eval-okay-regression-20220125.nix diff --git a/tests/lang/eval-okay-remove.exp b/tests/functional/lang/eval-okay-remove.exp similarity index 100% rename from tests/lang/eval-okay-remove.exp rename to tests/functional/lang/eval-okay-remove.exp diff --git a/tests/lang/eval-okay-remove.nix b/tests/functional/lang/eval-okay-remove.nix similarity index 100% rename from tests/lang/eval-okay-remove.nix rename to tests/functional/lang/eval-okay-remove.nix diff --git a/tests/lang/eval-okay-replacestrings.exp b/tests/functional/lang/eval-okay-replacestrings.exp similarity index 100% rename from tests/lang/eval-okay-replacestrings.exp rename to tests/functional/lang/eval-okay-replacestrings.exp diff --git a/tests/lang/eval-okay-replacestrings.nix b/tests/functional/lang/eval-okay-replacestrings.nix similarity index 100% rename from tests/lang/eval-okay-replacestrings.nix rename to tests/functional/lang/eval-okay-replacestrings.nix diff --git a/tests/lang/eval-okay-scope-1.exp b/tests/functional/lang/eval-okay-scope-1.exp similarity index 100% rename from tests/lang/eval-okay-scope-1.exp rename to tests/functional/lang/eval-okay-scope-1.exp diff --git a/tests/lang/eval-okay-scope-1.nix b/tests/functional/lang/eval-okay-scope-1.nix similarity index 100% rename from tests/lang/eval-okay-scope-1.nix rename to tests/functional/lang/eval-okay-scope-1.nix diff --git a/tests/lang/eval-okay-scope-2.exp b/tests/functional/lang/eval-okay-scope-2.exp similarity index 100% rename from tests/lang/eval-okay-scope-2.exp rename to tests/functional/lang/eval-okay-scope-2.exp diff --git a/tests/lang/eval-okay-scope-2.nix b/tests/functional/lang/eval-okay-scope-2.nix similarity index 100% rename from tests/lang/eval-okay-scope-2.nix rename to tests/functional/lang/eval-okay-scope-2.nix diff --git a/tests/lang/eval-okay-scope-3.exp b/tests/functional/lang/eval-okay-scope-3.exp similarity index 100% rename from tests/lang/eval-okay-scope-3.exp rename to tests/functional/lang/eval-okay-scope-3.exp diff --git a/tests/lang/eval-okay-scope-3.nix b/tests/functional/lang/eval-okay-scope-3.nix similarity index 100% rename from tests/lang/eval-okay-scope-3.nix rename to tests/functional/lang/eval-okay-scope-3.nix diff --git a/tests/lang/eval-okay-scope-4.exp b/tests/functional/lang/eval-okay-scope-4.exp similarity index 100% rename from tests/lang/eval-okay-scope-4.exp rename to tests/functional/lang/eval-okay-scope-4.exp diff --git a/tests/lang/eval-okay-scope-4.nix b/tests/functional/lang/eval-okay-scope-4.nix similarity index 100% rename from tests/lang/eval-okay-scope-4.nix rename to tests/functional/lang/eval-okay-scope-4.nix diff --git a/tests/lang/eval-okay-scope-6.exp b/tests/functional/lang/eval-okay-scope-6.exp similarity index 100% rename from tests/lang/eval-okay-scope-6.exp rename to tests/functional/lang/eval-okay-scope-6.exp diff --git a/tests/lang/eval-okay-scope-6.nix b/tests/functional/lang/eval-okay-scope-6.nix similarity index 100% rename from tests/lang/eval-okay-scope-6.nix rename to tests/functional/lang/eval-okay-scope-6.nix diff --git a/tests/lang/eval-okay-scope-7.exp b/tests/functional/lang/eval-okay-scope-7.exp similarity index 100% rename from tests/lang/eval-okay-scope-7.exp rename to tests/functional/lang/eval-okay-scope-7.exp diff --git a/tests/lang/eval-okay-scope-7.nix b/tests/functional/lang/eval-okay-scope-7.nix similarity index 100% rename from tests/lang/eval-okay-scope-7.nix rename to tests/functional/lang/eval-okay-scope-7.nix diff --git a/tests/lang/eval-okay-search-path.exp b/tests/functional/lang/eval-okay-search-path.exp similarity index 100% rename from tests/lang/eval-okay-search-path.exp rename to tests/functional/lang/eval-okay-search-path.exp diff --git a/tests/lang/eval-okay-search-path.flags b/tests/functional/lang/eval-okay-search-path.flags similarity index 100% rename from tests/lang/eval-okay-search-path.flags rename to tests/functional/lang/eval-okay-search-path.flags diff --git a/tests/lang/eval-okay-search-path.nix b/tests/functional/lang/eval-okay-search-path.nix similarity index 100% rename from tests/lang/eval-okay-search-path.nix rename to tests/functional/lang/eval-okay-search-path.nix diff --git a/tests/lang/eval-okay-seq.exp b/tests/functional/lang/eval-okay-seq.exp similarity index 100% rename from tests/lang/eval-okay-seq.exp rename to tests/functional/lang/eval-okay-seq.exp diff --git a/tests/lang/eval-okay-seq.nix b/tests/functional/lang/eval-okay-seq.nix similarity index 100% rename from tests/lang/eval-okay-seq.nix rename to tests/functional/lang/eval-okay-seq.nix diff --git a/tests/lang/eval-okay-sort.exp b/tests/functional/lang/eval-okay-sort.exp similarity index 100% rename from tests/lang/eval-okay-sort.exp rename to tests/functional/lang/eval-okay-sort.exp diff --git a/tests/lang/eval-okay-sort.nix b/tests/functional/lang/eval-okay-sort.nix similarity index 100% rename from tests/lang/eval-okay-sort.nix rename to tests/functional/lang/eval-okay-sort.nix diff --git a/tests/lang/eval-okay-splitversion.exp b/tests/functional/lang/eval-okay-splitversion.exp similarity index 100% rename from tests/lang/eval-okay-splitversion.exp rename to tests/functional/lang/eval-okay-splitversion.exp diff --git a/tests/lang/eval-okay-splitversion.nix b/tests/functional/lang/eval-okay-splitversion.nix similarity index 100% rename from tests/lang/eval-okay-splitversion.nix rename to tests/functional/lang/eval-okay-splitversion.nix diff --git a/tests/lang/eval-okay-string.exp b/tests/functional/lang/eval-okay-string.exp similarity index 100% rename from tests/lang/eval-okay-string.exp rename to tests/functional/lang/eval-okay-string.exp diff --git a/tests/lang/eval-okay-string.nix b/tests/functional/lang/eval-okay-string.nix similarity index 100% rename from tests/lang/eval-okay-string.nix rename to tests/functional/lang/eval-okay-string.nix diff --git a/tests/lang/eval-okay-strings-as-attrs-names.exp b/tests/functional/lang/eval-okay-strings-as-attrs-names.exp similarity index 100% rename from tests/lang/eval-okay-strings-as-attrs-names.exp rename to tests/functional/lang/eval-okay-strings-as-attrs-names.exp diff --git a/tests/lang/eval-okay-strings-as-attrs-names.nix b/tests/functional/lang/eval-okay-strings-as-attrs-names.nix similarity index 100% rename from tests/lang/eval-okay-strings-as-attrs-names.nix rename to tests/functional/lang/eval-okay-strings-as-attrs-names.nix diff --git a/tests/lang/eval-okay-substring.exp b/tests/functional/lang/eval-okay-substring.exp similarity index 100% rename from tests/lang/eval-okay-substring.exp rename to tests/functional/lang/eval-okay-substring.exp diff --git a/tests/lang/eval-okay-substring.nix b/tests/functional/lang/eval-okay-substring.nix similarity index 100% rename from tests/lang/eval-okay-substring.nix rename to tests/functional/lang/eval-okay-substring.nix diff --git a/tests/lang/eval-okay-tail-call-1.exp-disabled b/tests/functional/lang/eval-okay-tail-call-1.exp-disabled similarity index 100% rename from tests/lang/eval-okay-tail-call-1.exp-disabled rename to tests/functional/lang/eval-okay-tail-call-1.exp-disabled diff --git a/tests/lang/eval-okay-tail-call-1.nix b/tests/functional/lang/eval-okay-tail-call-1.nix similarity index 100% rename from tests/lang/eval-okay-tail-call-1.nix rename to tests/functional/lang/eval-okay-tail-call-1.nix diff --git a/tests/lang/eval-okay-tojson.exp b/tests/functional/lang/eval-okay-tojson.exp similarity index 100% rename from tests/lang/eval-okay-tojson.exp rename to tests/functional/lang/eval-okay-tojson.exp diff --git a/tests/lang/eval-okay-tojson.nix b/tests/functional/lang/eval-okay-tojson.nix similarity index 100% rename from tests/lang/eval-okay-tojson.nix rename to tests/functional/lang/eval-okay-tojson.nix diff --git a/tests/lang/eval-okay-toxml.exp b/tests/functional/lang/eval-okay-toxml.exp similarity index 100% rename from tests/lang/eval-okay-toxml.exp rename to tests/functional/lang/eval-okay-toxml.exp diff --git a/tests/lang/eval-okay-toxml.nix b/tests/functional/lang/eval-okay-toxml.nix similarity index 100% rename from tests/lang/eval-okay-toxml.nix rename to tests/functional/lang/eval-okay-toxml.nix diff --git a/tests/lang/eval-okay-toxml2.exp b/tests/functional/lang/eval-okay-toxml2.exp similarity index 100% rename from tests/lang/eval-okay-toxml2.exp rename to tests/functional/lang/eval-okay-toxml2.exp diff --git a/tests/lang/eval-okay-toxml2.nix b/tests/functional/lang/eval-okay-toxml2.nix similarity index 100% rename from tests/lang/eval-okay-toxml2.nix rename to tests/functional/lang/eval-okay-toxml2.nix diff --git a/tests/lang/eval-okay-tryeval.exp b/tests/functional/lang/eval-okay-tryeval.exp similarity index 100% rename from tests/lang/eval-okay-tryeval.exp rename to tests/functional/lang/eval-okay-tryeval.exp diff --git a/tests/lang/eval-okay-tryeval.nix b/tests/functional/lang/eval-okay-tryeval.nix similarity index 100% rename from tests/lang/eval-okay-tryeval.nix rename to tests/functional/lang/eval-okay-tryeval.nix diff --git a/tests/lang/eval-okay-types.exp b/tests/functional/lang/eval-okay-types.exp similarity index 100% rename from tests/lang/eval-okay-types.exp rename to tests/functional/lang/eval-okay-types.exp diff --git a/tests/lang/eval-okay-types.nix b/tests/functional/lang/eval-okay-types.nix similarity index 100% rename from tests/lang/eval-okay-types.nix rename to tests/functional/lang/eval-okay-types.nix diff --git a/tests/lang/eval-okay-versions.exp b/tests/functional/lang/eval-okay-versions.exp similarity index 100% rename from tests/lang/eval-okay-versions.exp rename to tests/functional/lang/eval-okay-versions.exp diff --git a/tests/lang/eval-okay-versions.nix b/tests/functional/lang/eval-okay-versions.nix similarity index 100% rename from tests/lang/eval-okay-versions.nix rename to tests/functional/lang/eval-okay-versions.nix diff --git a/tests/lang/eval-okay-with.exp b/tests/functional/lang/eval-okay-with.exp similarity index 100% rename from tests/lang/eval-okay-with.exp rename to tests/functional/lang/eval-okay-with.exp diff --git a/tests/lang/eval-okay-with.nix b/tests/functional/lang/eval-okay-with.nix similarity index 100% rename from tests/lang/eval-okay-with.nix rename to tests/functional/lang/eval-okay-with.nix diff --git a/tests/lang/eval-okay-xml.exp.xml b/tests/functional/lang/eval-okay-xml.exp.xml similarity index 100% rename from tests/lang/eval-okay-xml.exp.xml rename to tests/functional/lang/eval-okay-xml.exp.xml diff --git a/tests/lang/eval-okay-xml.nix b/tests/functional/lang/eval-okay-xml.nix similarity index 100% rename from tests/lang/eval-okay-xml.nix rename to tests/functional/lang/eval-okay-xml.nix diff --git a/tests/lang/eval-okay-zipAttrsWith.exp b/tests/functional/lang/eval-okay-zipAttrsWith.exp similarity index 100% rename from tests/lang/eval-okay-zipAttrsWith.exp rename to tests/functional/lang/eval-okay-zipAttrsWith.exp diff --git a/tests/lang/eval-okay-zipAttrsWith.nix b/tests/functional/lang/eval-okay-zipAttrsWith.nix similarity index 100% rename from tests/lang/eval-okay-zipAttrsWith.nix rename to tests/functional/lang/eval-okay-zipAttrsWith.nix diff --git a/tests/lang/framework.sh b/tests/functional/lang/framework.sh similarity index 100% rename from tests/lang/framework.sh rename to tests/functional/lang/framework.sh diff --git a/tests/lang/imported.nix b/tests/functional/lang/imported.nix similarity index 100% rename from tests/lang/imported.nix rename to tests/functional/lang/imported.nix diff --git a/tests/lang/imported2.nix b/tests/functional/lang/imported2.nix similarity index 100% rename from tests/lang/imported2.nix rename to tests/functional/lang/imported2.nix diff --git a/tests/lang/lib.nix b/tests/functional/lang/lib.nix similarity index 100% rename from tests/lang/lib.nix rename to tests/functional/lang/lib.nix diff --git a/tests/lang/parse-fail-dup-attrs-1.err.exp b/tests/functional/lang/parse-fail-dup-attrs-1.err.exp similarity index 100% rename from tests/lang/parse-fail-dup-attrs-1.err.exp rename to tests/functional/lang/parse-fail-dup-attrs-1.err.exp diff --git a/tests/lang/parse-fail-dup-attrs-1.nix b/tests/functional/lang/parse-fail-dup-attrs-1.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-1.nix rename to tests/functional/lang/parse-fail-dup-attrs-1.nix diff --git a/tests/lang/parse-fail-dup-attrs-2.err.exp b/tests/functional/lang/parse-fail-dup-attrs-2.err.exp similarity index 100% rename from tests/lang/parse-fail-dup-attrs-2.err.exp rename to tests/functional/lang/parse-fail-dup-attrs-2.err.exp diff --git a/tests/lang/parse-fail-dup-attrs-2.nix b/tests/functional/lang/parse-fail-dup-attrs-2.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-2.nix rename to tests/functional/lang/parse-fail-dup-attrs-2.nix diff --git a/tests/lang/parse-fail-dup-attrs-3.err.exp b/tests/functional/lang/parse-fail-dup-attrs-3.err.exp similarity index 100% rename from tests/lang/parse-fail-dup-attrs-3.err.exp rename to tests/functional/lang/parse-fail-dup-attrs-3.err.exp diff --git a/tests/lang/parse-fail-dup-attrs-3.nix b/tests/functional/lang/parse-fail-dup-attrs-3.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-3.nix rename to tests/functional/lang/parse-fail-dup-attrs-3.nix diff --git a/tests/lang/parse-fail-dup-attrs-4.err.exp b/tests/functional/lang/parse-fail-dup-attrs-4.err.exp similarity index 100% rename from tests/lang/parse-fail-dup-attrs-4.err.exp rename to tests/functional/lang/parse-fail-dup-attrs-4.err.exp diff --git a/tests/lang/parse-fail-dup-attrs-4.nix b/tests/functional/lang/parse-fail-dup-attrs-4.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-4.nix rename to tests/functional/lang/parse-fail-dup-attrs-4.nix diff --git a/tests/lang/parse-fail-dup-attrs-6.err.exp b/tests/functional/lang/parse-fail-dup-attrs-6.err.exp similarity index 100% rename from tests/lang/parse-fail-dup-attrs-6.err.exp rename to tests/functional/lang/parse-fail-dup-attrs-6.err.exp diff --git a/tests/lang/parse-fail-dup-attrs-7.err.exp b/tests/functional/lang/parse-fail-dup-attrs-7.err.exp similarity index 100% rename from tests/lang/parse-fail-dup-attrs-7.err.exp rename to tests/functional/lang/parse-fail-dup-attrs-7.err.exp diff --git a/tests/lang/parse-fail-dup-attrs-7.nix b/tests/functional/lang/parse-fail-dup-attrs-7.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-7.nix rename to tests/functional/lang/parse-fail-dup-attrs-7.nix diff --git a/tests/lang/parse-fail-dup-formals.err.exp b/tests/functional/lang/parse-fail-dup-formals.err.exp similarity index 100% rename from tests/lang/parse-fail-dup-formals.err.exp rename to tests/functional/lang/parse-fail-dup-formals.err.exp diff --git a/tests/lang/parse-fail-dup-formals.nix b/tests/functional/lang/parse-fail-dup-formals.nix similarity index 100% rename from tests/lang/parse-fail-dup-formals.nix rename to tests/functional/lang/parse-fail-dup-formals.nix diff --git a/tests/lang/parse-fail-eof-in-string.err.exp b/tests/functional/lang/parse-fail-eof-in-string.err.exp similarity index 100% rename from tests/lang/parse-fail-eof-in-string.err.exp rename to tests/functional/lang/parse-fail-eof-in-string.err.exp diff --git a/tests/lang/parse-fail-eof-in-string.nix b/tests/functional/lang/parse-fail-eof-in-string.nix similarity index 100% rename from tests/lang/parse-fail-eof-in-string.nix rename to tests/functional/lang/parse-fail-eof-in-string.nix diff --git a/tests/lang/parse-fail-mixed-nested-attrs1.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp similarity index 100% rename from tests/lang/parse-fail-mixed-nested-attrs1.err.exp rename to tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp diff --git a/tests/lang/parse-fail-mixed-nested-attrs1.nix b/tests/functional/lang/parse-fail-mixed-nested-attrs1.nix similarity index 100% rename from tests/lang/parse-fail-mixed-nested-attrs1.nix rename to tests/functional/lang/parse-fail-mixed-nested-attrs1.nix diff --git a/tests/lang/parse-fail-mixed-nested-attrs2.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp similarity index 100% rename from tests/lang/parse-fail-mixed-nested-attrs2.err.exp rename to tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp diff --git a/tests/lang/parse-fail-mixed-nested-attrs2.nix b/tests/functional/lang/parse-fail-mixed-nested-attrs2.nix similarity index 100% rename from tests/lang/parse-fail-mixed-nested-attrs2.nix rename to tests/functional/lang/parse-fail-mixed-nested-attrs2.nix diff --git a/tests/lang/parse-fail-patterns-1.err.exp b/tests/functional/lang/parse-fail-patterns-1.err.exp similarity index 100% rename from tests/lang/parse-fail-patterns-1.err.exp rename to tests/functional/lang/parse-fail-patterns-1.err.exp diff --git a/tests/lang/parse-fail-patterns-1.nix b/tests/functional/lang/parse-fail-patterns-1.nix similarity index 100% rename from tests/lang/parse-fail-patterns-1.nix rename to tests/functional/lang/parse-fail-patterns-1.nix diff --git a/tests/lang/parse-fail-regression-20060610.err.exp b/tests/functional/lang/parse-fail-regression-20060610.err.exp similarity index 100% rename from tests/lang/parse-fail-regression-20060610.err.exp rename to tests/functional/lang/parse-fail-regression-20060610.err.exp diff --git a/tests/lang/parse-fail-regression-20060610.nix b/tests/functional/lang/parse-fail-regression-20060610.nix similarity index 100% rename from tests/lang/parse-fail-regression-20060610.nix rename to tests/functional/lang/parse-fail-regression-20060610.nix diff --git a/tests/lang/parse-fail-undef-var-2.err.exp b/tests/functional/lang/parse-fail-undef-var-2.err.exp similarity index 100% rename from tests/lang/parse-fail-undef-var-2.err.exp rename to tests/functional/lang/parse-fail-undef-var-2.err.exp diff --git a/tests/lang/parse-fail-undef-var-2.nix b/tests/functional/lang/parse-fail-undef-var-2.nix similarity index 100% rename from tests/lang/parse-fail-undef-var-2.nix rename to tests/functional/lang/parse-fail-undef-var-2.nix diff --git a/tests/lang/parse-fail-undef-var.err.exp b/tests/functional/lang/parse-fail-undef-var.err.exp similarity index 100% rename from tests/lang/parse-fail-undef-var.err.exp rename to tests/functional/lang/parse-fail-undef-var.err.exp diff --git a/tests/lang/parse-fail-undef-var.nix b/tests/functional/lang/parse-fail-undef-var.nix similarity index 100% rename from tests/lang/parse-fail-undef-var.nix rename to tests/functional/lang/parse-fail-undef-var.nix diff --git a/tests/lang/parse-fail-utf8.err.exp b/tests/functional/lang/parse-fail-utf8.err.exp similarity index 100% rename from tests/lang/parse-fail-utf8.err.exp rename to tests/functional/lang/parse-fail-utf8.err.exp diff --git a/tests/lang/parse-fail-utf8.nix b/tests/functional/lang/parse-fail-utf8.nix similarity index 100% rename from tests/lang/parse-fail-utf8.nix rename to tests/functional/lang/parse-fail-utf8.nix diff --git a/tests/lang/parse-okay-1.exp b/tests/functional/lang/parse-okay-1.exp similarity index 100% rename from tests/lang/parse-okay-1.exp rename to tests/functional/lang/parse-okay-1.exp diff --git a/tests/lang/parse-okay-1.nix b/tests/functional/lang/parse-okay-1.nix similarity index 100% rename from tests/lang/parse-okay-1.nix rename to tests/functional/lang/parse-okay-1.nix diff --git a/tests/lang/parse-okay-crlf.exp b/tests/functional/lang/parse-okay-crlf.exp similarity index 100% rename from tests/lang/parse-okay-crlf.exp rename to tests/functional/lang/parse-okay-crlf.exp diff --git a/tests/lang/parse-okay-crlf.nix b/tests/functional/lang/parse-okay-crlf.nix similarity index 100% rename from tests/lang/parse-okay-crlf.nix rename to tests/functional/lang/parse-okay-crlf.nix diff --git a/tests/lang/parse-okay-dup-attrs-5.exp b/tests/functional/lang/parse-okay-dup-attrs-5.exp similarity index 100% rename from tests/lang/parse-okay-dup-attrs-5.exp rename to tests/functional/lang/parse-okay-dup-attrs-5.exp diff --git a/tests/lang/parse-okay-dup-attrs-5.nix b/tests/functional/lang/parse-okay-dup-attrs-5.nix similarity index 100% rename from tests/lang/parse-okay-dup-attrs-5.nix rename to tests/functional/lang/parse-okay-dup-attrs-5.nix diff --git a/tests/lang/parse-okay-dup-attrs-6.exp b/tests/functional/lang/parse-okay-dup-attrs-6.exp similarity index 100% rename from tests/lang/parse-okay-dup-attrs-6.exp rename to tests/functional/lang/parse-okay-dup-attrs-6.exp diff --git a/tests/lang/parse-okay-dup-attrs-6.nix b/tests/functional/lang/parse-okay-dup-attrs-6.nix similarity index 100% rename from tests/lang/parse-okay-dup-attrs-6.nix rename to tests/functional/lang/parse-okay-dup-attrs-6.nix diff --git a/tests/lang/parse-okay-mixed-nested-attrs-1.exp b/tests/functional/lang/parse-okay-mixed-nested-attrs-1.exp similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-1.exp rename to tests/functional/lang/parse-okay-mixed-nested-attrs-1.exp diff --git a/tests/lang/parse-okay-mixed-nested-attrs-1.nix b/tests/functional/lang/parse-okay-mixed-nested-attrs-1.nix similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-1.nix rename to tests/functional/lang/parse-okay-mixed-nested-attrs-1.nix diff --git a/tests/lang/parse-okay-mixed-nested-attrs-2.exp b/tests/functional/lang/parse-okay-mixed-nested-attrs-2.exp similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-2.exp rename to tests/functional/lang/parse-okay-mixed-nested-attrs-2.exp diff --git a/tests/lang/parse-okay-mixed-nested-attrs-2.nix b/tests/functional/lang/parse-okay-mixed-nested-attrs-2.nix similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-2.nix rename to tests/functional/lang/parse-okay-mixed-nested-attrs-2.nix diff --git a/tests/lang/parse-okay-mixed-nested-attrs-3.exp b/tests/functional/lang/parse-okay-mixed-nested-attrs-3.exp similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-3.exp rename to tests/functional/lang/parse-okay-mixed-nested-attrs-3.exp diff --git a/tests/lang/parse-okay-mixed-nested-attrs-3.nix b/tests/functional/lang/parse-okay-mixed-nested-attrs-3.nix similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-3.nix rename to tests/functional/lang/parse-okay-mixed-nested-attrs-3.nix diff --git a/tests/lang/parse-okay-regression-20041027.exp b/tests/functional/lang/parse-okay-regression-20041027.exp similarity index 100% rename from tests/lang/parse-okay-regression-20041027.exp rename to tests/functional/lang/parse-okay-regression-20041027.exp diff --git a/tests/lang/parse-okay-regression-20041027.nix b/tests/functional/lang/parse-okay-regression-20041027.nix similarity index 100% rename from tests/lang/parse-okay-regression-20041027.nix rename to tests/functional/lang/parse-okay-regression-20041027.nix diff --git a/tests/lang/parse-okay-regression-751.exp b/tests/functional/lang/parse-okay-regression-751.exp similarity index 100% rename from tests/lang/parse-okay-regression-751.exp rename to tests/functional/lang/parse-okay-regression-751.exp diff --git a/tests/lang/parse-okay-regression-751.nix b/tests/functional/lang/parse-okay-regression-751.nix similarity index 100% rename from tests/lang/parse-okay-regression-751.nix rename to tests/functional/lang/parse-okay-regression-751.nix diff --git a/tests/lang/parse-okay-subversion.exp b/tests/functional/lang/parse-okay-subversion.exp similarity index 100% rename from tests/lang/parse-okay-subversion.exp rename to tests/functional/lang/parse-okay-subversion.exp diff --git a/tests/lang/parse-okay-subversion.nix b/tests/functional/lang/parse-okay-subversion.nix similarity index 100% rename from tests/lang/parse-okay-subversion.nix rename to tests/functional/lang/parse-okay-subversion.nix diff --git a/tests/lang/parse-okay-url.exp b/tests/functional/lang/parse-okay-url.exp similarity index 100% rename from tests/lang/parse-okay-url.exp rename to tests/functional/lang/parse-okay-url.exp diff --git a/tests/lang/parse-okay-url.nix b/tests/functional/lang/parse-okay-url.nix similarity index 100% rename from tests/lang/parse-okay-url.nix rename to tests/functional/lang/parse-okay-url.nix diff --git a/tests/lang/readDir/bar b/tests/functional/lang/readDir/bar similarity index 100% rename from tests/lang/readDir/bar rename to tests/functional/lang/readDir/bar diff --git a/tests/lang/readDir/foo/git-hates-directories b/tests/functional/lang/readDir/foo/git-hates-directories similarity index 100% rename from tests/lang/readDir/foo/git-hates-directories rename to tests/functional/lang/readDir/foo/git-hates-directories diff --git a/tests/lang/readDir/ldir b/tests/functional/lang/readDir/ldir similarity index 100% rename from tests/lang/readDir/ldir rename to tests/functional/lang/readDir/ldir diff --git a/tests/lang/readDir/linked b/tests/functional/lang/readDir/linked similarity index 100% rename from tests/lang/readDir/linked rename to tests/functional/lang/readDir/linked diff --git a/tests/legacy-ssh-store.sh b/tests/functional/legacy-ssh-store.sh similarity index 100% rename from tests/legacy-ssh-store.sh rename to tests/functional/legacy-ssh-store.sh diff --git a/tests/linux-sandbox-cert-test.nix b/tests/functional/linux-sandbox-cert-test.nix similarity index 100% rename from tests/linux-sandbox-cert-test.nix rename to tests/functional/linux-sandbox-cert-test.nix diff --git a/tests/linux-sandbox.sh b/tests/functional/linux-sandbox.sh similarity index 100% rename from tests/linux-sandbox.sh rename to tests/functional/linux-sandbox.sh diff --git a/tests/local-store.sh b/tests/functional/local-store.sh similarity index 100% rename from tests/local-store.sh rename to tests/functional/local-store.sh diff --git a/tests/local.mk b/tests/functional/local.mk similarity index 90% rename from tests/local.mk rename to tests/functional/local.mk index ac418dc0d..4d29f60a1 100644 --- a/tests/local.mk +++ b/tests/functional/local.mk @@ -139,15 +139,16 @@ ifeq ($(HAVE_LOCAL_NIX_BUILD), 1) endif endif -tests/test-libstoreconsumer.sh.test: tests/test-libstoreconsumer/test-libstoreconsumer -tests/plugins.sh: tests/plugins/libplugintest.$(SO_EXT) +$(d)/test-libstoreconsumer.sh.test $(d)/test-libstoreconsumer.sh.test-debug: \ + $(d)/test-libstoreconsumer/test-libstoreconsumer +$(d)/plugins.sh.test $(d)/plugins.sh.test-debug: \ + $(d)/plugins/libplugintest.$(SO_EXT) install-tests += $(foreach x, $(nix_tests), $(d)/$(x)) -clean-files += \ +test-clean-files := \ $(d)/common/vars-and-functions.sh \ $(d)/config.nix -test-deps += \ - tests/common/vars-and-functions.sh \ - tests/config.nix +clean-files += $(test-clean-files) +test-deps += $(test-clean-files) diff --git a/tests/logging.sh b/tests/functional/logging.sh similarity index 100% rename from tests/logging.sh rename to tests/functional/logging.sh diff --git a/tests/misc.sh b/tests/functional/misc.sh similarity index 100% rename from tests/misc.sh rename to tests/functional/misc.sh diff --git a/tests/multiple-outputs.nix b/tests/functional/multiple-outputs.nix similarity index 100% rename from tests/multiple-outputs.nix rename to tests/functional/multiple-outputs.nix diff --git a/tests/multiple-outputs.sh b/tests/functional/multiple-outputs.sh similarity index 100% rename from tests/multiple-outputs.sh rename to tests/functional/multiple-outputs.sh diff --git a/tests/nar-access.nix b/tests/functional/nar-access.nix similarity index 100% rename from tests/nar-access.nix rename to tests/functional/nar-access.nix diff --git a/tests/nar-access.sh b/tests/functional/nar-access.sh similarity index 100% rename from tests/nar-access.sh rename to tests/functional/nar-access.sh diff --git a/tests/nested-sandboxing.sh b/tests/functional/nested-sandboxing.sh similarity index 75% rename from tests/nested-sandboxing.sh rename to tests/functional/nested-sandboxing.sh index d9fa788aa..61fe043c6 100644 --- a/tests/nested-sandboxing.sh +++ b/tests/functional/nested-sandboxing.sh @@ -1,5 +1,5 @@ source common.sh -# This test is run by `tests/nested-sandboxing/runner.nix` in an extra layer of sandboxing. +# This test is run by `tests/functional/nested-sandboxing/runner.nix` in an extra layer of sandboxing. [[ -d /nix/store ]] || skipTest "running this test without Nix's deps being drawn from /nix/store is not yet supported" requireSandboxSupport diff --git a/tests/nested-sandboxing/command.sh b/tests/functional/nested-sandboxing/command.sh similarity index 100% rename from tests/nested-sandboxing/command.sh rename to tests/functional/nested-sandboxing/command.sh diff --git a/tests/nested-sandboxing/runner.nix b/tests/functional/nested-sandboxing/runner.nix similarity index 100% rename from tests/nested-sandboxing/runner.nix rename to tests/functional/nested-sandboxing/runner.nix diff --git a/tests/nix-build-examples.nix b/tests/functional/nix-build-examples.nix similarity index 100% rename from tests/nix-build-examples.nix rename to tests/functional/nix-build-examples.nix diff --git a/tests/nix-build.sh b/tests/functional/nix-build.sh similarity index 100% rename from tests/nix-build.sh rename to tests/functional/nix-build.sh diff --git a/tests/nix-channel.sh b/tests/functional/nix-channel.sh similarity index 100% rename from tests/nix-channel.sh rename to tests/functional/nix-channel.sh diff --git a/tests/nix-collect-garbage-d.sh b/tests/functional/nix-collect-garbage-d.sh similarity index 100% rename from tests/nix-collect-garbage-d.sh rename to tests/functional/nix-collect-garbage-d.sh diff --git a/tests/nix-copy-ssh-ng.sh b/tests/functional/nix-copy-ssh-ng.sh similarity index 100% rename from tests/nix-copy-ssh-ng.sh rename to tests/functional/nix-copy-ssh-ng.sh diff --git a/tests/nix-copy-ssh.sh b/tests/functional/nix-copy-ssh.sh similarity index 100% rename from tests/nix-copy-ssh.sh rename to tests/functional/nix-copy-ssh.sh diff --git a/tests/nix-daemon-untrusting.sh b/tests/functional/nix-daemon-untrusting.sh similarity index 100% rename from tests/nix-daemon-untrusting.sh rename to tests/functional/nix-daemon-untrusting.sh diff --git a/tests/nix-profile.sh b/tests/functional/nix-profile.sh similarity index 100% rename from tests/nix-profile.sh rename to tests/functional/nix-profile.sh diff --git a/tests/nix-shell.sh b/tests/functional/nix-shell.sh similarity index 100% rename from tests/nix-shell.sh rename to tests/functional/nix-shell.sh diff --git a/tests/nix_path.sh b/tests/functional/nix_path.sh similarity index 100% rename from tests/nix_path.sh rename to tests/functional/nix_path.sh diff --git a/tests/optimise-store.sh b/tests/functional/optimise-store.sh similarity index 100% rename from tests/optimise-store.sh rename to tests/functional/optimise-store.sh diff --git a/tests/output-normalization.sh b/tests/functional/output-normalization.sh similarity index 100% rename from tests/output-normalization.sh rename to tests/functional/output-normalization.sh diff --git a/tests/parallel.builder.sh b/tests/functional/parallel.builder.sh similarity index 100% rename from tests/parallel.builder.sh rename to tests/functional/parallel.builder.sh diff --git a/tests/parallel.nix b/tests/functional/parallel.nix similarity index 100% rename from tests/parallel.nix rename to tests/functional/parallel.nix diff --git a/tests/parallel.sh b/tests/functional/parallel.sh similarity index 100% rename from tests/parallel.sh rename to tests/functional/parallel.sh diff --git a/tests/pass-as-file.sh b/tests/functional/pass-as-file.sh similarity index 100% rename from tests/pass-as-file.sh rename to tests/functional/pass-as-file.sh diff --git a/tests/path-from-hash-part.sh b/tests/functional/path-from-hash-part.sh similarity index 100% rename from tests/path-from-hash-part.sh rename to tests/functional/path-from-hash-part.sh diff --git a/tests/path.nix b/tests/functional/path.nix similarity index 100% rename from tests/path.nix rename to tests/functional/path.nix diff --git a/tests/placeholders.sh b/tests/functional/placeholders.sh similarity index 100% rename from tests/placeholders.sh rename to tests/functional/placeholders.sh diff --git a/tests/plugins.sh b/tests/functional/plugins.sh similarity index 100% rename from tests/plugins.sh rename to tests/functional/plugins.sh diff --git a/tests/plugins/local.mk b/tests/functional/plugins/local.mk similarity index 100% rename from tests/plugins/local.mk rename to tests/functional/plugins/local.mk diff --git a/tests/plugins/plugintest.cc b/tests/functional/plugins/plugintest.cc similarity index 100% rename from tests/plugins/plugintest.cc rename to tests/functional/plugins/plugintest.cc diff --git a/tests/post-hook.sh b/tests/functional/post-hook.sh similarity index 100% rename from tests/post-hook.sh rename to tests/functional/post-hook.sh diff --git a/tests/pure-eval.nix b/tests/functional/pure-eval.nix similarity index 100% rename from tests/pure-eval.nix rename to tests/functional/pure-eval.nix diff --git a/tests/pure-eval.sh b/tests/functional/pure-eval.sh similarity index 100% rename from tests/pure-eval.sh rename to tests/functional/pure-eval.sh diff --git a/tests/push-to-store-old.sh b/tests/functional/push-to-store-old.sh similarity index 100% rename from tests/push-to-store-old.sh rename to tests/functional/push-to-store-old.sh diff --git a/tests/push-to-store.sh b/tests/functional/push-to-store.sh similarity index 100% rename from tests/push-to-store.sh rename to tests/functional/push-to-store.sh diff --git a/tests/read-only-store.sh b/tests/functional/read-only-store.sh similarity index 100% rename from tests/read-only-store.sh rename to tests/functional/read-only-store.sh diff --git a/tests/readfile-context.nix b/tests/functional/readfile-context.nix similarity index 100% rename from tests/readfile-context.nix rename to tests/functional/readfile-context.nix diff --git a/tests/readfile-context.sh b/tests/functional/readfile-context.sh similarity index 100% rename from tests/readfile-context.sh rename to tests/functional/readfile-context.sh diff --git a/tests/recursive.nix b/tests/functional/recursive.nix similarity index 100% rename from tests/recursive.nix rename to tests/functional/recursive.nix diff --git a/tests/recursive.sh b/tests/functional/recursive.sh similarity index 100% rename from tests/recursive.sh rename to tests/functional/recursive.sh diff --git a/tests/referrers.sh b/tests/functional/referrers.sh similarity index 100% rename from tests/referrers.sh rename to tests/functional/referrers.sh diff --git a/tests/remote-store.sh b/tests/functional/remote-store.sh similarity index 100% rename from tests/remote-store.sh rename to tests/functional/remote-store.sh diff --git a/tests/repair.sh b/tests/functional/repair.sh similarity index 100% rename from tests/repair.sh rename to tests/functional/repair.sh diff --git a/tests/repl.sh b/tests/functional/repl.sh similarity index 100% rename from tests/repl.sh rename to tests/functional/repl.sh diff --git a/tests/restricted.nix b/tests/functional/restricted.nix similarity index 100% rename from tests/restricted.nix rename to tests/functional/restricted.nix diff --git a/tests/restricted.sh b/tests/functional/restricted.sh similarity index 95% rename from tests/restricted.sh rename to tests/functional/restricted.sh index 17f310a4b..197ae7a10 100644 --- a/tests/restricted.sh +++ b/tests/functional/restricted.sh @@ -9,10 +9,10 @@ nix-instantiate --restrict-eval ./simple.nix -I src=. nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh (! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix') -nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=.. +nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=../.. -(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel') -nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel' -I src=../src +(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel') +nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel' -I src=../../src (! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ') nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ' -I src=. diff --git a/tests/search.nix b/tests/functional/search.nix similarity index 100% rename from tests/search.nix rename to tests/functional/search.nix diff --git a/tests/search.sh b/tests/functional/search.sh similarity index 100% rename from tests/search.sh rename to tests/functional/search.sh diff --git a/tests/secure-drv-outputs.nix b/tests/functional/secure-drv-outputs.nix similarity index 100% rename from tests/secure-drv-outputs.nix rename to tests/functional/secure-drv-outputs.nix diff --git a/tests/secure-drv-outputs.sh b/tests/functional/secure-drv-outputs.sh similarity index 100% rename from tests/secure-drv-outputs.sh rename to tests/functional/secure-drv-outputs.sh diff --git a/tests/selfref-gc.sh b/tests/functional/selfref-gc.sh similarity index 100% rename from tests/selfref-gc.sh rename to tests/functional/selfref-gc.sh diff --git a/tests/shell-hello.nix b/tests/functional/shell-hello.nix similarity index 100% rename from tests/shell-hello.nix rename to tests/functional/shell-hello.nix diff --git a/tests/shell.nix b/tests/functional/shell.nix similarity index 100% rename from tests/shell.nix rename to tests/functional/shell.nix diff --git a/tests/shell.sh b/tests/functional/shell.sh similarity index 100% rename from tests/shell.sh rename to tests/functional/shell.sh diff --git a/tests/shell.shebang.rb b/tests/functional/shell.shebang.rb similarity index 100% rename from tests/shell.shebang.rb rename to tests/functional/shell.shebang.rb diff --git a/tests/shell.shebang.sh b/tests/functional/shell.shebang.sh similarity index 100% rename from tests/shell.shebang.sh rename to tests/functional/shell.shebang.sh diff --git a/tests/signing.sh b/tests/functional/signing.sh similarity index 100% rename from tests/signing.sh rename to tests/functional/signing.sh diff --git a/tests/simple-failing.nix b/tests/functional/simple-failing.nix similarity index 100% rename from tests/simple-failing.nix rename to tests/functional/simple-failing.nix diff --git a/tests/simple.builder.sh b/tests/functional/simple.builder.sh similarity index 100% rename from tests/simple.builder.sh rename to tests/functional/simple.builder.sh diff --git a/tests/simple.nix b/tests/functional/simple.nix similarity index 100% rename from tests/simple.nix rename to tests/functional/simple.nix diff --git a/tests/simple.sh b/tests/functional/simple.sh similarity index 100% rename from tests/simple.sh rename to tests/functional/simple.sh diff --git a/tests/ssh-relay.sh b/tests/functional/ssh-relay.sh similarity index 100% rename from tests/ssh-relay.sh rename to tests/functional/ssh-relay.sh diff --git a/tests/store-ping.sh b/tests/functional/store-ping.sh similarity index 100% rename from tests/store-ping.sh rename to tests/functional/store-ping.sh diff --git a/tests/structured-attrs-shell.nix b/tests/functional/structured-attrs-shell.nix similarity index 100% rename from tests/structured-attrs-shell.nix rename to tests/functional/structured-attrs-shell.nix diff --git a/tests/structured-attrs.nix b/tests/functional/structured-attrs.nix similarity index 100% rename from tests/structured-attrs.nix rename to tests/functional/structured-attrs.nix diff --git a/tests/structured-attrs.sh b/tests/functional/structured-attrs.sh similarity index 100% rename from tests/structured-attrs.sh rename to tests/functional/structured-attrs.sh diff --git a/tests/substitute-with-invalid-ca.sh b/tests/functional/substitute-with-invalid-ca.sh similarity index 100% rename from tests/substitute-with-invalid-ca.sh rename to tests/functional/substitute-with-invalid-ca.sh diff --git a/tests/suggestions.sh b/tests/functional/suggestions.sh similarity index 100% rename from tests/suggestions.sh rename to tests/functional/suggestions.sh diff --git a/tests/supplementary-groups.sh b/tests/functional/supplementary-groups.sh similarity index 100% rename from tests/supplementary-groups.sh rename to tests/functional/supplementary-groups.sh diff --git a/tests/tarball.sh b/tests/functional/tarball.sh similarity index 100% rename from tests/tarball.sh rename to tests/functional/tarball.sh diff --git a/tests/test-infra.sh b/tests/functional/test-infra.sh similarity index 100% rename from tests/test-infra.sh rename to tests/functional/test-infra.sh diff --git a/tests/test-libstoreconsumer.sh b/tests/functional/test-libstoreconsumer.sh similarity index 100% rename from tests/test-libstoreconsumer.sh rename to tests/functional/test-libstoreconsumer.sh diff --git a/tests/test-libstoreconsumer/README.md b/tests/functional/test-libstoreconsumer/README.md similarity index 100% rename from tests/test-libstoreconsumer/README.md rename to tests/functional/test-libstoreconsumer/README.md diff --git a/tests/test-libstoreconsumer/local.mk b/tests/functional/test-libstoreconsumer/local.mk similarity index 100% rename from tests/test-libstoreconsumer/local.mk rename to tests/functional/test-libstoreconsumer/local.mk diff --git a/tests/test-libstoreconsumer/main.cc b/tests/functional/test-libstoreconsumer/main.cc similarity index 100% rename from tests/test-libstoreconsumer/main.cc rename to tests/functional/test-libstoreconsumer/main.cc diff --git a/tests/timeout.nix b/tests/functional/timeout.nix similarity index 100% rename from tests/timeout.nix rename to tests/functional/timeout.nix diff --git a/tests/timeout.sh b/tests/functional/timeout.sh similarity index 100% rename from tests/timeout.sh rename to tests/functional/timeout.sh diff --git a/tests/toString-path.sh b/tests/functional/toString-path.sh similarity index 100% rename from tests/toString-path.sh rename to tests/functional/toString-path.sh diff --git a/tests/undefined-variable.nix b/tests/functional/undefined-variable.nix similarity index 100% rename from tests/undefined-variable.nix rename to tests/functional/undefined-variable.nix diff --git a/tests/user-envs-migration.sh b/tests/functional/user-envs-migration.sh similarity index 100% rename from tests/user-envs-migration.sh rename to tests/functional/user-envs-migration.sh diff --git a/tests/user-envs.builder.sh b/tests/functional/user-envs.builder.sh similarity index 100% rename from tests/user-envs.builder.sh rename to tests/functional/user-envs.builder.sh diff --git a/tests/user-envs.nix b/tests/functional/user-envs.nix similarity index 100% rename from tests/user-envs.nix rename to tests/functional/user-envs.nix diff --git a/tests/user-envs.sh b/tests/functional/user-envs.sh similarity index 100% rename from tests/user-envs.sh rename to tests/functional/user-envs.sh diff --git a/tests/why-depends.sh b/tests/functional/why-depends.sh similarity index 100% rename from tests/why-depends.sh rename to tests/functional/why-depends.sh diff --git a/tests/zstd.sh b/tests/functional/zstd.sh similarity index 100% rename from tests/zstd.sh rename to tests/functional/zstd.sh From 644ebaab5f96089bc3a8fd3873e8e7f2131a9074 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 6 Oct 2023 10:53:01 -0400 Subject: [PATCH 352/909] Define NixOS tests in `tests/nixos/default.nix` rather than `flake.nix` I think the our `flake.nix` is currently too large and too scary looking. I think this matters --- if Nix cannot dog-food itself in a way that is elegant, why should other people have confidence that their own code can be elegant and easy to maintain? We could do this at many points in time, but I think around now, when we are thinking about stabilizing parts of Flakes, is an especially good time. This is a first step to make the `flake.nix` smaller, and make individual components responsible for their own packaging. I hope we can do this many more follow-ups like it, until the top-level `flake.nix` is very small and just coordinates between other things. --- flake.nix | 74 ++++++++++++----------------------------- tests/nixos/default.nix | 41 +++++++++++++++++++++++ 2 files changed, 62 insertions(+), 53 deletions(-) create mode 100644 tests/nixos/default.nix diff --git a/flake.nix b/flake.nix index c331b2651..bec62fb95 100644 --- a/flake.nix +++ b/flake.nix @@ -509,18 +509,6 @@ }; }; - nixos-lib = import (nixpkgs + "/nixos/lib") { }; - - # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests - runNixOSTestFor = system: test: nixos-lib.runTest { - imports = [ test ]; - hostPkgs = nixpkgsFor.${system}.native; - defaults = { - nixpkgs.pkgs = nixpkgsFor.${system}.native; - }; - _module.args.nixpkgs = nixpkgs; - }; - in { # A Nixpkgs overlay that overrides the 'nix' and # 'nix.perl-bindings' packages. @@ -627,49 +615,29 @@ }; # System tests. - tests.authorization = runNixOSTestFor "x86_64-linux" ./tests/nixos/authorization.nix; + tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { - tests.remoteBuilds = runNixOSTestFor "x86_64-linux" ./tests/nixos/remote-builds.nix; + # Make sure that nix-env still produces the exact same result + # on a particular version of Nixpkgs. + evalNixpkgs = + with nixpkgsFor.x86_64-linux.native; + runCommand "eval-nixos" { buildInputs = [ nix ]; } + '' + type -p nix-env + # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. + time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages + [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]] + mkdir $out + ''; - tests.nix-copy-closure = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy-closure.nix; - - tests.nix-copy = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy.nix; - - tests.nssPreload = runNixOSTestFor "x86_64-linux" ./tests/nixos/nss-preload.nix; - - tests.githubFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/github-flakes.nix; - - tests.sourcehutFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/sourcehut-flakes.nix; - - tests.tarballFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/tarball-flakes.nix; - - tests.containers = runNixOSTestFor "x86_64-linux" ./tests/nixos/containers/containers.nix; - - tests.setuid = lib.genAttrs - ["i686-linux" "x86_64-linux"] - (system: runNixOSTestFor system ./tests/nixos/setuid.nix); - - - # Make sure that nix-env still produces the exact same result - # on a particular version of Nixpkgs. - tests.evalNixpkgs = - with nixpkgsFor.x86_64-linux.native; - runCommand "eval-nixos" { buildInputs = [ nix ]; } - '' - type -p nix-env - # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. - time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages - [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]] - mkdir $out - ''; - - tests.nixpkgsLibTests = - forAllSystems (system: - import (nixpkgs + "/lib/tests/release.nix") - { pkgs = nixpkgsFor.${system}.native; - nixVersions = [ self.packages.${system}.nix ]; - } - ); + nixpkgsLibTests = + forAllSystems (system: + import (nixpkgs + "/lib/tests/release.nix") + { pkgs = nixpkgsFor.${system}.native; + nixVersions = [ self.packages.${system}.nix ]; + } + ); + }; metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { pkgs = nixpkgsFor.x86_64-linux.native; diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix new file mode 100644 index 000000000..b391d7ef2 --- /dev/null +++ b/tests/nixos/default.nix @@ -0,0 +1,41 @@ +{ lib, nixpkgs, nixpkgsFor }: + +let + + nixos-lib = import (nixpkgs + "/nixos/lib") { }; + + # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests + runNixOSTestFor = system: test: nixos-lib.runTest { + imports = [ test ]; + hostPkgs = nixpkgsFor.${system}.native; + defaults = { + nixpkgs.pkgs = nixpkgsFor.${system}.native; + }; + _module.args.nixpkgs = nixpkgs; + }; + +in + +{ + authorization = runNixOSTestFor "x86_64-linux" ./authorization.nix; + + remoteBuilds = runNixOSTestFor "x86_64-linux" ./remote-builds.nix; + + nix-copy-closure = runNixOSTestFor "x86_64-linux" ./nix-copy-closure.nix; + + nix-copy = runNixOSTestFor "x86_64-linux" ./nix-copy.nix; + + nssPreload = runNixOSTestFor "x86_64-linux" ./nss-preload.nix; + + githubFlakes = runNixOSTestFor "x86_64-linux" ./github-flakes.nix; + + sourcehutFlakes = runNixOSTestFor "x86_64-linux" ./sourcehut-flakes.nix; + + tarballFlakes = runNixOSTestFor "x86_64-linux" ./tarball-flakes.nix; + + containers = runNixOSTestFor "x86_64-linux" ./containers/containers.nix; + + setuid = lib.genAttrs + ["i686-linux" "x86_64-linux"] + (system: runNixOSTestFor system ./setuid.nix); +} From 517c547dec086f6332292332571d05dd1cbb6f4b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 6 Oct 2023 23:34:08 +0200 Subject: [PATCH 353/909] remove duplicate redirects entry --- doc/manual/redirects.js | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index b2fad19bb..9d083a43d 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -336,7 +336,6 @@ const redirects = { "simple-values": "#primitives", "lists": "#list", "strings": "#string", - "lists": "#list", "attribute-sets": "#attribute-set", }, "installation/installing-binary.html": { From a67cee965a72f60da73713f47ee665a9ffe1873d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 7 Oct 2023 02:27:06 +0200 Subject: [PATCH 354/909] expand on interpolated expressions --- .../src/language/string-interpolation.md | 120 ++++++++++++++++-- doc/manual/src/language/values.md | 12 +- src/libexpr/primops.cc | 4 +- 3 files changed, 116 insertions(+), 20 deletions(-) diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index ddc6b8230..85e02df37 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -1,19 +1,12 @@ # String interpolation -String interpolation is a language feature where a [string], [path], or [attribute name] can contain expressions enclosed in `${ }` (dollar-sign with curly brackets). +String interpolation is a language feature where a [string], [path], or [attribute name][attribute set] can contain expressions enclosed in `${ }` (dollar-sign with curly brackets). -Such a string is an *interpolated string*, and an expression inside is an *interpolated expression*. - -Interpolated expressions must evaluate to one of the following: - -- a [string] -- a [path] -- a [derivation] +Such a construct is called *interpolated string*, and the expression inside is an [interpolated expression](#interpolated-expression). [string]: ./values.md#type-string [path]: ./values.md#type-path -[attribute name]: ./values.md#attribute-set -[derivation]: ../glossary.md#gloss-derivation +[attribute set]: ./values.md#attribute-set ## Examples @@ -80,3 +73,110 @@ let name = "foo"; in ``` { foo = "bar"; } + +# Interpolated expression + +An interpolated expression must evaluate to one of the following: + +- a [string] +- a [path] +- an [attribute set] that has a `__toString` attribute or an `outPath` attribute + + - `__toString` must be a function that takes the attribute set itself and returns a string + - `outPath` must be a string + + This includes [derivations](./derivations.md) or [flake inputs](@docroot@/command-ref/new-cli/nix3-flake.md#flake-inputs) (experimental). + +A string interpolates to itself. + +A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](../glossary.md#gloss-store-object). + +[store path]: ../glossary.md#gloss-store-path + +> **Example** +> +> ```console +> $ mkdir foo +> ``` +> +> Reference the empty directory in an interpolated expression: +> +> ```nix +> "${./foo}" +> ``` +> +> "/nix/store/2hhl2nz5v0khbn06ys82nrk99aa1xxdw-foo" + +A derivation interpolates to the [store path] of its first [output](./derivations.md#attr-outputs). + +> **Example** +> +> ```nix +> let +> pkgs = import {}; +> in +> "${pkgs.hello}" +> ``` +> +> "/nix/store/4xpfqf29z4m8vbhrqcz064wfmb46w5r7-hello-2.12.1" + +An attribute set interpolates to the return value of the function in the `__toString` applied to the attribute set itself. + +> **Example** +> +> ```nix +> let +> a = { +> value = 1; +> __toString = self: toString (self.value + 1); +> }; +> in +> "${a}" +> ``` +> +> "2" + +An attribute set also interpolates to the value of its `outPath` attribute. + +> **Example** +> +> ```nix +> let +> a = { outPath = "foo"; }; +> in +> "${a}" +> ``` +> +> "foo" + +If both `__toString` and `outPath` are present in an attribute set, `__toString` takes precedence. + +> **Example** +> +> ```nix +> let +> a = { __toString = _: "yes"; outPath = throw "no"; }; +> in +> "${a}" +> ``` +> +> "yes" + +If neither is present, an error is thrown. + +> **Example** +> +> ```nix +> let +> a = {}; +> in +> "${a}" +> ``` +> +> error: cannot coerce a set to a string +> +> at «string»:4:2: +> +> 3| in +> 4| "${a}" +> | ^ diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md index 2ae3e143a..fe0e2e4d5 100644 --- a/doc/manual/src/language/values.md +++ b/doc/manual/src/language/values.md @@ -112,18 +112,16 @@ environment variable `NIX_PATH` will be searched for the given file or directory name. - When an [interpolated string][string interpolation] evaluates to a path, the path is first copied into the Nix store and the resulting string is the [store path] of the newly created [store object]. - - [store path]: ../glossary.md#gloss-store-path - [store object]: ../glossary.md#gloss-store-object - For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` in the current directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. Note that the Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. For example, assume you used a file path in an interpolated string during a `nix repl` session. - Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new store path, since Nix might not re-read the file contents. + Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. - Paths themselves, except those in angle brackets (`< >`), support [string interpolation]. + [store path]: ../glossary.md#gloss-store-path + + Paths, except those in angle brackets (`< >`), support [string interpolation] and can be used in [interpolated expressions]. + [interpolated expressions]: ./string-interpolation.md#interpolated-expressions At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5de1b2828..1d04aeb2e 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -260,9 +260,7 @@ static RegisterPrimOp primop_import({ .doc = R"( Load, parse and return the Nix expression in the file *path*. - The value *path* can be a path, a string, or an attribute set with an - `__toString` attribute or a `outPath` attribute (as derivations or flake - inputs typically have). + The *path* argument must meet the same criteria as an [interpolated expression](@docroot@/language/string-interpolation.md#interpolated-expression). If *path* is a directory, the file `default.nix` in that directory is loaded. From a7ba8c3f4a69118c4c936fd8fd1415c0446803ca Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 7 Oct 2023 02:46:59 +0200 Subject: [PATCH 355/909] complete example on attribute name interpolation --- .../src/language/string-interpolation.md | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 85e02df37..2e650e348 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -63,16 +63,32 @@ you can instead write ### Attribute name -Attribute names can be created dynamically with string interpolation: + -```nix -let name = "foo"; in -{ - ${name} = "bar"; -} -``` +Attribute names can be interpolated strings. - { foo = "bar"; } +> **Example** +> +> ```nix +> let name = "foo"; in +> { ${name} = 123; } +> ``` +> +> { foo = 123; } + +Attributes can be selected with interpolated strings. + +> **Example** +> +> ```nix +> let name = "foo"; in +> { foo = 123; }.${name} +> ``` +> +> 123 # Interpolated expression From a86a3e5e59dd932d80431e9bbbb9ec3f883b2899 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 7 Oct 2023 03:12:21 +0200 Subject: [PATCH 356/909] document that pure-eval also disables `builtins.nixPath` --- src/libexpr/eval-settings.hh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 19ba679be..6a5c448b4 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -40,7 +40,10 @@ struct EvalSettings : Config Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state: - Restrict file system and network access to files specified by cryptographic hash - - Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) + - Disable impure constants: + - [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) + - [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) + - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath) )" }; From 2fe1ccf7973891cfbd921279a5f94879b2c7dce3 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 7 Oct 2023 03:20:20 +0200 Subject: [PATCH 357/909] describe the effect of `restrict-eval` in a more focused manner --- src/libexpr/eval-settings.hh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 19ba679be..2f534f496 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -29,10 +29,12 @@ struct EvalSettings : Config this, false, "restrict-eval", R"( If set to `true`, the Nix evaluator will not allow access to any - files outside of the Nix search path (as set via the `NIX_PATH` - environment variable or the `-I` option), or to URIs outside of - [`allowed-uris`](../command-ref/conf-file.md#conf-allowed-uris). - The default is `false`. + files outside of + [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath), + or to URIs outside of + [`allowed-uris`](@docroot@/command-ref/conf-file.md#conf-allowed-uris). + + Also the default value for [`nix-path`](#conf-nix-path) is ignored, such that only explicitly set search path entries are taken into account. )"}; Setting pureEval{this, false, "pure-eval", From 630580162686dfd9a74d23acdee7b24ebd6b7c4c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 7 Oct 2023 02:53:41 +0200 Subject: [PATCH 358/909] reword and reformat description of `builtins.import` --- src/libexpr/primops.cc | 101 ++++++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 46 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 1d04aeb2e..1ff2d5f0b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -258,62 +258,71 @@ static RegisterPrimOp primop_import({ .args = {"path"}, // TODO turn "normal path values" into link below .doc = R"( - Load, parse and return the Nix expression in the file *path*. - - The *path* argument must meet the same criteria as an [interpolated expression](@docroot@/language/string-interpolation.md#interpolated-expression). - - If *path* is a directory, the file `default.nix` in that directory - is loaded. - - Evaluation aborts if the file doesn’t exist or contains - an incorrect Nix expression. `import` implements Nix’s module - system: you can put any Nix expression (such as a set or a - function) in a separate file, and use it from Nix expressions in - other files. + Load, parse, and return the Nix expression in the file *path*. > **Note** > > Unlike some languages, `import` is a regular function in Nix. - > Paths using the angle bracket syntax (e.g., `import` *\*) - > are normal [path values](@docroot@/language/values.md#type-path). - A Nix expression loaded by `import` must not contain any *free - variables* (identifiers that are not defined in the Nix expression - itself and are not built-in). Therefore, it cannot refer to - variables that are in scope at the call site. For instance, if you - have a calling expression + The *path* argument must meet the same criteria as an [interpolated expression](@docroot@/language/string-interpolation.md#interpolated-expression). - ```nix - rec { - x = 123; - y = import ./foo.nix; - } - ``` + If *path* is a directory, the file `default.nix` in that directory is used if it exists. - then the following `foo.nix` will give an error: + > **Example** + > + > ```console + > $ echo 123 > default.nix + > ``` + > + > Import `default.nix` from the current directory. + > + > ```nix + > import ./. + > ``` + > + > 123 - ```nix - x + 456 - ``` + Evaluation aborts if the file doesn’t exist or contains an invalid Nix expression. - since `x` is not in scope in `foo.nix`. If you want `x` to be - available in `foo.nix`, you should pass it as a function argument: + A Nix expression loaded by `import` must not contain any *free variables*, that is, identifiers that are not defined in the Nix expression itself and are not built-in. + Therefore, it cannot refer to variables that are in scope at the call site. - ```nix - rec { - x = 123; - y = import ./foo.nix x; - } - ``` - - and - - ```nix - x: x + 456 - ``` - - (The function argument doesn’t have to be called `x` in `foo.nix`; - any name would work.) + > **Example** + > + > If you have a calling expression + > + > ```nix + > rec { + > x = 123; + > y = import ./foo.nix; + > } + > ``` + > + > then the following `foo.nix` will give an error: + > + > ```nix + > # foo.nix + > x + 456 + > ``` + > + > since `x` is not in scope in `foo.nix`. + > If you want `x` to be available in `foo.nix`, pass it as a function argument: + > + > ```nix + > rec { + > x = 123; + > y = import ./foo.nix x; + > } + > ``` + > + > and + > + > ```nix + > # foo.nix + > x: x + 456 + > ``` + > + > The function argument doesn’t have to be called `x` in `foo.nix`; any name would work. )", .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { From f00a5eb11b53a800850468f5e9e4658d05b419f9 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 7 Oct 2023 03:10:20 +0200 Subject: [PATCH 359/909] introduce lookup paths as a distinct language construct so far they did not really have a name, and were at best referred to as "angle bracket syntax". --- doc/manual/src/SUMMARY.md.in | 1 + .../src/language/constructs/lookup-path.md | 27 +++++++++++++++++++ doc/manual/src/language/values.md | 9 +++---- src/libexpr/primops.cc | 6 ++--- 4 files changed, 34 insertions(+), 9 deletions(-) create mode 100644 doc/manual/src/language/constructs/lookup-path.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 2e631058f..e49e77cf5 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -30,6 +30,7 @@ - [Data Types](language/values.md) - [Language Constructs](language/constructs.md) - [String interpolation](language/string-interpolation.md) + - [Lookup path](language/constructs/lookup-path.md) - [Operators](language/operators.md) - [Derivations](language/derivations.md) - [Advanced Attributes](language/advanced-attributes.md) diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/src/language/constructs/lookup-path.md new file mode 100644 index 000000000..8ec835309 --- /dev/null +++ b/doc/manual/src/language/constructs/lookup-path.md @@ -0,0 +1,27 @@ +# Lookup path + +> **Syntax** +> +> *lookup-path* = `<` *identifier* [ `/` *identifier* `]`... `>` + +A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/values.md#type-path) if the identifier matches a search path entry. + +The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). + +See [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile) for details on lookup path resolution. + +> **Example** +> +> ```nix +> +>``` +> +> /nix/var/nix/profiles/per-user/root/channels/nixpkgs + +> **Example** +> +> ```nix +> +>``` +> +> /nix/var/nix/profiles/per-user/root/channels/nixpkgs/nixos diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md index 2ae3e143a..4d8950e85 100644 --- a/doc/manual/src/language/values.md +++ b/doc/manual/src/language/values.md @@ -107,11 +107,6 @@ e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`. - Paths can also be specified between angle brackets, e.g. - ``. This means that the directories listed in the - environment variable `NIX_PATH` will be searched for the given file - or directory name. - When an [interpolated string][string interpolation] evaluates to a path, the path is first copied into the Nix store and the resulting string is the [store path] of the newly created [store object]. [store path]: ../glossary.md#gloss-store-path @@ -123,13 +118,15 @@ For example, assume you used a file path in an interpolated string during a `nix repl` session. Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new store path, since Nix might not re-read the file contents. - Paths themselves, except those in angle brackets (`< >`), support [string interpolation]. + Paths support [string interpolation]. At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division operation. `./a.${foo}/b.${bar}` is a path. + [Lookup paths](./constructs/lookup-path.md) such as `` resolve to path values. + - Boolean *Booleans* with values `true` and `false`. diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5de1b2828..a8cb4ccfe 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1723,7 +1723,7 @@ static RegisterPrimOp primop_findFile(PrimOp { If the suffix is found inside that directory, then the entry is a match; the combined absolute path of the directory (now downloaded if need be) and the suffix is returned. - The syntax + [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions can be [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath): ```nix @@ -4389,9 +4389,9 @@ void EvalState::createBaseEnv() addConstant("__nixPath", v, { .type = nList, .doc = R"( - The search path used to resolve angle bracket path lookups. + List of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). - Angle bracket expressions can be + Lookup path expressions can be [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.findFile`](./builtins.html#builtins-findFile): From 896a90520280f2645c32b9a0136ab281843942c3 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 9 Oct 2023 10:07:46 +0200 Subject: [PATCH 360/909] AE -> BE; fix redirects --- doc/manual/redirects.js | 1 + doc/manual/src/contributing/testing.md | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 9d083a43d..d1b10109d 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -354,6 +354,7 @@ const redirects = { "installer-tests": "testing.html#installer-tests", "one-time-setup": "testing.html#one-time-setup", "using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing", + "characterization-testing": "#characterisation-testing-unit", } }; diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index 1a6388e40..3d75ebe7b 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -62,11 +62,11 @@ The path to the `unit-test-data` directory is passed to the unit test executable You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable. -### Characterization testing +### Characterisation testing { #characaterisation-testing-unit } -See [below](#characterization-testing-1) for a broader discussion of characterization testing. +See [functional characterisation testing](#characterisation-testing-functional) for a broader discussion of characterisation testing. -Like with the functional characterization, `_NIX_TEST_ACCEPT=1` is also used. +Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used. For example: ```shell-session $ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN @@ -77,8 +77,8 @@ $ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN [ SKIPPED ] WorkerProtoTest.storePath_write ... ``` -will regenerate the "golden master" expected result for the `libnixstore` characterization tests. -The characterization tests will mark themselves "skipped" since they regenerated the expected result instead of actually testing anything. +will regenerate the "golden master" expected result for the `libnixstore` characterisation tests. +The characterisation tests will mark themselves "skipped" since they regenerated the expected result instead of actually testing anything. ## Functional tests @@ -195,9 +195,9 @@ To remove any traces of that: git clean -x --force tests ``` -### Characterization testing +### Characterisation testing { #characterisation-testing-functional } -Occasionally, Nix utilizes a technique called [Characterization Testing](https://en.wikipedia.org/wiki/Characterization_test) as part of the functional tests. +Occasionally, Nix utilizes a technique called [Characterisation Testing](https://en.wikipedia.org/wiki/Characterization_test) as part of the functional tests. This technique is to include the exact output/behavior of a former version of Nix in a test in order to check that Nix continues to produce the same behavior going forward. For example, this technique is used for the language tests, to check both the printed final value if evaluation was successful, and any errors and warnings encountered. @@ -208,7 +208,7 @@ For example: ```bash _NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test ``` -This convention is shared with the [characterization unit tests](#characterization-testing-1) too. +This convention is shared with the [characterisation unit tests](#characterisation-testing-unit) too. An interesting situation to document is the case when these tests are "overfitted". The language tests are, again, an example of this. @@ -221,7 +221,7 @@ Diagnostic outputs are indeed not a stable interface, but they still are importa By recording the expected output, the test suite guards against accidental changes, and ensure the *result* (not just the code that implements it) of the diagnostic code paths are under code review. Regressions are caught, and improvements always show up in code review. -To ensure that characterization testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. +To ensure that characterisation testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. ## Integration tests @@ -235,7 +235,7 @@ You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix- After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch. -Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): +Creating a Cachix cache for your installer tests and adding its authorisation token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): - The `installer` job generates installers for the platforms below and uploads them to your Cachix cache: - `x86_64-linux` From 0246de1896df732739b0874c83b7d05becc83cf3 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 9 Oct 2023 10:14:25 +0200 Subject: [PATCH 361/909] remove unnecessary indentation from markdown list this makes it a bit easier to work with, as some tooling doesn't work well with too much indentation. --- doc/manual/src/command-ref/env-common.md | 175 ++++++++++++----------- 1 file changed, 95 insertions(+), 80 deletions(-) diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index b4a9bb2a9..34e0dbfbd 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -2,109 +2,124 @@ Most Nix commands interpret the following environment variables: - - [`IN_NIX_SHELL`](#env-IN_NIX_SHELL)\ - Indicator that tells if the current environment was set up by - `nix-shell`. It can have the values `pure` or `impure`. +- [`IN_NIX_SHELL`](#env-IN_NIX_SHELL) - - [`NIX_PATH`](#env-NIX_PATH)\ - A colon-separated list of directories used to look up the location of Nix - expressions using [paths](@docroot@/language/values.md#type-path) - enclosed in angle brackets (i.e., ``), - e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the - [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). + Indicator that tells if the current environment was set up by + `nix-shell`. It can have the values `pure` or `impure`. - If `NIX_PATH` is not set at all, Nix will fall back to the following list in [impure](@docroot@/command-ref/conf-file.md#conf-pure-eval) and [unrestricted](@docroot@/command-ref/conf-file.md#conf-restrict-eval) evaluation mode: +- [`NIX_PATH`](#env-NIX_PATH) - 1. `$HOME/.nix-defexpr/channels` - 2. `nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixpkgs` - 3. `/nix/var/nix/profiles/per-user/root/channels` + A colon-separated list of directories used to look up the location of Nix + expressions using [paths](@docroot@/language/values.md#type-path) + enclosed in angle brackets (i.e., ``), + e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the + [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). - If `NIX_PATH` is set to an empty string, resolving search paths will always fail. - For example, attempting to use `` will produce: + If `NIX_PATH` is not set at all, Nix will fall back to the following list in [impure](@docroot@/command-ref/conf-file.md#conf-pure-eval) and [unrestricted](@docroot@/command-ref/conf-file.md#conf-restrict-eval) evaluation mode: - error: file 'nixpkgs' was not found in the Nix search path + 1. `$HOME/.nix-defexpr/channels` + 2. `nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixpkgs` + 3. `/nix/var/nix/profiles/per-user/root/channels` - - [`NIX_IGNORE_SYMLINK_STORE`](#env-NIX_IGNORE_SYMLINK_STORE)\ - Normally, the Nix store directory (typically `/nix/store`) is not - allowed to contain any symlink components. This is to prevent - “impure” builds. Builders sometimes “canonicalise” paths by - resolving all symlink components. Thus, builds on different machines - (with `/nix/store` resolving to different locations) could yield - different results. This is generally not a problem, except when - builds are deployed to machines where `/nix/store` resolves - differently. If you are sure that you’re not going to do that, you - can set `NIX_IGNORE_SYMLINK_STORE` to `1`. + If `NIX_PATH` is set to an empty string, resolving search paths will always fail. + For example, attempting to use `` will produce: - Note that if you’re symlinking the Nix store so that you can put it - on another file system than the root file system, on Linux you’re - better off using `bind` mount points, e.g., + error: file 'nixpkgs' was not found in the Nix search path - ```console - $ mkdir /nix - $ mount -o bind /mnt/otherdisk/nix /nix - ``` +- [`NIX_IGNORE_SYMLINK_STORE`](#env-NIX_IGNORE_SYMLINK_STORE) - Consult the mount 8 manual page for details. + Normally, the Nix store directory (typically `/nix/store`) is not + allowed to contain any symlink components. This is to prevent + “impure” builds. Builders sometimes “canonicalise” paths by + resolving all symlink components. Thus, builds on different machines + (with `/nix/store` resolving to different locations) could yield + different results. This is generally not a problem, except when + builds are deployed to machines where `/nix/store` resolves + differently. If you are sure that you’re not going to do that, you + can set `NIX_IGNORE_SYMLINK_STORE` to `1`. - - [`NIX_STORE_DIR`](#env-NIX_STORE_DIR)\ - Overrides the location of the Nix store (default `prefix/store`). + Note that if you’re symlinking the Nix store so that you can put it + on another file system than the root file system, on Linux you’re + better off using `bind` mount points, e.g., - - [`NIX_DATA_DIR`](#env-NIX_DATA_DIR)\ - Overrides the location of the Nix static data directory (default - `prefix/share`). + ```console + $ mkdir /nix + $ mount -o bind /mnt/otherdisk/nix /nix + ``` - - [`NIX_LOG_DIR`](#env-NIX_LOG_DIR)\ - Overrides the location of the Nix log directory (default - `prefix/var/log/nix`). + Consult the mount 8 manual page for details. - - [`NIX_STATE_DIR`](#env-NIX_STATE_DIR)\ - Overrides the location of the Nix state directory (default - `prefix/var/nix`). +- [`NIX_STORE_DIR`](#env-NIX_STORE_DIR) - - [`NIX_CONF_DIR`](#env-NIX_CONF_DIR)\ - Overrides the location of the system Nix configuration directory - (default `prefix/etc/nix`). + Overrides the location of the Nix store (default `prefix/store`). - - [`NIX_CONFIG`](#env-NIX_CONFIG)\ - Applies settings from Nix configuration from the environment. - The content is treated as if it was read from a Nix configuration file. - Settings are separated by the newline character. +- [`NIX_DATA_DIR`](#env-NIX_DATA_DIR) - - [`NIX_USER_CONF_FILES`](#env-NIX_USER_CONF_FILES)\ - Overrides the location of the Nix user configuration files to load from. + Overrides the location of the Nix static data directory (default + `prefix/share`). - The default are the locations according to the [XDG Base Directory Specification]. - See the [XDG Base Directories](#xdg-base-directories) sub-section for details. +- [`NIX_LOG_DIR`](#env-NIX_LOG_DIR) - The variable is treated as a list separated by the `:` token. + Overrides the location of the Nix log directory (default + `prefix/var/log/nix`). - - [`TMPDIR`](#env-TMPDIR)\ - Use the specified directory to store temporary files. In particular, - this includes temporary build directories; these can take up - substantial amounts of disk space. The default is `/tmp`. +- [`NIX_STATE_DIR`](#env-NIX_STATE_DIR) - - [`NIX_REMOTE`](#env-NIX_REMOTE)\ - This variable should be set to `daemon` if you want to use the Nix - daemon to execute Nix operations. This is necessary in [multi-user - Nix installations](@docroot@/installation/multi-user.md). If the Nix - daemon's Unix socket is at some non-standard path, this variable - should be set to `unix://path/to/socket`. Otherwise, it should be - left unset. + Overrides the location of the Nix state directory (default + `prefix/var/nix`). - - [`NIX_SHOW_STATS`](#env-NIX_SHOW_STATS)\ - If set to `1`, Nix will print some evaluation statistics, such as - the number of values allocated. +- [`NIX_CONF_DIR`](#env-NIX_CONF_DIR) - - [`NIX_COUNT_CALLS`](#env-NIX_COUNT_CALLS)\ - If set to `1`, Nix will print how often functions were called during - Nix expression evaluation. This is useful for profiling your Nix - expressions. + Overrides the location of the system Nix configuration directory + (default `prefix/etc/nix`). - - [`GC_INITIAL_HEAP_SIZE`](#env-GC_INITIAL_HEAP_SIZE)\ - If Nix has been configured to use the Boehm garbage collector, this - variable sets the initial size of the heap in bytes. It defaults to - 384 MiB. Setting it to a low value reduces memory consumption, but - will increase runtime due to the overhead of garbage collection. +- [`NIX_CONFIG`](#env-NIX_CONFIG) + + Applies settings from Nix configuration from the environment. + The content is treated as if it was read from a Nix configuration file. + Settings are separated by the newline character. + +- [`NIX_USER_CONF_FILES`](#env-NIX_USER_CONF_FILES) + + Overrides the location of the Nix user configuration files to load from. + + The default are the locations according to the [XDG Base Directory Specification]. + See the [XDG Base Directories](#xdg-base-directories) sub-section for details. + + The variable is treated as a list separated by the `:` token. + +- [`TMPDIR`](#env-TMPDIR) + + Use the specified directory to store temporary files. In particular, + this includes temporary build directories; these can take up + substantial amounts of disk space. The default is `/tmp`. + +- [`NIX_REMOTE`](#env-NIX_REMOTE) + + This variable should be set to `daemon` if you want to use the Nix + daemon to execute Nix operations. This is necessary in [multi-user + Nix installations](@docroot@/installation/multi-user.md). If the Nix + daemon's Unix socket is at some non-standard path, this variable + should be set to `unix://path/to/socket`. Otherwise, it should be + left unset. + +- [`NIX_SHOW_STATS`](#env-NIX_SHOW_STATS) + + If set to `1`, Nix will print some evaluation statistics, such as + the number of values allocated. + +- [`NIX_COUNT_CALLS`](#env-NIX_COUNT_CALLS) + + If set to `1`, Nix will print how often functions were called during + Nix expression evaluation. This is useful for profiling your Nix + expressions. + +- [`GC_INITIAL_HEAP_SIZE`](#env-GC_INITIAL_HEAP_SIZE) + + If Nix has been configured to use the Boehm garbage collector, this + variable sets the initial size of the heap in bytes. It defaults to + 384 MiB. Setting it to a low value reduces memory consumption, but + will increase runtime due to the overhead of garbage collection. ## XDG Base Directories From 47b3508665bb9dc6a8467841ce487c7426080f17 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 6 Oct 2023 11:57:31 -0400 Subject: [PATCH 362/909] Use positive source filtering for the standalone functional tests job Additionally this skipping of the building is reimplemented to be a bit more robust and use the same idioms as the functionality for skipping the tests. In particular, it will now work even if the source files exist, so we can do this during development too. --- Makefile | 15 +++++++--- Makefile.config.in | 3 +- configure.ac | 11 +++++-- flake.nix | 60 +++++++++++++++++++++++++-------------- local.mk | 2 -- tests/functional/local.mk | 6 +--- 6 files changed, 61 insertions(+), 36 deletions(-) diff --git a/Makefile b/Makefile index 6658e3490..4f4ac0c6e 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,7 @@ +-include Makefile.config +clean-files += Makefile.config + +ifeq ($(ENABLE_BUILD), yes) makefiles = \ mk/precompiled-headers.mk \ local.mk \ @@ -18,15 +22,18 @@ makefiles = \ misc/upstart/local.mk \ doc/manual/local.mk \ doc/internal-api/local.mk +endif --include Makefile.config - -ifeq ($(tests), yes) +ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes) UNIT_TEST_ENV = _NIX_TEST_UNIT_DATA=unit-test-data makefiles += \ src/libutil/tests/local.mk \ src/libstore/tests/local.mk \ - src/libexpr/tests/local.mk \ + src/libexpr/tests/local.mk +endif + +ifeq ($(ENABLE_TESTS), yes) +makefiles += \ tests/functional/local.mk \ tests/functional/ca/local.mk \ tests/functional/dyn-drv/local.mk \ diff --git a/Makefile.config.in b/Makefile.config.in index 707cfe0e3..19992fa20 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -46,5 +46,6 @@ sandbox_shell = @sandbox_shell@ storedir = @storedir@ sysconfdir = @sysconfdir@ system = @system@ -tests = @tests@ +ENABLE_BUILD = @ENABLE_BUILD@ +ENABLE_TESTS = @ENABLE_TESTS@ internal_api_docs = @internal_api_docs@ diff --git a/configure.ac b/configure.ac index 6d78237f0..225baf6b5 100644 --- a/configure.ac +++ b/configure.ac @@ -152,12 +152,17 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then LDFLAGS="-latomic $LDFLAGS" fi +# Running the functional tests without building Nix is useful for testing +# different pre-built versions of Nix against each other. +AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]), + ENABLE_BUILD=$enableval, ENABLE_BUILD=yes) +AC_SUBST(ENABLE_BUILD) # Building without tests is useful for bootstrapping with a smaller footprint # or running the tests in a separate derivation. Otherwise, we do compile and # run them. AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]), - tests=$enableval, tests=yes) -AC_SUBST(tests) + ENABLE_TESTS=$enableval, ENABLE_TESTS=yes) +AC_SUBST(ENABLE_TESTS) # Building without API docs is the default as Nix' C++ interfaces are internal and unstable. AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]), @@ -289,7 +294,7 @@ if test "$gc" = yes; then fi -if test "$tests" = yes; then +if test "$ENABLE_TESTS" = yes; then # Look for gtest. PKG_CHECK_MODULES([GTEST], [gtest_main]) diff --git a/flake.nix b/flake.nix index bec62fb95..1fe26853f 100644 --- a/flake.nix +++ b/flake.nix @@ -59,29 +59,40 @@ # that would interfere with repo semantics. fileset.fileFilter (f: f.name != ".gitignore") ./.; + configureFiles = fileset.unions [ + ./.version + ./configure.ac + ./m4 + # TODO: do we really need README.md? It doesn't seem used in the build. + ./README.md + ]; + + topLevelBuildFiles = fileset.unions [ + ./local.mk + ./Makefile + ./Makefile.config.in + ./mk + ]; + + functionalTestFiles = fileset.unions [ + ./tests/functional + (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) + ]; + nixSrc = fileset.toSource { root = ./.; fileset = fileset.intersect baseFiles (fileset.unions [ - ./.version + configureFiles + topLevelBuildFiles ./boehmgc-coroutine-sp-fallback.diff - ./bootstrap.sh - ./configure.ac ./doc - ./local.mk - ./m4 - ./Makefile - ./Makefile.config.in ./misc - ./mk ./precompiled-headers.h ./src - ./tests/functional ./unit-test-data ./COPYING ./scripts/local.mk - (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) - # TODO: do we really need README.md? It doesn't seem used in the build. - ./README.md + functionalTestFiles ]); }; @@ -252,7 +263,6 @@ testNixVersions = pkgs: client: daemon: with commonDeps { inherit pkgs; }; with pkgs.lib; pkgs.stdenv.mkDerivation { NIX_DAEMON_PACKAGE = daemon; NIX_CLIENT_PACKAGE = client; - HAVE_LOCAL_NIX_BUILD = false; name = "nix-tests" + optionalString @@ -261,7 +271,14 @@ "-${client.version}-against-${daemon.version}"; inherit version; - src = nixSrc; + src = fileset.toSource { + root = ./.; + fileset = fileset.intersect baseFiles (fileset.unions [ + configureFiles + topLevelBuildFiles + functionalTestFiles + ]); + }; VERSION_SUFFIX = versionSuffix; @@ -271,19 +288,20 @@ enableParallelBuilding = true; - configureFlags = testConfigureFlags; # otherwise configure fails + configureFlags = + testConfigureFlags # otherwise configure fails + ++ [ "--disable-build" ]; + dontBuild = true; doInstallCheck = true; - buildPhase = '' - # Remove the source files to make sure that we're not accidentally rebuilding Nix - rm src/**/*.cc - ''; - installPhase = '' mkdir -p $out ''; - installCheckPhase = "make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES"; + installCheckPhase = '' + mkdir -p src/nix-channel + make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES + ''; }; binaryTarball = nix: pkgs: diff --git a/local.mk b/local.mk index 6951c179e..3f3abb9f0 100644 --- a/local.mk +++ b/local.mk @@ -1,5 +1,3 @@ -clean-files += Makefile.config - GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch # Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers. ERROR_SWITCH_ENUM = -Werror=switch-enum diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 4d29f60a1..6f6c94fe6 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -1,7 +1,3 @@ -# whether to run the tests that assume that we have a local build of -# Nix -HAVE_LOCAL_NIX_BUILD ?= 1 - nix_tests = \ test-infra.sh \ init.sh \ @@ -131,7 +127,7 @@ ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh endif -ifeq ($(HAVE_LOCAL_NIX_BUILD), 1) +ifeq ($(ENABLE_BUILD), yes) nix_tests += test-libstoreconsumer.sh ifeq ($(BUILD_SHARED_LIBS), 1) From 6654b4e3b4cf4b2181a0d14aa792d56eeab6219a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 9 Oct 2023 07:56:59 -0400 Subject: [PATCH 363/909] Use positive source filtering for the Perl bindings --- flake.nix | 10 +++++++++- perl/Makefile | 8 +++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 1fe26853f..301e65545 100644 --- a/flake.nix +++ b/flake.nix @@ -477,7 +477,15 @@ passthru.perl-bindings = with final; perl.pkgs.toPerlModule (currentStdenv.mkDerivation { name = "nix-perl-${version}"; - src = self; + src = fileset.toSource { + root = ./.; + fileset = fileset.intersect baseFiles (fileset.unions [ + ./perl + ./.version + ./m4 + ./mk + ]); + }; nativeBuildInputs = [ buildPackages.autoconf-archive diff --git a/perl/Makefile b/perl/Makefile index c2c95f255..832668dd1 100644 --- a/perl/Makefile +++ b/perl/Makefile @@ -1,6 +1,12 @@ makefiles = local.mk -GLOBAL_CXXFLAGS += -g -Wall -std=c++2a -I ../src +GLOBAL_CXXFLAGS += -g -Wall -std=c++2a + +# A convenience for concurrent development of Nix and its Perl bindings. +# Not needed in a standalone build of the Perl bindings. +ifneq ("$(wildcard ../src)", "") + GLOBAL_CXXFLAGS += -I ../src +endif -include Makefile.config From 81d3a8542a6c919bb2302a877fd4443a6c07a2dc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 9 Oct 2023 10:21:18 -0400 Subject: [PATCH 364/909] doc: Slight reword of "interpolated expression" in paragraph I was sleepy and confused that "interpolated expression" was a new type of thing at first. This nudges the reader to understand that its just a regular expression, and these conditions are imposed by the interpolation operation. --- doc/manual/src/language/string-interpolation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 2e650e348..e999b287b 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -92,7 +92,7 @@ Attributes can be selected with interpolated strings. # Interpolated expression -An interpolated expression must evaluate to one of the following: +An expression that is interpolated must evaluate to one of the following: - a [string] - a [path] From c32084a12cb922f54829b329fa1527f865e5b8ca Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 9 Oct 2023 16:25:53 +0200 Subject: [PATCH 365/909] printStats -> maybePrintStats --- src/libcmd/command.cc | 2 +- src/libexpr/eval.cc | 2 +- src/libexpr/eval.hh | 2 +- src/nix-build/nix-build.cc | 2 +- src/nix-env/nix-env.cc | 2 +- src/nix-instantiate/nix-instantiate.cc | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 4fc197956..a88ba8134 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -98,7 +98,7 @@ EvalCommand::EvalCommand() EvalCommand::~EvalCommand() { if (evalState) - evalState->printStats(); + evalState->maybePrintStats(); } ref EvalCommand::getEvalStore() diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 600444955..b18eb5266 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2491,7 +2491,7 @@ bool EvalState::fullGC() { #endif } -void EvalState::printStats() +void EvalState::maybePrintStats() { bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0"; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 3bfa34ef6..c95558952 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -714,7 +714,7 @@ public: * Performs a full memory GC before printing the statistics, so that the * GC statistics are more accurate. */ - void printStats(); + void maybePrintStats(); /** * Print statistics, unconditionally, cheaply, without performing a GC first. diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index e2189fc66..2895c5e3c 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -344,7 +344,7 @@ static void main_nix_build(int argc, char * * argv) } } - state->printStats(); + state->maybePrintStats(); auto buildPaths = [&](const std::vector & paths) { /* Note: we do this even when !printMissing to efficiently diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index e455a50fd..01742daa8 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1531,7 +1531,7 @@ static int main_nix_env(int argc, char * * argv) op(globals, std::move(opFlags), std::move(opArgs)); - globals.state->printStats(); + globals.state->maybePrintStats(); return 0; } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 446b27e66..d40196497 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -189,7 +189,7 @@ static int main_nix_instantiate(int argc, char * * argv) evalOnly, outputKind, xmlOutputSourceLocation, e); } - state->printStats(); + state->maybePrintStats(); return 0; } From 2f0b508c29f1c8eedc1f93c1a2dc31e8eb60de9a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 9 Oct 2023 12:52:58 -0400 Subject: [PATCH 366/909] Get rid of `bootstrap.sh` For people working on Nix with `nix develop`, it's better to just use `autoreconfPhase` and `configurePhase`, which is standard Nixpkgs / nix shell make from Nixpkgs practice --- it is good to emphasize the degree to which Nix is *just* a regular C++ project which can be worked on in the regular way. (For people running `nix-shell`, the story is similar, except `configurePhase` would use non-writable store paths, which matters for hte times we use output paths before `make install`, so I kept the existing `./configure ...` instruction.) For people building Nix without Nix (e.g. packaging it for another distro) they also don't need `bootstrap.sh`, and can just run `autoreconf -vfi` directly. (More likely, they have their own idioms to do this just as we have `autoreconfPhase`.) --- bootstrap.sh | 4 ---- doc/manual/src/contributing/hacking.md | 6 +++--- doc/manual/src/installation/building-source.md | 2 +- 3 files changed, 4 insertions(+), 8 deletions(-) delete mode 100755 bootstrap.sh diff --git a/bootstrap.sh b/bootstrap.sh deleted file mode 100755 index e3e259351..000000000 --- a/bootstrap.sh +++ /dev/null @@ -1,4 +0,0 @@ -#! /bin/sh -e -rm -f aclocal.m4 -mkdir -p config -exec autoreconf -vfi diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index db393942c..38c144fcc 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -42,8 +42,8 @@ $ nix develop .#native-clang11StdenvPackages To build Nix itself in this shell: ```console -[nix-shell]$ ./bootstrap.sh -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out +[nix-shell]$ autoreconfPhase +[nix-shell]$ configurePhase [nix-shell]$ make -j $NIX_BUILD_CORES ``` @@ -86,7 +86,7 @@ $ nix-shell --attr devShells.x86_64-linux.native-clang11StdenvPackages To build Nix itself in this shell: ```console -[nix-shell]$ ./bootstrap.sh +[nix-shell]$ autoreconfPhase [nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out [nix-shell]$ make -j $NIX_BUILD_CORES ``` diff --git a/doc/manual/src/installation/building-source.md b/doc/manual/src/installation/building-source.md index ed1efffd8..7dad9805a 100644 --- a/doc/manual/src/installation/building-source.md +++ b/doc/manual/src/installation/building-source.md @@ -3,7 +3,7 @@ After cloning Nix's Git repository, issue the following commands: ```console -$ ./bootstrap.sh +$ autoreconf -vfi $ ./configure options... $ make $ make install From be81764320fc28131d23b85575076218eb7424c0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Mar 2022 04:39:57 +0000 Subject: [PATCH 367/909] Factor out bits of the worker protocol to use elsewhere This introduces some shared infrastructure for our notion of protocols. We can then define multiple protocols in terms of that notion. We an also express how particular protocols depend on each other. For example, we can define a common protocol and a worker protocol, where the second depends on the first in terms of the data types it can read and write. The "serve" protocol can just use the common one for now, but will eventually need its own machinary just like the worker protocol for version-aware serialisers --- doc/internal-api/doxygen.cfg.in | 30 ++++ src/libstore/build/derivation-goal.cc | 10 +- src/libstore/common-protocol-impl.hh | 41 +++++ src/libstore/common-protocol.cc | 98 +++++++++++ src/libstore/common-protocol.hh | 106 ++++++++++++ src/libstore/derivations.cc | 12 +- src/libstore/export-import.cc | 12 +- src/libstore/legacy-ssh-store.cc | 44 ++--- .../length-prefixed-protocol-helper.hh | 162 ++++++++++++++++++ src/libstore/worker-protocol-impl.hh | 111 ++++-------- src/libstore/worker-protocol.cc | 81 +-------- src/libstore/worker-protocol.hh | 60 ++----- src/nix-store/nix-store.cc | 26 +-- 13 files changed, 542 insertions(+), 251 deletions(-) create mode 100644 src/libstore/common-protocol-impl.hh create mode 100644 src/libstore/common-protocol.cc create mode 100644 src/libstore/common-protocol.hh create mode 100644 src/libstore/length-prefixed-protocol-helper.hh diff --git a/doc/internal-api/doxygen.cfg.in b/doc/internal-api/doxygen.cfg.in index 8f526536d..599be2470 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/doc/internal-api/doxygen.cfg.in @@ -54,6 +54,23 @@ INPUT = \ src/nix-env \ src/nix-store +# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names +# in the source code. If set to NO, only conditional compilation will be +# performed. Macro expansion can be done in a controlled way by setting +# EXPAND_ONLY_PREDEF to YES. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +MACRO_EXPANSION = YES + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then +# the macro expansion is limited to the macros specified with the PREDEFINED and +# EXPAND_AS_DEFINED tags. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_ONLY_PREDEF = YES + # The INCLUDE_PATH tag can be used to specify one or more directories that # contain include files that are not input files but should be processed by the # preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of @@ -61,3 +78,16 @@ INPUT = \ # This tag requires that the tag SEARCH_INCLUDES is set to YES. INCLUDE_PATH = @RAPIDCHECK_HEADERS@ + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this +# tag can be used to specify a list of macro names that should be expanded. The +# macro definition that is found in the sources will be used. Use the PREDEFINED +# tag if you want to use a different macro definition that overrules the +# definition found in the source code. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_AS_DEFINED = \ + DECLARE_COMMON_SERIALISER \ + DECLARE_WORKER_SERIALISER \ + DECLARE_SERVE_SERIALISER \ + LENGTH_PREFIXED_PROTO_HELPER diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 83c0a3135..dc4d91079 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -8,8 +8,8 @@ #include "util.hh" #include "archive.hh" #include "compression.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include "topo-sort.hh" #include "callback.hh" #include "local-store.hh" // TODO remove, along with remaining downcasts @@ -1185,11 +1185,11 @@ HookReply DerivationGoal::tryBuildHook() throw; } - WorkerProto::WriteConn conn { hook->sink }; + CommonProto::WriteConn conn { hook->sink }; /* Tell the hook all the inputs that have to be copied to the remote system. */ - WorkerProto::write(worker.store, conn, inputPaths); + CommonProto::write(worker.store, conn, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ @@ -1200,7 +1200,7 @@ HookReply DerivationGoal::tryBuildHook() if (buildMode != bmCheck && status.known && status.known->isValid()) continue; missingOutputs.insert(outputName); } - WorkerProto::write(worker.store, conn, missingOutputs); + CommonProto::write(worker.store, conn, missingOutputs); } hook->sink = FdSink(); diff --git a/src/libstore/common-protocol-impl.hh b/src/libstore/common-protocol-impl.hh new file mode 100644 index 000000000..079c182b8 --- /dev/null +++ b/src/libstore/common-protocol-impl.hh @@ -0,0 +1,41 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * This file is an exmample of the "impl.hh" pattern. See the + * contributing guide. + */ + +#include "common-protocol.hh" +#include "length-prefixed-protocol-helper.hh" + +namespace nix { + +/* protocol-agnostic templates */ + +#define COMMON_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ + TEMPLATE T CommonProto::Serialise< T >::read(const Store & store, CommonProto::ReadConn conn) \ + { \ + return LengthPrefixedProtoHelper::read(store, conn); \ + } \ + TEMPLATE void CommonProto::Serialise< T >::write(const Store & store, CommonProto::WriteConn conn, const T & t) \ + { \ + LengthPrefixedProtoHelper::write(store, conn, t); \ + } + +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) + +#define COMMA_ , +COMMON_USE_LENGTH_PREFIX_SERIALISER( + template, + std::map) +#undef COMMA_ + + +/* protocol-specific templates */ + +} diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc new file mode 100644 index 000000000..f906814bc --- /dev/null +++ b/src/libstore/common-protocol.cc @@ -0,0 +1,98 @@ +#include "serialise.hh" +#include "util.hh" +#include "path-with-outputs.hh" +#include "store-api.hh" +#include "build-result.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" +#include "archive.hh" +#include "derivations.hh" + +#include + +namespace nix { + +/* protocol-agnostic definitions */ + +std::string CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return readString(conn.from); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const std::string & str) +{ + conn.to << str; +} + + +StorePath CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return store.parseStorePath(readString(conn.from)); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const StorePath & storePath) +{ + conn.to << store.printStorePath(storePath); +} + + +ContentAddress CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return ContentAddress::parse(readString(conn.from)); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const ContentAddress & ca) +{ + conn.to << renderContentAddress(ca); +} + + +Realisation CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + std::string rawInput = readString(conn.from); + return Realisation::fromJSON( + nlohmann::json::parse(rawInput), + "remote-protocol" + ); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const Realisation & realisation) +{ + conn.to << realisation.toJSON().dump(); +} + + +DrvOutput CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return DrvOutput::parse(readString(conn.from)); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) +{ + conn.to << drvOutput.to_string(); +} + + +std::optional CommonProto::Serialise>::read(const Store & store, CommonProto::ReadConn conn) +{ + auto s = readString(conn.from); + return s == "" ? std::optional {} : store.parseStorePath(s); +} + +void CommonProto::Serialise>::write(const Store & store, CommonProto::WriteConn conn, const std::optional & storePathOpt) +{ + conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); +} + + +std::optional CommonProto::Serialise>::read(const Store & store, CommonProto::ReadConn conn) +{ + return ContentAddress::parseOpt(readString(conn.from)); +} + +void CommonProto::Serialise>::write(const Store & store, CommonProto::WriteConn conn, const std::optional & caOpt) +{ + conn.to << (caOpt ? renderContentAddress(*caOpt) : ""); +} + +} diff --git a/src/libstore/common-protocol.hh b/src/libstore/common-protocol.hh new file mode 100644 index 000000000..f3f28972a --- /dev/null +++ b/src/libstore/common-protocol.hh @@ -0,0 +1,106 @@ +#pragma once +///@file + +#include "serialise.hh" + +namespace nix { + +class Store; +struct Source; + +// items being serialized +class StorePath; +struct ContentAddress; +struct DrvOutput; +struct Realisation; + + +/** + * Shared serializers between the worker protocol, serve protocol, and a + * few others. + * + * This `struct` is basically just a `namespace`; We use a type rather + * than a namespace just so we can use it as a template argument. + */ +struct CommonProto +{ + /** + * A unidirectional read connection, to be used by the read half of the + * canonical serializers below. + */ + struct ReadConn { + Source & from; + }; + + /** + * A unidirectional write connection, to be used by the write half of the + * canonical serializers below. + */ + struct WriteConn { + Sink & to; + }; + + template + struct Serialise; + + /** + * Wrapper function around `CommonProto::Serialise::write` that allows us to + * infer the type instead of having to write it down explicitly. + */ + template + static void write(const Store & store, WriteConn conn, const T & t) + { + CommonProto::Serialise::write(store, conn, t); + } +}; + +#define DECLARE_COMMON_SERIALISER(T) \ + struct CommonProto::Serialise< T > \ + { \ + static T read(const Store & store, CommonProto::ReadConn conn); \ + static void write(const Store & store, CommonProto::WriteConn conn, const T & str); \ + } + +template<> +DECLARE_COMMON_SERIALISER(std::string); +template<> +DECLARE_COMMON_SERIALISER(StorePath); +template<> +DECLARE_COMMON_SERIALISER(ContentAddress); +template<> +DECLARE_COMMON_SERIALISER(DrvOutput); +template<> +DECLARE_COMMON_SERIALISER(Realisation); + +template +DECLARE_COMMON_SERIALISER(std::vector); +template +DECLARE_COMMON_SERIALISER(std::set); +template +DECLARE_COMMON_SERIALISER(std::tuple); + +#define COMMA_ , +template +DECLARE_COMMON_SERIALISER(std::map); +#undef COMMA_ + +/** + * These use the empty string for the null case, relying on the fact + * that the underlying types never serialize to the empty string. + * + * We do this instead of a generic std::optional instance because + * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For + * the same reason, we don't have a std::variant instances (ordinal + * tags 0...n). + * + * We could the generic instances and then these as specializations for + * compatability, but that's proven a bit finnicky, and also makes the + * worker protocol harder to implement in other languages where such + * specializations may not be allowed. + */ +template<> +DECLARE_COMMON_SERIALISER(std::optional); +template<> +DECLARE_COMMON_SERIALISER(std::optional); + +} diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 67069c3c9..fc17e520c 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -4,8 +4,8 @@ #include "globals.hh" #include "util.hh" #include "split.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include "fs-accessor.hh" #include #include @@ -895,8 +895,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = WorkerProto::Serialise::read(store, - WorkerProto::ReadConn { .from = in }); + drv.inputSrcs = CommonProto::Serialise::read(store, + CommonProto::ReadConn { .from = in }); in >> drv.platform >> drv.builder; drv.args = readStrings(in); @@ -944,8 +944,8 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, }, i.second.raw); } - WorkerProto::write(store, - WorkerProto::WriteConn { .to = out }, + CommonProto::write(store, + CommonProto::WriteConn { .to = out }, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index e866aeb42..87b2f8741 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -1,8 +1,8 @@ #include "serialise.hh" #include "store-api.hh" #include "archive.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include @@ -46,8 +46,8 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - WorkerProto::write(*this, - WorkerProto::WriteConn { .to = teeSink }, + CommonProto::write(*this, + CommonProto::WriteConn { .to = teeSink }, info->references); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") @@ -76,8 +76,8 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) //Activity act(*logger, lvlInfo, "importing path '%s'", info.path); - auto references = WorkerProto::Serialise::read(*this, - WorkerProto::ReadConn { .from = source }); + auto references = CommonProto::Serialise::read(*this, + CommonProto::ReadConn { .from = source }); auto deriver = readString(source); auto narHash = hashString(htSHA256, saved.s); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 78b05031a..7bf4476d0 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -6,8 +6,8 @@ #include "build-result.hh" #include "store-api.hh" #include "path-with-outputs.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include "ssh.hh" #include "derivations.hh" #include "callback.hh" @@ -50,37 +50,37 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor bool good = true; /** - * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the - * factored out worker protocol searlizers with a + * Coercion to `CommonProto::ReadConn`. This makes it easy to use the + * factored out common protocol serialisers with a * `LegacySSHStore::Connection`. * - * The worker protocol connection types are unidirectional, unlike + * The common protocol connection types are unidirectional, unlike * this type. * - * @todo Use server protocol serializers, not worker protocol + * @todo Use server protocol serializers, not common protocol * serializers, once we have made that distiction. */ - operator WorkerProto::ReadConn () + operator CommonProto::ReadConn () { - return WorkerProto::ReadConn { + return CommonProto::ReadConn { .from = from, }; } /* - * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the - * factored out worker protocol searlizers with a + * Coercion to `CommonProto::WriteConn`. This makes it easy to use the + * factored out common protocol searlizers with a * `LegacySSHStore::Connection`. * - * The worker protocol connection types are unidirectional, unlike + * The common protocol connection types are unidirectional, unlike * this type. * - * @todo Use server protocol serializers, not worker protocol + * @todo Use server protocol serializers, not common protocol * serializers, once we have made that distiction. */ - operator WorkerProto::WriteConn () + operator CommonProto::WriteConn () { - return WorkerProto::WriteConn { + return CommonProto::WriteConn { .to = to, }; } @@ -183,7 +183,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = WorkerProto::Serialise::read(*this, *conn); + info->references = CommonProto::Serialise::read(*this, *conn); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -217,7 +217,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - WorkerProto::write(*this, *conn, info.references); + CommonProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize @@ -246,7 +246,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - WorkerProto::write(*this, *conn, info.references); + CommonProto::write(*this, *conn, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -331,7 +331,7 @@ public: if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - auto builtOutputs = WorkerProto::Serialise::read(*this, *conn); + auto builtOutputs = CommonProto::Serialise::read(*this, *conn); for (auto && [output, realisation] : builtOutputs) status.builtOutputs.insert_or_assign( std::move(output.outputName), @@ -409,10 +409,10 @@ public: conn->to << ServeProto::Command::QueryClosure << includeOutputs; - WorkerProto::write(*this, *conn, paths); + CommonProto::write(*this, *conn, paths); conn->to.flush(); - for (auto & i : WorkerProto::Serialise::read(*this, *conn)) + for (auto & i : CommonProto::Serialise::read(*this, *conn)) out.insert(i); } @@ -425,10 +425,10 @@ public: << ServeProto::Command::QueryValidPaths << false // lock << maybeSubstitute; - WorkerProto::write(*this, *conn, paths); + CommonProto::write(*this, *conn, paths); conn->to.flush(); - return WorkerProto::Serialise::read(*this, *conn); + return CommonProto::Serialise::read(*this, *conn); } void connect() override diff --git a/src/libstore/length-prefixed-protocol-helper.hh b/src/libstore/length-prefixed-protocol-helper.hh new file mode 100644 index 000000000..4061b0cd6 --- /dev/null +++ b/src/libstore/length-prefixed-protocol-helper.hh @@ -0,0 +1,162 @@ +#pragma once +/** + * @file Reusable serialisers for serialization container types in a + * length-prefixed manner. + * + * Used by both the Worker and Serve protocols. + */ + +#include "types.hh" + +namespace nix { + +class Store; + +/** + * Reusable serialisers for serialization container types in a + * length-prefixed manner. + * + * @param T The type of the collection being serialised + * + * @param Inner This the most important parameter; this is the "inner" + * protocol. The user of this will substitute `MyProtocol` or similar + * when making a `MyProtocol::Serialiser>`. Note that the + * inside is allowed to call to call `Inner::Serialiser` on different + * types. This is especially important for `std::map` which doesn't have + * a single `T` but one `K` and one `V`. + */ +template +struct LengthPrefixedProtoHelper; + +/*! + * \typedef LengthPrefixedProtoHelper::S + * + * Read this as simply `using S = Inner::Serialise;`. + * + * It would be nice to use that directly, but C++ doesn't seem to allow + * it. The `typename` keyword needed to refer to `Inner` seems to greedy + * (low precedence), and then C++ complains that `Serialise` is not a + * type parameter but a real type. + * + * Making this `S` alias seems to be the only way to avoid these issues. + */ + +#define LENGTH_PREFIXED_PROTO_HELPER(Inner, T) \ + struct LengthPrefixedProtoHelper< Inner, T > \ + { \ + static T read(const Store & store, typename Inner::ReadConn conn); \ + static void write(const Store & store, typename Inner::WriteConn conn, const T & str); \ + private: \ + template using S = typename Inner::template Serialise; \ + } + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector); + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set); + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple); + +template +#define _X std::map +LENGTH_PREFIXED_PROTO_HELPER(Inner, _X); +#undef _X + +template +std::vector +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + std::vector resSet; + auto size = readNum(conn.from); + while (size--) { + resSet.push_back(S::read(store, conn)); + } + return resSet; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::vector & resSet) +{ + conn.to << resSet.size(); + for (auto & key : resSet) { + S::write(store, conn, key); + } +} + +template +std::set +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + std::set resSet; + auto size = readNum(conn.from); + while (size--) { + resSet.insert(S::read(store, conn)); + } + return resSet; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::set & resSet) +{ + conn.to << resSet.size(); + for (auto & key : resSet) { + S::write(store, conn, key); + } +} + +template +std::map +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + std::map resMap; + auto size = readNum(conn.from); + while (size--) { + auto k = S::read(store, conn); + auto v = S::read(store, conn); + resMap.insert_or_assign(std::move(k), std::move(v)); + } + return resMap; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::map & resMap) +{ + conn.to << resMap.size(); + for (auto & i : resMap) { + S::write(store, conn, i.first); + S::write(store, conn, i.second); + } +} + +template +std::tuple +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + return std::tuple { + S::read(store, conn)..., + }; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::tuple & res) +{ + std::apply([&](const Us &... args) { + (S::write(store, conn, args), ...); + }, res); +} + +} diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh index 4f797f95a..c043588d6 100644 --- a/src/libstore/worker-protocol-impl.hh +++ b/src/libstore/worker-protocol-impl.hh @@ -9,86 +9,51 @@ */ #include "worker-protocol.hh" +#include "length-prefixed-protocol-helper.hh" namespace nix { +/* protocol-agnostic templates */ + +#define WORKER_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ + TEMPLATE T WorkerProto::Serialise< T >::read(const Store & store, WorkerProto::ReadConn conn) \ + { \ + return LengthPrefixedProtoHelper::read(store, conn); \ + } \ + TEMPLATE void WorkerProto::Serialise< T >::write(const Store & store, WorkerProto::WriteConn conn, const T & t) \ + { \ + LengthPrefixedProtoHelper::write(store, conn, t); \ + } + +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) + +#define COMMA_ , +WORKER_USE_LENGTH_PREFIX_SERIALISER( + template, + std::map) +#undef COMMA_ + +/** + * Use `CommonProto` where possible. + */ template -std::vector WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) +struct WorkerProto::Serialise { - std::vector resSet; - auto size = readNum(conn.from); - while (size--) { - resSet.push_back(WorkerProto::Serialise::read(store, conn)); + static T read(const Store & store, WorkerProto::ReadConn conn) + { + return CommonProto::Serialise::read(store, + CommonProto::ReadConn { .from = conn.from }); } - return resSet; -} - -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::vector & resSet) -{ - conn.to << resSet.size(); - for (auto & key : resSet) { - WorkerProto::Serialise::write(store, conn, key); + static void write(const Store & store, WorkerProto::WriteConn conn, const T & t) + { + CommonProto::Serialise::write(store, + CommonProto::WriteConn { .to = conn.to }, + t); } -} +}; -template -std::set WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - std::set resSet; - auto size = readNum(conn.from); - while (size--) { - resSet.insert(WorkerProto::Serialise::read(store, conn)); - } - return resSet; -} - -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::set & resSet) -{ - conn.to << resSet.size(); - for (auto & key : resSet) { - WorkerProto::Serialise::write(store, conn, key); - } -} - -template -std::map WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - std::map resMap; - auto size = readNum(conn.from); - while (size--) { - auto k = WorkerProto::Serialise::read(store, conn); - auto v = WorkerProto::Serialise::read(store, conn); - resMap.insert_or_assign(std::move(k), std::move(v)); - } - return resMap; -} - -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::map & resMap) -{ - conn.to << resMap.size(); - for (auto & i : resMap) { - WorkerProto::Serialise::write(store, conn, i.first); - WorkerProto::Serialise::write(store, conn, i.second); - } -} - -template -std::tuple WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - return std::tuple { - WorkerProto::Serialise::read(store, conn)..., - }; -} - -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::tuple & res) -{ - std::apply([&](const Us &... args) { - (WorkerProto::Serialise::write(store, conn, args), ...); - }, res); -} +/* protocol-specific templates */ } diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index a23130743..415e66f16 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -12,27 +12,7 @@ namespace nix { -std::string WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return readString(conn.from); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const std::string & str) -{ - conn.to << str; -} - - -StorePath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return store.parseStorePath(readString(conn.from)); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const StorePath & storePath) -{ - conn.to << store.printStorePath(storePath); -} - +/* protocol-specific definitions */ std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) { @@ -68,17 +48,6 @@ void WorkerProto::Serialise>::write(const Store & sto } -ContentAddress WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return ContentAddress::parse(readString(conn.from)); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const ContentAddress & ca) -{ - conn.to << renderContentAddress(ca); -} - - DerivedPath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { auto s = readString(conn.from); @@ -91,32 +60,6 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto } -Realisation WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - std::string rawInput = readString(conn.from); - return Realisation::fromJSON( - nlohmann::json::parse(rawInput), - "remote-protocol" - ); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const Realisation & realisation) -{ - conn.to << realisation.toJSON().dump(); -} - - -DrvOutput WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return DrvOutput::parse(readString(conn.from)); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const DrvOutput & drvOutput) -{ - conn.to << drvOutput.to_string(); -} - - KeyedBuildResult WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { auto path = WorkerProto::Serialise::read(store, conn); @@ -168,26 +111,4 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto } -std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - auto s = readString(conn.from); - return s == "" ? std::optional {} : store.parseStorePath(s); -} - -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & storePathOpt) -{ - conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); -} - - -std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - return ContentAddress::parseOpt(readString(conn.from)); -} - -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & caOpt) -{ - conn.to << (caOpt ? renderContentAddress(*caOpt) : ""); -} - } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index b7f42f24d..c84060103 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "serialise.hh" +#include "common-protocol.hh" namespace nix { @@ -28,11 +28,7 @@ class Store; struct Source; // items being serialised -class StorePath; -struct ContentAddress; struct DerivedPath; -struct DrvOutput; -struct Realisation; struct BuildResult; struct KeyedBuildResult; enum TrustedFlag : bool; @@ -193,60 +189,32 @@ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op) * be legal specialization syntax. See below for what that looks like in * practice. */ -#define MAKE_WORKER_PROTO(T) \ - struct WorkerProto::Serialise< T > { \ +#define DECLARE_WORKER_SERIALISER(T) \ + struct WorkerProto::Serialise< T > \ + { \ static T read(const Store & store, WorkerProto::ReadConn conn); \ static void write(const Store & store, WorkerProto::WriteConn conn, const T & t); \ }; template<> -MAKE_WORKER_PROTO(std::string); +DECLARE_WORKER_SERIALISER(DerivedPath); template<> -MAKE_WORKER_PROTO(StorePath); +DECLARE_WORKER_SERIALISER(BuildResult); template<> -MAKE_WORKER_PROTO(ContentAddress); +DECLARE_WORKER_SERIALISER(KeyedBuildResult); template<> -MAKE_WORKER_PROTO(DerivedPath); -template<> -MAKE_WORKER_PROTO(DrvOutput); -template<> -MAKE_WORKER_PROTO(Realisation); -template<> -MAKE_WORKER_PROTO(BuildResult); -template<> -MAKE_WORKER_PROTO(KeyedBuildResult); -template<> -MAKE_WORKER_PROTO(std::optional); +DECLARE_WORKER_SERIALISER(std::optional); template -MAKE_WORKER_PROTO(std::vector); +DECLARE_WORKER_SERIALISER(std::vector); template -MAKE_WORKER_PROTO(std::set); +DECLARE_WORKER_SERIALISER(std::set); template -MAKE_WORKER_PROTO(std::tuple); +DECLARE_WORKER_SERIALISER(std::tuple); +#define COMMA_ , template -#define X_ std::map -MAKE_WORKER_PROTO(X_); -#undef X_ - -/** - * These use the empty string for the null case, relying on the fact - * that the underlying types never serialise to the empty string. - * - * We do this instead of a generic std::optional instance because - * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For - * the same reason, we don't have a std::variant instances (ordinal - * tags 0...n). - * - * We could the generic instances and then these as specializations for - * compatability, but that's proven a bit finnicky, and also makes the - * worker protocol harder to implement in other languages where such - * specializations may not be allowed. - */ -template<> -MAKE_WORKER_PROTO(std::optional); -template<> -MAKE_WORKER_PROTO(std::optional); +DECLARE_WORKER_SERIALISER(std::map); +#undef COMMA_ } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 96c3f7d7e..6fc22214a 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -11,8 +11,8 @@ #include "serve-protocol.hh" #include "shared.hh" #include "util.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include "graphml.hh" #include "legacy.hh" #include "path-with-outputs.hh" @@ -821,8 +821,8 @@ static void opServe(Strings opFlags, Strings opArgs) out.flush(); unsigned int clientVersion = readInt(in); - WorkerProto::ReadConn rconn { .from = in }; - WorkerProto::WriteConn wconn { .to = out }; + CommonProto::ReadConn rconn { .from = in }; + CommonProto::WriteConn wconn { .to = out }; auto getBuildSettings = [&]() { // FIXME: changing options here doesn't work if we're @@ -867,7 +867,7 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::QueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - auto paths = WorkerProto::Serialise::read(*store, rconn); + auto paths = CommonProto::Serialise::read(*store, rconn); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -876,19 +876,19 @@ static void opServe(Strings opFlags, Strings opArgs) store->substitutePaths(paths); } - WorkerProto::write(*store, wconn, store->queryValidPaths(paths)); + CommonProto::write(*store, wconn, store->queryValidPaths(paths)); break; } case ServeProto::Command::QueryPathInfos: { - auto paths = WorkerProto::Serialise::read(*store, rconn); + auto paths = CommonProto::Serialise::read(*store, rconn); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - WorkerProto::write(*store, wconn, info->references); + CommonProto::write(*store, wconn, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -916,7 +916,7 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::ExportPaths: { readInt(in); // obsolete - store->exportPaths(WorkerProto::Serialise::read(*store, rconn), out); + store->exportPaths(CommonProto::Serialise::read(*store, rconn), out); break; } @@ -962,7 +962,7 @@ static void opServe(Strings opFlags, Strings opArgs) DrvOutputs builtOutputs; for (auto & [output, realisation] : status.builtOutputs) builtOutputs.insert_or_assign(realisation.id, realisation); - WorkerProto::write(*store, wconn, builtOutputs); + CommonProto::write(*store, wconn, builtOutputs); } break; @@ -971,9 +971,9 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::QueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; - store->computeFSClosure(WorkerProto::Serialise::read(*store, rconn), + store->computeFSClosure(CommonProto::Serialise::read(*store, rconn), closure, false, includeOutputs); - WorkerProto::write(*store, wconn, closure); + CommonProto::write(*store, wconn, closure); break; } @@ -988,7 +988,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(*store, rconn); + info.references = CommonProto::Serialise::read(*store, rconn); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = ContentAddress::parseOpt(readString(in)); From 4de54b21907e3fc41b9e6250396eb01d2b10c4db Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 4 Oct 2023 23:27:50 -0400 Subject: [PATCH 368/909] Unit test the "common protocol" too Copy the relevant tests to ensure the new interfaces added in the last commit are tested. Perhaps I should try to deduplicat these tests some more. However its not clear how to do that outside of a big ugly C++ macro. https://github.com/google/googletest/blob/main/docs/advanced.md has some stuff but it is cumbersome and I didn't figure it out yet. This is done in a separate commit in order to be sure that the first commit really didn't change any behavior; if we changed the implementation and the tests at once, it would be harder to tell whether or not some behavioral changes slipped in what is supposed to be a "pure refactor". Co-Authored-By: Valentin Gagarin --- src/libstore/tests/characterization.hh | 23 +++ src/libstore/tests/common-protocol.cc | 152 ++++++++++++++++++ src/libstore/tests/protocol.hh | 88 ++++++++++ src/libstore/tests/worker-protocol.cc | 90 ++--------- .../common-protocol/content-address.bin | Bin 0 -> 208 bytes .../libstore/common-protocol/drv-output.bin | Bin 0 -> 176 bytes .../optional-content-address.bin | Bin 0 -> 64 bytes .../common-protocol/optional-store-path.bin | Bin 0 -> 72 bytes .../libstore/common-protocol/realisation.bin | Bin 0 -> 520 bytes .../libstore/common-protocol/set.bin | Bin 0 -> 152 bytes .../libstore/common-protocol/store-path.bin | Bin 0 -> 120 bytes .../libstore/common-protocol/string.bin | Bin 0 -> 88 bytes .../libstore/common-protocol/vector.bin | Bin 0 -> 152 bytes 13 files changed, 280 insertions(+), 73 deletions(-) create mode 100644 src/libstore/tests/characterization.hh create mode 100644 src/libstore/tests/common-protocol.cc create mode 100644 src/libstore/tests/protocol.hh create mode 100644 unit-test-data/libstore/common-protocol/content-address.bin create mode 100644 unit-test-data/libstore/common-protocol/drv-output.bin create mode 100644 unit-test-data/libstore/common-protocol/optional-content-address.bin create mode 100644 unit-test-data/libstore/common-protocol/optional-store-path.bin create mode 100644 unit-test-data/libstore/common-protocol/realisation.bin create mode 100644 unit-test-data/libstore/common-protocol/set.bin create mode 100644 unit-test-data/libstore/common-protocol/store-path.bin create mode 100644 unit-test-data/libstore/common-protocol/string.bin create mode 100644 unit-test-data/libstore/common-protocol/vector.bin diff --git a/src/libstore/tests/characterization.hh b/src/libstore/tests/characterization.hh new file mode 100644 index 000000000..5f366cb42 --- /dev/null +++ b/src/libstore/tests/characterization.hh @@ -0,0 +1,23 @@ +#pragma once +///@file + +namespace nix { + +/** + * The path to the `unit-test-data` directory. See the contributing + * guide in the manual for further details. + */ +static Path getUnitTestData() { + return getEnv("_NIX_TEST_UNIT_DATA").value(); +} + +/** + * Whether we should update "golden masters" instead of running tests + * against them. See the contributing guide in the manual for further + * details. + */ +static bool testAccept() { + return getEnv("_NIX_TEST_ACCEPT") == "1"; +} + +} diff --git a/src/libstore/tests/common-protocol.cc b/src/libstore/tests/common-protocol.cc new file mode 100644 index 000000000..de57211f0 --- /dev/null +++ b/src/libstore/tests/common-protocol.cc @@ -0,0 +1,152 @@ +#include + +#include +#include + +#include "common-protocol.hh" +#include "common-protocol-impl.hh" +#include "build-result.hh" +#include "tests/protocol.hh" +#include "tests/characterization.hh" + +namespace nix { + +const char commonProtoDir[] = "common-protocol"; + +using CommonProtoTest = ProtoTest; + +CHARACTERIZATION_TEST( + CommonProtoTest, + string, + "string", + (std::tuple { + "", + "hi", + "white rabbit", + "大白兔", + "oh no \0\0\0 what was that!", + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + storePath, + "store-path", + (std::tuple { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + contentAddress, + "content-address", + (std::tuple { + ContentAddress { + .method = TextIngestionMethod {}, + .hash = hashString(HashType::htSHA256, "Derive(...)"), + }, + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + ContentAddress { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + drvOutput, + "drv-output", + (std::tuple { + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + realisation, + "realisation", + (std::tuple { + Realisation { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + }, + Realisation { + .id = { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + .dependentRealisations = { + { + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + vector, + "vector", + (std::tuple, std::vector, std::vector, std::vector>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + set, + "set", + (std::tuple, std::set, std::set, std::set>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + optionalStorePath, + "optional-store-path", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + }, + })) + +CHARACTERIZATION_TEST( + CommonProtoTest, + optionalContentAddress, + "optional-content-address", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + }, + })) + +} diff --git a/src/libstore/tests/protocol.hh b/src/libstore/tests/protocol.hh new file mode 100644 index 000000000..86a900757 --- /dev/null +++ b/src/libstore/tests/protocol.hh @@ -0,0 +1,88 @@ +#include +#include + +#include "tests/libstore.hh" +#include "tests/characterization.hh" + +namespace nix { + +template +class ProtoTest : public LibStoreTest +{ + /** + * Read this as simply `using S = Inner::Serialise;`. + * + * See `LengthPrefixedProtoHelper::S` for the same trick, and its + * rationale. + */ + template using S = typename Proto::template Serialise; + +public: + Path unitTestData = getUnitTestData() + "/libstore/" + protocolDir; + + Path goldenMaster(std::string_view testStem) { + return unitTestData + "/" + testStem + ".bin"; + } + + /** + * Golden test for `T` reading + */ + template + void readTest(PathView testStem, T value) + { + if (testAccept()) + { + GTEST_SKIP() << "Cannot read golden master because another test is also updating it"; + } + else + { + auto expected = readFile(goldenMaster(testStem)); + + T got = ({ + StringSource from { expected }; + S::read( + *store, + typename Proto::ReadConn { .from = from }); + }); + + ASSERT_EQ(got, value); + } + } + + /** + * Golden test for `T` write + */ + template + void writeTest(PathView testStem, const T & value) + { + auto file = goldenMaster(testStem); + + StringSink to; + Proto::write( + *store, + typename Proto::WriteConn { .to = to }, + value); + + if (testAccept()) + { + createDirs(dirOf(file)); + writeFile(file, to.s); + GTEST_SKIP() << "Updating golden master"; + } + else + { + auto expected = readFile(file); + ASSERT_EQ(to.s, expected); + } + } +}; + +#define CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VALUE) \ + TEST_F(FIXTURE, NAME ## _read) { \ + readTest(STEM, VALUE); \ + } \ + TEST_F(FIXTURE, NAME ## _write) { \ + writeTest(STEM, VALUE); \ + } + +} diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index fa7cbe121..59d7ff96d 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -7,85 +7,17 @@ #include "worker-protocol-impl.hh" #include "derived-path.hh" #include "build-result.hh" -#include "tests/libstore.hh" +#include "tests/protocol.hh" +#include "tests/characterization.hh" namespace nix { -class WorkerProtoTest : public LibStoreTest -{ -public: - Path unitTestData = getEnv("_NIX_TEST_UNIT_DATA").value() + "/libstore/worker-protocol"; +const char workerProtoDir[] = "worker-protocol"; - bool testAccept() { - return getEnv("_NIX_TEST_ACCEPT") == "1"; - } - - Path goldenMaster(std::string_view testStem) { - return unitTestData + "/" + testStem + ".bin"; - } - - /** - * Golden test for `T` reading - */ - template - void readTest(PathView testStem, T value) - { - if (testAccept()) - { - GTEST_SKIP() << "Cannot read golden master because another test is also updating it"; - } - else - { - auto expected = readFile(goldenMaster(testStem)); - - T got = ({ - StringSource from { expected }; - WorkerProto::Serialise::read( - *store, - WorkerProto::ReadConn { .from = from }); - }); - - ASSERT_EQ(got, value); - } - } - - /** - * Golden test for `T` write - */ - template - void writeTest(PathView testStem, const T & value) - { - auto file = goldenMaster(testStem); - - StringSink to; - WorkerProto::write( - *store, - WorkerProto::WriteConn { .to = to }, - value); - - if (testAccept()) - { - createDirs(dirOf(file)); - writeFile(file, to.s); - GTEST_SKIP() << "Updating golden master"; - } - else - { - auto expected = readFile(file); - ASSERT_EQ(to.s, expected); - } - } -}; - -#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ - TEST_F(WorkerProtoTest, NAME ## _read) { \ - readTest(STEM, VALUE); \ - } \ - TEST_F(WorkerProtoTest, NAME ## _write) { \ - writeTest(STEM, VALUE); \ - } +using WorkerProtoTest = ProtoTest; CHARACTERIZATION_TEST( + WorkerProtoTest, string, "string", (std::tuple { @@ -97,6 +29,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, storePath, "store-path", (std::tuple { @@ -105,6 +38,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, contentAddress, "content-address", (std::tuple { @@ -123,6 +57,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, derivedPath, "derived-path", (std::tuple { @@ -138,6 +73,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, drvOutput, "drv-output", (std::tuple { @@ -152,6 +88,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, realisation, "realisation", (std::tuple { @@ -183,6 +120,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, buildResult, "build-result", ({ @@ -240,6 +178,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, keyedBuildResult, "keyed-build-result", ({ @@ -275,6 +214,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, optionalTrustedFlag, "optional-trusted-flag", (std::tuple, std::optional, std::optional> { @@ -284,6 +224,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, vector, "vector", (std::tuple, std::vector, std::vector, std::vector>> { @@ -294,6 +235,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, set, "set", (std::tuple, std::set, std::set, std::set>> { @@ -304,6 +246,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, optionalStorePath, "optional-store-path", (std::tuple, std::optional> { @@ -314,6 +257,7 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( + WorkerProtoTest, optionalContentAddress, "optional-content-address", (std::tuple, std::optional> { diff --git a/unit-test-data/libstore/common-protocol/content-address.bin b/unit-test-data/libstore/common-protocol/content-address.bin new file mode 100644 index 0000000000000000000000000000000000000000..8f14bcdb3e50cc72d87106ce159a6fd7b9f75713 GIT binary patch literal 208 zcmY+;K@P$o5QSmy;|7WYrYjRqQfMm$+LWPzDW_MfG4bucKks(>Y#V56lkFOiEzi24 zUMdC5VUCa86OJ&gL0BF^B{HBQEY%f|I<6v7#q+l_PBirI5N~-md%37WE|g33QTE2k zc~-WF&R}7Oxc@o)T?ojpD!+*5=xpO;%IoCKSV5FMh={>xePK|8<${>E)*huNHZ(pX literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/common-protocol/drv-output.bin b/unit-test-data/libstore/common-protocol/drv-output.bin new file mode 100644 index 0000000000000000000000000000000000000000..800a45fd8757a15e8810066aed48be56a600b7d3 GIT binary patch literal 176 zcmY++xeWp_5I|8{*$(Wn=b{B@!gGlfp_LHTb8N(qe)KM%U7Sq@}q)U|6h9n2j1!MT?w+C~^gO WO9=?G&ojrYjy`x4ZufnEe#tjMpDPdm literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/common-protocol/optional-content-address.bin b/unit-test-data/libstore/common-protocol/optional-content-address.bin new file mode 100644 index 0000000000000000000000000000000000000000..f8cfe65ba27fd78ec7787eed7ebec8dfdead2fba GIT binary patch literal 64 zcmZQzfB#ECTBU06lLKJOBUy literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/common-protocol/realisation.bin b/unit-test-data/libstore/common-protocol/realisation.bin new file mode 100644 index 0000000000000000000000000000000000000000..2176c6c4afd96b32fe372de6084ac7f4c7a11d49 GIT binary patch literal 520 zcmdUrO-{o=429t+opq7s&z_l_03#3x z?rW}!>Uc!$VBY>Sr2mUC`<2<)qY;)1KN Po44)luetw6d9AunL$;dR literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/common-protocol/set.bin b/unit-test-data/libstore/common-protocol/set.bin new file mode 100644 index 0000000000000000000000000000000000000000..ce11ede7fe7aa061a30b211a18cd7a336f879de8 GIT binary patch literal 152 ucmZQzfB;4)4WpQ03@8obCnXkvMPU52{CpHXOdBEdVDg4g4KThDln(&+h63;a literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/common-protocol/store-path.bin b/unit-test-data/libstore/common-protocol/store-path.bin new file mode 100644 index 0000000000000000000000000000000000000000..3fc05f2981d3398ab30ec34125c903cdcefa8729 GIT binary patch literal 120 tcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7Eu*F?srQlM;)-IsqP{BM|@q literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/common-protocol/string.bin b/unit-test-data/libstore/common-protocol/string.bin new file mode 100644 index 0000000000000000000000000000000000000000..aa7b5a604745b735473ec34283177e69320776e4 GIT binary patch literal 88 zcmZQzfB+^aoskJ)@Id+H8JQ)i3Pp)YNtq=eAx^0H( Date: Tue, 10 Oct 2023 00:32:36 +0200 Subject: [PATCH 369/909] annotate admonitions showing syntax also fix typos --- .../src/language/constructs/lookup-path.md | 2 +- doc/manual/src/language/operators.md | 16 ++++++++++++++++ doc/manual/src/language/values.md | 6 +++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/src/language/constructs/lookup-path.md index 8ec835309..e87d2922b 100644 --- a/doc/manual/src/language/constructs/lookup-path.md +++ b/doc/manual/src/language/constructs/lookup-path.md @@ -2,7 +2,7 @@ > **Syntax** > -> *lookup-path* = `<` *identifier* [ `/` *identifier* `]`... `>` +> *lookup-path* = `<` *identifier* [ `/` *identifier* ]... `>` A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/values.md#type-path) if the identifier matches a search path entry. diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index f8382ae19..a22c71109 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -35,6 +35,8 @@ ## Attribute selection +> **Syntax** +> > *attrset* `.` *attrpath* \[ `or` *expr* \] Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. @@ -42,12 +44,16 @@ If the attribute doesn’t exist, return the *expr* after `or` if provided, othe An attribute path is a dot-separated list of [attribute names](./values.md#attribute-set). +> **Syntax** +> > *attrpath* = *name* [ `.` *name* ]... [Attribute selection]: #attribute-selection ## Has attribute +> **Syntax** +> > *attrset* `?` *attrpath* Test whether [attribute set] *attrset* contains the attribute denoted by *attrpath*. @@ -70,6 +76,8 @@ The `+` operator is overloaded to also work on strings and paths. ## String concatenation +> **Syntax** +> > *string* `+` *string* Concatenate two [string]s and merge their string contexts. @@ -78,6 +86,8 @@ Concatenate two [string]s and merge their string contexts. ## Path concatenation +> **Syntax** +> > *path* `+` *path* Concatenate two [path]s. @@ -87,6 +97,8 @@ The result is a path. ## Path and string concatenation +> **Syntax** +> > *path* + *string* Concatenate *[path]* with *[string]*. @@ -100,6 +112,8 @@ The result is a path. ## String and path concatenation +> **Syntax** +> > *string* + *path* Concatenate *[string]* with *[path]*. @@ -117,6 +131,8 @@ The result is a string. ## Update +> **Syntax** +> > *attrset1* // *attrset2* Update [attribute set] *attrset1* with names and values from *attrset2*. diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md index 8856d33ae..0bb656746 100644 --- a/doc/manual/src/language/values.md +++ b/doc/manual/src/language/values.md @@ -162,13 +162,17 @@ An attribute set is a collection of name-value-pairs (called *attributes*) enclo An attribute name can be an identifier or a [string](#string). An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). +> **Syntax** +> > *name* = *identifier* | *string* \ > *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` Names and values are separated by an equal sign (`=`). Each value is an arbitrary expression terminated by a semicolon (`;`). -> *attrset* = `{` [ *name* `=` *expr* `;` `]`... `}` +> **Syntax** +> +> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}` Attributes can appear in any order. An attribute name may only occur once. From 7642894a4e1c9916f92a234c8cbaef604a7b7d00 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 7 Oct 2023 03:49:49 +0200 Subject: [PATCH 370/909] reword documentation on lookup path resolution --- src/libexpr/primops.cc | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index bfcac7510..3bed7c5aa 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1698,13 +1698,14 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V static RegisterPrimOp primop_findFile(PrimOp { .name = "__findFile", - .args = {"search path", "lookup path"}, + .args = {"search-path", "lookup-path"}, .doc = R"( - Look up the given path with the given search path. + Find *lookup-path* in *search-path*. - A search path is represented list of [attribute sets](./values.md#attribute-set) with two attributes, `prefix`, and `path`. - `prefix` is a relative path. - `path` denotes a file system location; the exact syntax depends on the command line interface. + A search path is represented list of [attribute sets](./values.md#attribute-set) with two attributes: + - `prefix` is a relative path. + - `path` denotes a file system location + The exact syntax depends on the command line interface. Examples of search path attribute sets: @@ -1722,13 +1723,12 @@ static RegisterPrimOp primop_findFile(PrimOp { } ``` - The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/values.html#type-path) of the match. + The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/values.html#type-path) of the match: - This is the process for each entry: - If the lookup path matches `prefix`, then the remainder of the lookup path (the "suffix") is searched for within the directory denoted by `patch`. - Note that the `path` may need to be downloaded at this point to look inside. - If the suffix is found inside that directory, then the entry is a match; - the combined absolute path of the directory (now downloaded if need be) and the suffix is returned. + - If *lookup-path* matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`. + Note that the `path` may need to be downloaded at this point to look inside. + - If the suffix is found inside that directory, then the entry is a match. + The combined absolute path of the directory (now downloaded if need be) and the suffix is returned. [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions can be [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath): From f7b8f8aff63b1a4c7289c423d7877c95c0c6ae1f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 26 May 2023 14:11:08 -0400 Subject: [PATCH 371/909] Introduce separate Serve protocol serialisers To start, it is just a clone of the common protocol. But now that we have the separate protocol implementations, we can add versioning information without the versions of one protocol leaking into another. Using the infrastructure from the previous commit, we don't have to duplicate code for shared behavior. Motivation: No more perverse incentives. [0] did some awkward things because the serialisers did not store the version. I don't want anyone making changes to be pushed towards keeping the serialization logic with the core data types just because it's easier or the alternative is tedious. The actual versioning of the Worker and Serve protocol serialisers (Common remains unversioned as the underlying mini-protocols are not versioned) will happen in subsequent commits / PRs. [0]: fe1f34fa60ad79e339c38e58af071a44774663f7 --- src/libstore/legacy-ssh-store.cc | 45 +++--- src/libstore/serve-protocol-impl.hh | 59 +++++++ src/libstore/serve-protocol.cc | 15 ++ src/libstore/serve-protocol.hh | 87 ++++++++++ src/libstore/tests/serve-protocol.cc | 152 ++++++++++++++++++ src/nix-store/nix-store.cc | 25 ++- .../serve-protocol/content-address.bin | Bin 0 -> 208 bytes .../libstore/serve-protocol/drv-output.bin | Bin 0 -> 176 bytes .../optional-content-address.bin | Bin 0 -> 64 bytes .../serve-protocol/optional-store-path.bin | Bin 0 -> 72 bytes .../libstore/serve-protocol/realisation.bin | Bin 0 -> 520 bytes .../libstore/serve-protocol/set.bin | Bin 0 -> 152 bytes .../libstore/serve-protocol/store-path.bin | Bin 0 -> 120 bytes .../libstore/serve-protocol/string.bin | Bin 0 -> 88 bytes .../libstore/serve-protocol/vector.bin | Bin 0 -> 152 bytes 15 files changed, 344 insertions(+), 39 deletions(-) create mode 100644 src/libstore/serve-protocol-impl.hh create mode 100644 src/libstore/serve-protocol.cc create mode 100644 src/libstore/tests/serve-protocol.cc create mode 100644 unit-test-data/libstore/serve-protocol/content-address.bin create mode 100644 unit-test-data/libstore/serve-protocol/drv-output.bin create mode 100644 unit-test-data/libstore/serve-protocol/optional-content-address.bin create mode 100644 unit-test-data/libstore/serve-protocol/optional-store-path.bin create mode 100644 unit-test-data/libstore/serve-protocol/realisation.bin create mode 100644 unit-test-data/libstore/serve-protocol/set.bin create mode 100644 unit-test-data/libstore/serve-protocol/store-path.bin create mode 100644 unit-test-data/libstore/serve-protocol/string.bin create mode 100644 unit-test-data/libstore/serve-protocol/vector.bin diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 7bf4476d0..703ded0b2 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -3,11 +3,10 @@ #include "pool.hh" #include "remote-store.hh" #include "serve-protocol.hh" +#include "serve-protocol-impl.hh" #include "build-result.hh" #include "store-api.hh" #include "path-with-outputs.hh" -#include "common-protocol.hh" -#include "common-protocol-impl.hh" #include "ssh.hh" #include "derivations.hh" #include "callback.hh" @@ -50,37 +49,31 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor bool good = true; /** - * Coercion to `CommonProto::ReadConn`. This makes it easy to use the - * factored out common protocol serialisers with a + * Coercion to `ServeProto::ReadConn`. This makes it easy to use the + * factored out serve protocol searlizers with a * `LegacySSHStore::Connection`. * - * The common protocol connection types are unidirectional, unlike + * The serve protocol connection types are unidirectional, unlike * this type. - * - * @todo Use server protocol serializers, not common protocol - * serializers, once we have made that distiction. */ - operator CommonProto::ReadConn () + operator ServeProto::ReadConn () { - return CommonProto::ReadConn { + return ServeProto::ReadConn { .from = from, }; } /* - * Coercion to `CommonProto::WriteConn`. This makes it easy to use the - * factored out common protocol searlizers with a + * Coercion to `ServeProto::WriteConn`. This makes it easy to use the + * factored out serve protocol searlizers with a * `LegacySSHStore::Connection`. * - * The common protocol connection types are unidirectional, unlike + * The serve protocol connection types are unidirectional, unlike * this type. - * - * @todo Use server protocol serializers, not common protocol - * serializers, once we have made that distiction. */ - operator CommonProto::WriteConn () + operator ServeProto::WriteConn () { - return CommonProto::WriteConn { + return ServeProto::WriteConn { .to = to, }; } @@ -183,7 +176,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = CommonProto::Serialise::read(*this, *conn); + info->references = ServeProto::Serialise::read(*this, *conn); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -217,7 +210,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - CommonProto::write(*this, *conn, info.references); + ServeProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize @@ -246,7 +239,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - CommonProto::write(*this, *conn, info.references); + ServeProto::write(*this, *conn, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -331,7 +324,7 @@ public: if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - auto builtOutputs = CommonProto::Serialise::read(*this, *conn); + auto builtOutputs = ServeProto::Serialise::read(*this, *conn); for (auto && [output, realisation] : builtOutputs) status.builtOutputs.insert_or_assign( std::move(output.outputName), @@ -409,10 +402,10 @@ public: conn->to << ServeProto::Command::QueryClosure << includeOutputs; - CommonProto::write(*this, *conn, paths); + ServeProto::write(*this, *conn, paths); conn->to.flush(); - for (auto & i : CommonProto::Serialise::read(*this, *conn)) + for (auto & i : ServeProto::Serialise::read(*this, *conn)) out.insert(i); } @@ -425,10 +418,10 @@ public: << ServeProto::Command::QueryValidPaths << false // lock << maybeSubstitute; - CommonProto::write(*this, *conn, paths); + ServeProto::write(*this, *conn, paths); conn->to.flush(); - return CommonProto::Serialise::read(*this, *conn); + return ServeProto::Serialise::read(*this, *conn); } void connect() override diff --git a/src/libstore/serve-protocol-impl.hh b/src/libstore/serve-protocol-impl.hh new file mode 100644 index 000000000..a3ce81026 --- /dev/null +++ b/src/libstore/serve-protocol-impl.hh @@ -0,0 +1,59 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * This file is an exmample of the "impl.hh" pattern. See the + * contributing guide. + */ + +#include "serve-protocol.hh" +#include "length-prefixed-protocol-helper.hh" + +namespace nix { + +/* protocol-agnostic templates */ + +#define SERVE_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ + TEMPLATE T ServeProto::Serialise< T >::read(const Store & store, ServeProto::ReadConn conn) \ + { \ + return LengthPrefixedProtoHelper::read(store, conn); \ + } \ + TEMPLATE void ServeProto::Serialise< T >::write(const Store & store, ServeProto::WriteConn conn, const T & t) \ + { \ + LengthPrefixedProtoHelper::write(store, conn, t); \ + } + +SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) + +#define COMMA_ , +SERVE_USE_LENGTH_PREFIX_SERIALISER( + template, + std::map) +#undef COMMA_ + +/** + * Use `CommonProto` where possible. + */ +template +struct ServeProto::Serialise +{ + static T read(const Store & store, ServeProto::ReadConn conn) + { + return CommonProto::Serialise::read(store, + CommonProto::ReadConn { .from = conn.from }); + } + static void write(const Store & store, ServeProto::WriteConn conn, const T & t) + { + CommonProto::Serialise::write(store, + CommonProto::WriteConn { .to = conn.to }, + t); + } +}; + +/* protocol-specific templates */ + +} diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc new file mode 100644 index 000000000..16a62b5bc --- /dev/null +++ b/src/libstore/serve-protocol.cc @@ -0,0 +1,15 @@ +#include "serialise.hh" +#include "util.hh" +#include "path-with-outputs.hh" +#include "store-api.hh" +#include "serve-protocol.hh" +#include "serve-protocol-impl.hh" +#include "archive.hh" + +#include + +namespace nix { + +/* protocol-specific definitions */ + +} diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 7e43b3969..e2345d450 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include "common-protocol.hh" + namespace nix { #define SERVE_MAGIC_1 0x390c9deb @@ -10,6 +12,11 @@ namespace nix { #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) + +class Store; +struct Source; + + /** * The "serve protocol", used by ssh:// stores. * @@ -22,6 +29,57 @@ struct ServeProto * Enumeration of all the request types for the protocol. */ enum struct Command : uint64_t; + + /** + * A unidirectional read connection, to be used by the read half of the + * canonical serializers below. + * + * This currently is just a `Source &`, but more fields will be added + * later. + */ + struct ReadConn { + Source & from; + }; + + /** + * A unidirectional write connection, to be used by the write half of the + * canonical serializers below. + * + * This currently is just a `Sink &`, but more fields will be added + * later. + */ + struct WriteConn { + Sink & to; + }; + + /** + * Data type for canonical pairs of serialisers for the serve protocol. + * + * See https://en.cppreference.com/w/cpp/language/adl for the broader + * concept of what is going on here. + */ + template + struct Serialise; + // This is the definition of `Serialise` we *want* to put here, but + // do not do so. + // + // See `worker-protocol.hh` for a longer explanation. +#if 0 + { + static T read(const Store & store, ReadConn conn); + static void write(const Store & store, WriteConn conn, const T & t); + }; +#endif + + /** + * Wrapper function around `ServeProto::Serialise::write` that allows us to + * infer the type instead of having to write it down explicitly. + */ + template + static void write(const Store & store, WriteConn conn, const T & t) + { + ServeProto::Serialise::write(store, conn, t); + } }; enum struct ServeProto::Command : uint64_t @@ -58,4 +116,33 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op) return s << (uint64_t) op; } +/** + * Declare a canonical serialiser pair for the worker protocol. + * + * We specialise the struct merely to indicate that we are implementing + * the function for the given type. + * + * Some sort of `template<...>` must be used with the caller for this to + * be legal specialization syntax. See below for what that looks like in + * practice. + */ +#define DECLARE_SERVE_SERIALISER(T) \ + struct ServeProto::Serialise< T > \ + { \ + static T read(const Store & store, ServeProto::ReadConn conn); \ + static void write(const Store & store, ServeProto::WriteConn conn, const T & t); \ + }; + +template +DECLARE_SERVE_SERIALISER(std::vector); +template +DECLARE_SERVE_SERIALISER(std::set); +template +DECLARE_SERVE_SERIALISER(std::tuple); + +#define COMMA_ , +template +DECLARE_SERVE_SERIALISER(std::map); +#undef COMMA_ + } diff --git a/src/libstore/tests/serve-protocol.cc b/src/libstore/tests/serve-protocol.cc new file mode 100644 index 000000000..5d7df56cf --- /dev/null +++ b/src/libstore/tests/serve-protocol.cc @@ -0,0 +1,152 @@ +#include + +#include +#include + +#include "serve-protocol.hh" +#include "serve-protocol-impl.hh" +#include "build-result.hh" +#include "tests/protocol.hh" +#include "tests/characterization.hh" + +namespace nix { + +const char commonProtoDir[] = "serve-protocol"; + +using ServeProtoTest = ProtoTest; + +CHARACTERIZATION_TEST( + ServeProtoTest, + string, + "string", + (std::tuple { + "", + "hi", + "white rabbit", + "大白兔", + "oh no \0\0\0 what was that!", + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + storePath, + "store-path", + (std::tuple { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + contentAddress, + "content-address", + (std::tuple { + ContentAddress { + .method = TextIngestionMethod {}, + .hash = hashString(HashType::htSHA256, "Derive(...)"), + }, + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + ContentAddress { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + drvOutput, + "drv-output", + (std::tuple { + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + realisation, + "realisation", + (std::tuple { + Realisation { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + }, + Realisation { + .id = { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + .dependentRealisations = { + { + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + vector, + "vector", + (std::tuple, std::vector, std::vector, std::vector>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + set, + "set", + (std::tuple, std::set, std::set, std::set>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + optionalStorePath, + "optional-store-path", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + }, + })) + +CHARACTERIZATION_TEST( + ServeProtoTest, + optionalContentAddress, + "optional-content-address", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + }, + })) + +} diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 6fc22214a..9b6c80a75 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -9,10 +9,9 @@ #include "local-store.hh" #include "monitor-fd.hh" #include "serve-protocol.hh" +#include "serve-protocol-impl.hh" #include "shared.hh" #include "util.hh" -#include "common-protocol.hh" -#include "common-protocol-impl.hh" #include "graphml.hh" #include "legacy.hh" #include "path-with-outputs.hh" @@ -821,8 +820,8 @@ static void opServe(Strings opFlags, Strings opArgs) out.flush(); unsigned int clientVersion = readInt(in); - CommonProto::ReadConn rconn { .from = in }; - CommonProto::WriteConn wconn { .to = out }; + ServeProto::ReadConn rconn { .from = in }; + ServeProto::WriteConn wconn { .to = out }; auto getBuildSettings = [&]() { // FIXME: changing options here doesn't work if we're @@ -867,7 +866,7 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::QueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - auto paths = CommonProto::Serialise::read(*store, rconn); + auto paths = ServeProto::Serialise::read(*store, rconn); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -876,19 +875,19 @@ static void opServe(Strings opFlags, Strings opArgs) store->substitutePaths(paths); } - CommonProto::write(*store, wconn, store->queryValidPaths(paths)); + ServeProto::write(*store, wconn, store->queryValidPaths(paths)); break; } case ServeProto::Command::QueryPathInfos: { - auto paths = CommonProto::Serialise::read(*store, rconn); + auto paths = ServeProto::Serialise::read(*store, rconn); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - CommonProto::write(*store, wconn, info->references); + ServeProto::write(*store, wconn, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -916,7 +915,7 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::ExportPaths: { readInt(in); // obsolete - store->exportPaths(CommonProto::Serialise::read(*store, rconn), out); + store->exportPaths(ServeProto::Serialise::read(*store, rconn), out); break; } @@ -962,7 +961,7 @@ static void opServe(Strings opFlags, Strings opArgs) DrvOutputs builtOutputs; for (auto & [output, realisation] : status.builtOutputs) builtOutputs.insert_or_assign(realisation.id, realisation); - CommonProto::write(*store, wconn, builtOutputs); + ServeProto::write(*store, wconn, builtOutputs); } break; @@ -971,9 +970,9 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::QueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; - store->computeFSClosure(CommonProto::Serialise::read(*store, rconn), + store->computeFSClosure(ServeProto::Serialise::read(*store, rconn), closure, false, includeOutputs); - CommonProto::write(*store, wconn, closure); + ServeProto::write(*store, wconn, closure); break; } @@ -988,7 +987,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = CommonProto::Serialise::read(*store, rconn); + info.references = ServeProto::Serialise::read(*store, rconn); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = ContentAddress::parseOpt(readString(in)); diff --git a/unit-test-data/libstore/serve-protocol/content-address.bin b/unit-test-data/libstore/serve-protocol/content-address.bin new file mode 100644 index 0000000000000000000000000000000000000000..8f14bcdb3e50cc72d87106ce159a6fd7b9f75713 GIT binary patch literal 208 zcmY+;K@P$o5QSmy;|7WYrYjRqQfMm$+LWPzDW_MfG4bucKks(>Y#V56lkFOiEzi24 zUMdC5VUCa86OJ&gL0BF^B{HBQEY%f|I<6v7#q+l_PBirI5N~-md%37WE|g33QTE2k zc~-WF&R}7Oxc@o)T?ojpD!+*5=xpO;%IoCKSV5FMh={>xePK|8<${>E)*huNHZ(pX literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/serve-protocol/drv-output.bin b/unit-test-data/libstore/serve-protocol/drv-output.bin new file mode 100644 index 0000000000000000000000000000000000000000..800a45fd8757a15e8810066aed48be56a600b7d3 GIT binary patch literal 176 zcmY++xeWp_5I|8{*$(Wn=b{B@!gGlfp_LHTb8N(qe)KM%U7Sq@}q)U|6h9n2j1!MT?w+C~^gO WO9=?G&ojrYjy`x4ZufnEe#tjMpDPdm literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/serve-protocol/optional-content-address.bin b/unit-test-data/libstore/serve-protocol/optional-content-address.bin new file mode 100644 index 0000000000000000000000000000000000000000..f8cfe65ba27fd78ec7787eed7ebec8dfdead2fba GIT binary patch literal 64 zcmZQzfB#ECTBU06lLKJOBUy literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/serve-protocol/realisation.bin b/unit-test-data/libstore/serve-protocol/realisation.bin new file mode 100644 index 0000000000000000000000000000000000000000..2176c6c4afd96b32fe372de6084ac7f4c7a11d49 GIT binary patch literal 520 zcmdUrO-{o=429t+opq7s&z_l_03#3x z?rW}!>Uc!$VBY>Sr2mUC`<2<)qY;)1KN Po44)luetw6d9AunL$;dR literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/serve-protocol/set.bin b/unit-test-data/libstore/serve-protocol/set.bin new file mode 100644 index 0000000000000000000000000000000000000000..ce11ede7fe7aa061a30b211a18cd7a336f879de8 GIT binary patch literal 152 ucmZQzfB;4)4WpQ03@8obCnXkvMPU52{CpHXOdBEdVDg4g4KThDln(&+h63;a literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/serve-protocol/store-path.bin b/unit-test-data/libstore/serve-protocol/store-path.bin new file mode 100644 index 0000000000000000000000000000000000000000..3fc05f2981d3398ab30ec34125c903cdcefa8729 GIT binary patch literal 120 tcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7Eu*F?srQlM;)-IsqP{BM|@q literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/serve-protocol/string.bin b/unit-test-data/libstore/serve-protocol/string.bin new file mode 100644 index 0000000000000000000000000000000000000000..aa7b5a604745b735473ec34283177e69320776e4 GIT binary patch literal 88 zcmZQzfB+^aoskJ)@Id+H8JQ)i3Pp)YNtq=eAx^0H( Date: Wed, 11 Oct 2023 19:58:42 +0800 Subject: [PATCH 372/909] Allow CLI to pass environment variables to FOD builder (#8830) Add a new experimental `impure-env` setting that is a key-value list of environment variables to inject into FOD derivations that specify the corresponding `impureEnvVars`. This allows clients to make use of this feature (without having to change the environment of the daemon itself) and might eventually deprecate the current behaviour (pick whatever is in the environment of the daemon) as it's more principled and might prevent information leakage. --- .../src/language/advanced-attributes.md | 7 ++++ src/libstore/build/local-derivation-goal.cc | 14 ++++++-- src/libstore/globals.hh | 20 +++++++++++ src/libutil/experimental-features.cc | 9 ++++- src/libutil/experimental-features.hh | 1 + tests/functional/impure-env.nix | 16 +++++++++ tests/functional/impure-env.sh | 33 +++++++++++++++++++ tests/functional/local.mk | 3 +- 8 files changed, 99 insertions(+), 4 deletions(-) create mode 100644 tests/functional/impure-env.nix create mode 100644 tests/functional/impure-env.sh diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 7d97b24b6..4e4372331 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -112,6 +112,13 @@ Derivations can declare some infrequently used optional attributes. > environmental variables come from the environment of the > `nix-build`. + If the [`configurable-impure-env` experimental + feature](@docroot@/contributing/experimental-features.md#xp-feature-configurable-impure-env) + is enabled, these environment variables can also be controlled + through the + [`impure-env`](@docroot@/command-ref/conf-file.md#conf-impure-env) + configuration setting. + - [`outputHash`]{#adv-attr-outputHash}; [`outputHashAlgo`]{#adv-attr-outputHashAlgo}; [`outputHashMode`]{#adv-attr-outputHashMode}\ These attributes declare that the derivation is a so-called *fixed-output derivation*, which means that a cryptographic hash of diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 64b55ca6a..817b20b2f 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1135,8 +1135,18 @@ void LocalDerivationGoal::initEnv() fixed-output derivations is by definition pure (since we already know the cryptographic hash of the output). */ if (!derivationType->isSandboxed()) { - for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) - env[i] = getEnv(i).value_or(""); + auto & impureEnv = settings.impureEnv.get(); + if (!impureEnv.empty()) + experimentalFeatureSettings.require(Xp::ConfigurableImpureEnv); + + for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) { + auto envVar = impureEnv.find(i); + if (envVar != impureEnv.end()) { + env[i] = envVar->second; + } else { + env[i] = getEnv(i).value_or(""); + } + } } /* Currently structured log messages piggyback on stderr, but we diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 0569f43b9..bd793c3d6 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -4,6 +4,7 @@ #include "types.hh" #include "config.hh" #include "util.hh" +#include "experimental-features.hh" #include #include @@ -1052,6 +1053,25 @@ public: ``` )" }; + + Setting impureEnv {this, {}, "impure-env", + R"( + A list of items, each in the format of: + + - `name=value`: Set environment variable `name` to `value`. + + If the user is trusted (see `trusted-users` option), when building + a fixed-output derivation, environment variables set in this option + will be passed to the builder if they are listed in [`impureEnvVars`](@docroot@/language/advanced-attributes.md##adv-attr-impureEnvVars). + + This option is useful for, e.g., setting `https_proxy` for + fixed-output derivations and in a multi-user Nix installation, or + setting private access tokens when fetching a private repository. + )", + {}, // aliases + true, // document default + Xp::ConfigurableImpureEnv + }; }; diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 203455b63..74af9aae0 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -221,6 +221,13 @@ constexpr std::array xpFeatureDetails = {{ Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs. )", }, + { + .tag = Xp::ConfigurableImpureEnv, + .name = "configurable-impure-env", + .description = R"( + Allow the use of the [impure-env](@docroot@/command-ref/conf-file.md#conf-impure-env) setting. + )", + } }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index add592ae6..e02f8353e 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -31,6 +31,7 @@ enum struct ExperimentalFeature DynamicDerivations, ParseTomlTimestamps, ReadOnlyLocalStore, + ConfigurableImpureEnv, }; /** diff --git a/tests/functional/impure-env.nix b/tests/functional/impure-env.nix new file mode 100644 index 000000000..2b0380ed7 --- /dev/null +++ b/tests/functional/impure-env.nix @@ -0,0 +1,16 @@ +{ var, value }: + +with import ./config.nix; + +mkDerivation { + name = "test"; + buildCommand = '' + echo ${var} = "''$${var}" + echo -n "''$${var}" > "$out" + ''; + + impureEnvVars = [ var ]; + + outputHashAlgo = "sha256"; + outputHash = builtins.hashString "sha256" value; +} diff --git a/tests/functional/impure-env.sh b/tests/functional/impure-env.sh new file mode 100644 index 000000000..d9e4a34a2 --- /dev/null +++ b/tests/functional/impure-env.sh @@ -0,0 +1,33 @@ +source common.sh + +# Needs the config option 'impure-env' to work +requireDaemonNewerThan "2.18.0pre20230816" + +enableFeatures "configurable-impure-env" +restartDaemon + +varTest() { + local var="$1"; shift + local value="$1"; shift + nix build --no-link -vL --argstr var "$var" --argstr value "$value" --impure "$@" --file impure-env.nix + clearStore +} + +clearStore +startDaemon + +varTest env_name value --impure-env env_name=value + +echo 'impure-env = set_in_config=config_value' >> "$NIX_CONF_DIR/nix.conf" +set_in_config=daemon_value restartDaemon + +varTest set_in_config config_value +varTest set_in_config client_value --impure-env set_in_config=client_value + +sed -i -e '/^trusted-users =/d' "$NIX_CONF_DIR/nix.conf" + +env_name=daemon_value restartDaemon + +varTest env_name daemon_value --impure-env env_name=client_value + +killDaemon diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 6f6c94fe6..643351163 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -121,7 +121,8 @@ nix_tests = \ path-from-hash-part.sh \ toString-path.sh \ read-only-store.sh \ - nested-sandboxing.sh + nested-sandboxing.sh \ + impure-env.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh From b4b1a07f9771d4fd106323672d833a1cb17c7364 Mon Sep 17 00:00:00 2001 From: vicky1999 Date: Fri, 13 Oct 2023 06:48:35 +0530 Subject: [PATCH 373/909] store info alias created --- src/nix/ping-store.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index ec450e8e0..5f8a10e22 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -47,3 +47,4 @@ struct CmdPingStore : StoreCommand, MixJSON }; static auto rCmdPingStore = registerCommand2({"store", "ping"}); +static auto rCmdInfoStore = registerCommand2({"store", "info"}); From 5c65379b228cee2c37408e1810c6654da5c91b90 Mon Sep 17 00:00:00 2001 From: vicky1999 Date: Fri, 13 Oct 2023 07:16:05 +0530 Subject: [PATCH 374/909] info store alias added to store-ping --- src/nix/ping-store.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 5f8a10e22..247d9c6bb 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -46,5 +46,15 @@ struct CmdPingStore : StoreCommand, MixJSON } }; +struct CmdInfoStore : CmdPingStore +{ + void run(nix::ref store) override + { + warn("'nix store info' is a deprecated alias for 'nix store ping'"); + CmdPingStore::run(store); + } +}; + + static auto rCmdPingStore = registerCommand2({"store", "ping"}); -static auto rCmdInfoStore = registerCommand2({"store", "info"}); +static auto rCmdInfoStore = registerCommand2({"store", "info"}); From db0d94560b2c72b695b420031ccdb8d1cbfb9f2b Mon Sep 17 00:00:00 2001 From: Michal Sojka Date: Sun, 11 Jun 2023 21:36:56 +0200 Subject: [PATCH 375/909] Document builtins.fetchTree Co-authored-by: Valentin Gagarin Supersedes #6740 --- src/libexpr/primops/fetchTree.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 124165896..af4036460 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -236,6 +236,10 @@ static RegisterPrimOp primop_fetchTree({ ```nix builtins.fetchTree "https://example.com/" ``` + + > **Note** + > + > This function requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. )", .fun = prim_fetchTree, .experimentalFeature = Xp::Flakes, From 856fe1353367836d7d9282ae52e91a47e6bd4683 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 28 Sep 2023 16:52:28 +0200 Subject: [PATCH 376/909] fetchTree cleanup Two changes: * The (probably unintentional) hack to handle paths as tarballs has been removed. This is almost certainly not what users expect and is inconsistent with flakeref handling everywhere else. * The hack to support scp-style Git URLs has been moved to the Git fetcher, so it's now supported not just by fetchTree but by flake inputs. --- src/libexpr/primops/fetchTree.cc | 55 ++++++++++---------------------- src/libfetchers/git.cc | 4 ++- src/libutil/url.cc | 17 ++++++++++ src/libutil/url.hh | 5 +++ src/nix/flake.md | 6 ++++ tests/functional/fetchGit.sh | 2 ++ tests/nixos/github-flakes.nix | 4 +++ 7 files changed, 54 insertions(+), 39 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index af4036460..920839058 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -25,7 +25,6 @@ void emitTreeAttrs( auto attrs = state.buildBindings(10); - state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath)); // FIXME: support arbitrary input attributes. @@ -71,36 +70,10 @@ void emitTreeAttrs( v.mkAttrs(attrs); } -std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file") -{ - state.checkURI(uri); - if (uri.find("://") == std::string::npos) { - const auto p = ParsedURL { - .scheme = defaultScheme, - .authority = "", - .path = uri - }; - return p.to_string(); - } else { - return uri; - } -} - -std::string fixURIForGit(std::string uri, EvalState & state) -{ - /* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes - * them by removing the `:` and assuming a scheme of `ssh://` - * */ - static std::regex scp_uri("([^/]*)@(.*):(.*)"); - if (uri[0] != '/' && std::regex_match(uri, scp_uri)) - return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh"); - else - return fixURI(uri, state); -} - struct FetchTreeParams { bool emptyRevFallback = false; bool allowNameArgument = false; + bool isFetchGit = false; }; static void fetchTree( @@ -108,11 +81,12 @@ static void fetchTree( const PosIdx pos, Value * * args, Value & v, - std::optional type, const FetchTreeParams & params = FetchTreeParams{} ) { fetchers::Input input; NixStringContext context; + std::optional type; + if (params.isFetchGit) type = "git"; state.forceValue(*args[0], pos); @@ -142,10 +116,8 @@ static void fetchTree( if (attr.value->type() == nPath || attr.value->type() == nString) { auto s = state.coerceToString(attr.pos, *attr.value, context, "", false, false).toOwned(); attrs.emplace(state.symbols[attr.name], - state.symbols[attr.name] == "url" - ? type == "git" - ? fixURIForGit(s, state) - : fixURI(s, state) + params.isFetchGit && state.symbols[attr.name] == "url" + ? fixGitURL(s) : s); } else if (attr.value->type() == nBool) @@ -170,13 +142,13 @@ static void fetchTree( "while evaluating the first argument passed to the fetcher", false, false).toOwned(); - if (type == "git") { + if (params.isFetchGit) { fetchers::Attrs attrs; attrs.emplace("type", "git"); - attrs.emplace("url", fixURIForGit(url, state)); + attrs.emplace("url", fixGitURL(url)); input = fetchers::Input::fromAttrs(std::move(attrs)); } else { - input = fetchers::Input::fromURL(fixURI(url, state)); + input = fetchers::Input::fromURL(url); } } @@ -186,6 +158,8 @@ static void fetchTree( if (evalSettings.pureEval && !input.isLocked()) state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", state.positions[pos])); + state.checkURI(input.toURLString()); + auto [tree, input2] = input.fetch(state.store); state.allowPath(tree.storePath); @@ -195,7 +169,7 @@ static void fetchTree( static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false }); + fetchTree(state, pos, args, v, { }); } static RegisterPrimOp primop_fetchTree({ @@ -392,7 +366,12 @@ static RegisterPrimOp primop_fetchTarball({ static void prim_fetchGit(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true }); + fetchTree(state, pos, args, v, + FetchTreeParams { + .emptyRevFallback = true, + .allowNameArgument = true, + .isFetchGit = true + }); } static RegisterPrimOp primop_fetchGit({ diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index f8d89ab2f..7e9f34790 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -293,7 +293,6 @@ struct GitInputScheme : InputScheme if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev") throw Error("unsupported Git input attribute '%s'", name); - parseURL(getStrAttr(attrs, "url")); maybeGetBoolAttr(attrs, "shallow"); maybeGetBoolAttr(attrs, "submodules"); maybeGetBoolAttr(attrs, "allRefs"); @@ -305,6 +304,9 @@ struct GitInputScheme : InputScheme Input input; input.attrs = attrs; + auto url = fixGitURL(getStrAttr(attrs, "url")); + parseURL(url); + input.attrs["url"] = url; return input; } diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 2970f8d2d..9b438e6cd 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -158,4 +158,21 @@ ParsedUrlScheme parseUrlScheme(std::string_view scheme) }; } +std::string fixGitURL(const std::string & url) +{ + std::regex scpRegex("([^/]*)@(.*):(.*)"); + if (!hasPrefix(url, "/") && std::regex_match(url, scpRegex)) + return std::regex_replace(url, scpRegex, "ssh://$1@$2/$3"); + else { + if (url.find("://") == std::string::npos) { + return (ParsedURL { + .scheme = "file", + .authority = "", + .path = url + }).to_string(); + } else + return url; + } +} + } diff --git a/src/libutil/url.hh b/src/libutil/url.hh index d2413ec0e..26c2dcc28 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -45,4 +45,9 @@ struct ParsedUrlScheme { ParsedUrlScheme parseUrlScheme(std::string_view scheme); +/* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes + them by removing the `:` and assuming a scheme of `ssh://`. Also + changes absolute paths into file:// URLs. */ +std::string fixGitURL(const std::string & url); + } diff --git a/src/nix/flake.md b/src/nix/flake.md index 939269c6a..f08648417 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -182,6 +182,12 @@ Currently the `type` attribute can be one of the following: git(+http|+https|+ssh|+git|+file|):(//)?(\?)? ``` + or + + ``` + @: + ``` + The `ref` attribute defaults to resolving the `HEAD` reference. The `rev` attribute must denote a commit that exists in the branch diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index 418b4f63f..fc89f2040 100644 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -35,6 +35,8 @@ unset _NIX_FORCE_HTTP path0=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath") path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$TEST_ROOT/worktree; }).outPath") [[ $path0 = $path0_ ]] +path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree git+file://$TEST_ROOT/worktree).outPath") +[[ $path0 = $path0_ ]] export _NIX_FORCE_HTTP=1 [[ $(tail -n 1 $path0/hello) = "hello" ]] diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index 6de702d17..62ae8871b 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -186,6 +186,10 @@ in client.succeed("nix registry pin nixpkgs") client.succeed("nix flake metadata nixpkgs --tarball-ttl 0 >&2") + # Test fetchTree on a github URL. + hash = client.succeed(f"nix eval --raw --expr '(fetchTree {info['url']}).narHash'") + assert hash == info['locked']['narHash'] + # Shut down the web server. The flake should be cached on the client. github.succeed("systemctl stop httpd.service") From 4ce7a53a9ce4eb223f594fcbf627f968f5df02fd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 28 Sep 2023 17:08:26 +0200 Subject: [PATCH 377/909] Update fetchTree docs --- src/libexpr/primops/fetchTree.cc | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 920839058..1e2cee77b 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -177,12 +177,12 @@ static RegisterPrimOp primop_fetchTree({ .args = {"input"}, .doc = R"( Fetch a source tree or a plain file using one of the supported backends. - *input* can be an attribute set representation of [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) or a URL. - The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is allowed. + *input* must be a [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references), either in attribute set representation or in the URL-like syntax. + The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is enabled. Here are some examples of how to use `fetchTree`: - - Fetch a GitHub repository: + - Fetch a GitHub repository using the attribute set representation: ```nix builtins.fetchTree { @@ -193,7 +193,7 @@ static RegisterPrimOp primop_fetchTree({ } ``` - This evaluates to attribute set: + This evaluates to the following attribute set: ``` { @@ -205,15 +205,12 @@ static RegisterPrimOp primop_fetchTree({ shortRev = "ae2e6b3"; } ``` - - Fetch a single file from a URL: - ```nix - builtins.fetchTree "https://example.com/" + - Fetch the same GitHub repository using the URL-like syntax: + + ``` + builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd" ``` - - > **Note** - > - > This function requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. )", .fun = prim_fetchTree, .experimentalFeature = Xp::Flakes, From 8eb4f735dc230fb8135b7e513cfc51eb66d937a9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 29 Sep 2023 10:48:27 +0200 Subject: [PATCH 378/909] fetchTree: Only use the registry if flakes are enabled --- src/libexpr/primops/fetchTree.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 1e2cee77b..3d7a85047 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -152,7 +152,7 @@ static void fetchTree( } } - if (!evalSettings.pureEval && !input.isDirect()) + if (!evalSettings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) input = lookupInRegistries(state.store, input).first; if (evalSettings.pureEval && !input.isLocked()) From 4112dd1fc93c9ff03a5a4e8be773c45ebefbbd1f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Sep 2023 17:08:04 +0200 Subject: [PATCH 379/909] Mark fetchTree as stable --- doc/manual/src/release-notes/rl-next.md | 4 +++- src/libexpr/primops/fetchTree.cc | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 418c1187c..c905f445f 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -6,4 +6,6 @@ - The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly. -- Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.` or `legacyPackages.`. \ No newline at end of file +- Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.` or `legacyPackages.`. + +- `builtins.fetchTree` is now marked as stable. diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 3d7a85047..0335e5f36 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -213,7 +213,6 @@ static RegisterPrimOp primop_fetchTree({ ``` )", .fun = prim_fetchTree, - .experimentalFeature = Xp::Flakes, }); static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v, From a23cc147cb93fe9138b9a2f8283ad3ebbee43088 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 13 Oct 2023 11:00:55 -0400 Subject: [PATCH 380/909] Factor out Perl bindings Nix package Progress breaking up `flake.nix` by introducing separate `default.nix` files which make sense on their own. (This one is a regular `callPackage`-able package.) --- flake.nix | 43 ++++------------------------------------ perl/default.nix | 51 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 39 deletions(-) create mode 100644 perl/default.nix diff --git a/flake.nix b/flake.nix index 301e65545..2263ea572 100644 --- a/flake.nix +++ b/flake.nix @@ -474,45 +474,10 @@ hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - passthru.perl-bindings = with final; perl.pkgs.toPerlModule (currentStdenv.mkDerivation { - name = "nix-perl-${version}"; - - src = fileset.toSource { - root = ./.; - fileset = fileset.intersect baseFiles (fileset.unions [ - ./perl - ./.version - ./m4 - ./mk - ]); - }; - - nativeBuildInputs = - [ buildPackages.autoconf-archive - buildPackages.autoreconfHook - buildPackages.pkg-config - ]; - - buildInputs = - [ nix - curl - bzip2 - xz - pkgs.perl - boost - ] - ++ lib.optional (currentStdenv.isLinux || currentStdenv.isDarwin) libsodium - ++ lib.optional currentStdenv.isDarwin darwin.apple_sdk.frameworks.Security; - - configureFlags = [ - "--with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}" - "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}" - ]; - - enableParallelBuilding = true; - - postUnpack = "sourceRoot=$sourceRoot/perl"; - }); + passthru.perl-bindings = final.callPackage ./perl { + inherit fileset; + stdenv = currentStdenv; + }; meta.platforms = lib.platforms.unix; }); diff --git a/perl/default.nix b/perl/default.nix new file mode 100644 index 000000000..4687976a1 --- /dev/null +++ b/perl/default.nix @@ -0,0 +1,51 @@ +{ lib, fileset +, stdenv +, perl, perlPackages +, autoconf-archive, autoreconfHook, pkg-config +, nix, curl, bzip2, xz, boost, libsodium, darwin +}: + +perl.pkgs.toPerlModule (stdenv.mkDerivation { + name = "nix-perl-${nix.version}"; + + src = fileset.toSource { + root = ../.; + fileset = fileset.unions [ + ../.version + ../m4 + ../mk + ./MANIFEST + ./Makefile + ./Makefile.config.in + ./configure.ac + ./lib + ./local.mk + ]; + }; + + nativeBuildInputs = + [ autoconf-archive + autoreconfHook + pkg-config + ]; + + buildInputs = + [ nix + curl + bzip2 + xz + perl + boost + ] + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium + ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; + + configureFlags = [ + "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}" + "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}" + ]; + + enableParallelBuilding = true; + + postUnpack = "sourceRoot=$sourceRoot/perl"; +}) From e5ce53f3db93ddbc9cca6d51d5fc0e2fd7335517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=A9clairevoyant?= <848000+eclairevoyant@users.noreply.github.com> Date: Sun, 15 Oct 2023 15:08:07 -0400 Subject: [PATCH 381/909] explicitly set meta.mainProgram --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 301e65545..05bb8fad4 100644 --- a/flake.nix +++ b/flake.nix @@ -515,6 +515,7 @@ }); meta.platforms = lib.platforms.unix; + meta.mainProgram = "nix"; }); lowdown-nix = with final; currentStdenv.mkDerivation rec { From c27d2f8da9643a0a87e36ae992b29ea00a50e0d9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Apr 2023 23:37:55 -0400 Subject: [PATCH 382/909] Add two more completions tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thanks @ncfavier for catching these regressions in my PR. Co-Authored-By: Naïm Favier --- tests/functional/completions.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/functional/completions.sh b/tests/functional/completions.sh index 19dc61098..7c1e4b287 100644 --- a/tests/functional/completions.sh +++ b/tests/functional/completions.sh @@ -48,6 +48,8 @@ EOF [[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]] ## With tilde expansion [[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]] +[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake show '~/foo' --update-input '')" == $'normal\na\t' ]] +[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix run '~/foo' --update-input '')" == $'normal\na\t' ]] ## Out of order [[ "$(NIX_GET_COMPLETIONS=3 nix build --update-input '' ./foo)" == $'normal\na\t' ]] [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --update-input '' ./bar)" == $'normal\na\t\nb\t' ]] From 483d99c622414504bb1b6a565f30ca6f21284315 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 16 Oct 2023 10:42:15 -0400 Subject: [PATCH 383/909] Add API docs to some args-related functionality --- src/libcmd/command.hh | 78 ++++++++++++++++++++++++++++++++++++------- src/libutil/args.hh | 59 ++++++++++++++++++++++++++++++-- 2 files changed, 123 insertions(+), 14 deletions(-) diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 96236b987..5c4569001 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -34,21 +34,28 @@ struct NixMultiCommand : virtual MultiCommand, virtual Command // For the overloaded run methods #pragma GCC diagnostic ignored "-Woverloaded-virtual" -/* A command that requires a Nix store. */ +/** + * A command that requires a \ref Store "Nix store". + */ struct StoreCommand : virtual Command { StoreCommand(); void run() override; ref getStore(); virtual ref createStore(); + /** + * Main entry point, with a `Store` provided + */ virtual void run(ref) = 0; private: std::shared_ptr _store; }; -/* A command that copies something between `--from` and `--to` - stores. */ +/** + * A command that copies something between `--from` and `--to` \ref + * Store stores. + */ struct CopyCommand : virtual StoreCommand { std::string srcUri, dstUri; @@ -60,6 +67,9 @@ struct CopyCommand : virtual StoreCommand ref getDstStore(); }; +/** + * A command that needs to evaluate Nix language expressions. + */ struct EvalCommand : virtual StoreCommand, MixEvalArgs { bool startReplOnEvalErrors = false; @@ -79,6 +89,10 @@ private: std::shared_ptr evalState; }; +/** + * A mixin class for commands that process flakes, adding a few standard + * flake-related options/flags. + */ struct MixFlakeOptions : virtual Args, EvalCommand { flake::LockFlags lockFlags; @@ -87,6 +101,14 @@ struct MixFlakeOptions : virtual Args, EvalCommand MixFlakeOptions(); + /** + * The completion for some of these flags depends on the flake(s) in + * question. + * + * This method should be implemented to gather all flakerefs the + * command is operating with (presumably specified via some other + * arguments) so that the completions for these flags can use them. + */ virtual std::vector getFlakesForCompletion() { return {}; } @@ -112,15 +134,29 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions virtual Strings getDefaultFlakeAttrPathPrefixes(); + /** + * Complete an installable from the given prefix. + */ void completeInstallable(std::string_view prefix); }; +/** + * A mixin class for commands that need a read-only flag. + * + * What exactly is "read-only" is unspecified, but it will usually be + * the \ref Store "Nix store". + */ struct MixReadOnlyOption : virtual Args { MixReadOnlyOption(); }; -/* Like InstallablesCommand but the installables are not loaded */ +/** + * Like InstallablesCommand but the installables are not loaded. + * + * This is needed by `CmdRepl` which wants to load (and reload) the + * installables itself. + */ struct RawInstallablesCommand : virtual Args, SourceExprCommand { RawInstallablesCommand(); @@ -129,7 +165,7 @@ struct RawInstallablesCommand : virtual Args, SourceExprCommand void run(ref store) override; - // FIXME make const after CmdRepl's override is fixed up + // FIXME make const after `CmdRepl`'s override is fixed up virtual void applyDefaultInstallables(std::vector & rawInstallables); bool readFromStdIn = false; @@ -140,8 +176,11 @@ private: std::vector rawInstallables; }; -/* A command that operates on a list of "installables", which can be - store paths, attribute paths, Nix expressions, etc. */ + +/** + * A command that operates on a list of "installables", which can be + * store paths, attribute paths, Nix expressions, etc. + */ struct InstallablesCommand : RawInstallablesCommand { virtual void run(ref store, Installables && installables) = 0; @@ -149,7 +188,9 @@ struct InstallablesCommand : RawInstallablesCommand void run(ref store, std::vector && rawInstallables) override; }; -/* A command that operates on exactly one "installable" */ +/** + * A command that operates on exactly one "installable". + */ struct InstallableCommand : virtual Args, SourceExprCommand { InstallableCommand(); @@ -175,7 +216,12 @@ struct MixOperateOnOptions : virtual Args MixOperateOnOptions(); }; -/* A command that operates on zero or more store paths. */ +/** + * A command that operates on zero or more extant store paths. + * + * If the argument the user passes is a some sort of recipe for a path + * not yet built, it must be built first. + */ struct BuiltPathsCommand : InstallablesCommand, virtual MixOperateOnOptions { private: @@ -207,7 +253,9 @@ struct StorePathsCommand : public BuiltPathsCommand void run(ref store, BuiltPaths && paths) override; }; -/* A command that operates on exactly one store path. */ +/** + * A command that operates on exactly one store path. + */ struct StorePathCommand : public StorePathsCommand { virtual void run(ref store, const StorePath & storePath) = 0; @@ -215,7 +263,9 @@ struct StorePathCommand : public StorePathsCommand void run(ref store, StorePaths && storePaths) override; }; -/* A helper class for registering commands globally. */ +/** + * A helper class for registering \ref Command commands globally. + */ struct RegisterCommand { typedef std::map, std::function()>> Commands; @@ -271,7 +321,11 @@ struct MixEnvironment : virtual Args { MixEnvironment(); - /* Modify global environ based on ignoreEnvironment, keep, and unset. It's expected that exec will be called before this class goes out of scope, otherwise environ will become invalid. */ + /*** + * Modify global environ based on `ignoreEnvironment`, `keep`, and + * `unset`. It's expected that exec will be called before this class + * goes out of scope, otherwise `environ` will become invalid. + */ void setEnviron(); }; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 1e5bac650..6457cceed 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -39,8 +39,21 @@ public: protected: + /** + * The largest `size_t` is used to indicate the "any" arity, for + * handlers/flags/arguments that accept an arbitrary number of + * arguments. + */ static const size_t ArityAny = std::numeric_limits::max(); + /** + * Arguments (flags/options and positional) have a "handler" which is + * caused when the argument is parsed. The handler has an arbitrary side + * effect, including possible affect further command-line parsing. + * + * There are many constructors in order to support many shorthand + * initializations, and this is used a lot. + */ struct Handler { std::function)> fun; @@ -110,7 +123,12 @@ protected: { } }; - /* Options. */ + /** + * Description of flags / options + * + * These are arguments like `-s` or `--long` that can (mostly) + * appear in any order. + */ struct Flag { typedef std::shared_ptr ptr; @@ -130,12 +148,30 @@ protected: static Flag mkHashTypeOptFlag(std::string && longName, std::optional * oht); }; + /** + * Index of all registered "long" flag descriptions (flags like + * `--long`). + */ std::map longFlags; + + /** + * Index of all registered "short" flag descriptions (flags like + * `-s`). + */ std::map shortFlags; + /** + * Process a single flag and its arguments, pulling from an iterator + * of raw CLI args as needed. + */ virtual bool processFlag(Strings::iterator & pos, Strings::iterator end); - /* Positional arguments. */ + /** + * Description of positional arguments + * + * These are arguments that do not start with a `-`, and for which + * the order does matter. + */ struct ExpectedArg { std::string label; @@ -144,8 +180,24 @@ protected: std::function completer; }; + /** + * Queue of expected positional argument forms. + * + * Positional arugment descriptions are inserted on the back. + * + * As positional arguments are passed, these are popped from the + * front, until there are hopefully none left as all args that were + * expected in fact were passed. + */ std::list expectedArgs; + /** + * Process some positional arugments + * + * @param finish: We have parsed everything else, and these are the only + * arguments left. Used because we accumulate some "pending args" we might + * have left over. + */ virtual bool processArgs(const Strings & args, bool finish); virtual Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos) @@ -204,6 +256,9 @@ public: friend class MultiCommand; + /** + * The parent command, used if this is a subcommand. + */ MultiCommand * parent = nullptr; private: From f7a36f981276a535962c1f5b0d02b00b9cd05298 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 16 Oct 2023 12:52:59 -0400 Subject: [PATCH 384/909] Fix language tests a bit - Remove some stray saved error messages that didn't correspond to any test, because they were renamed in d11faa01b5b16773ba97bcb852b4123992829afc. - Need `--eval` in test failure test in order to get in "read-only" mode where we don't try to write to the store. (The other tests already do this.) - Need `--strict` so top-level attribute sets are still forced, like they are without `--eval`. --- tests/functional/lang.sh | 2 +- .../lang/eval-fail-antiquoted-path.err.exp | 1 - .../lang/eval-fail-bad-antiquote-1.err.exp | 10 ---------- .../lang/eval-fail-bad-antiquote-2.err.exp | 1 - .../lang/eval-fail-bad-antiquote-3.err.exp | 10 ---------- .../eval-fail-bad-string-interpolation-2.err.exp | 2 +- .../lang/eval-fail-dup-dynamic-attrs.err.exp | 12 +++++++++++- .../functional/lang/eval-fail-nonexist-path.err.exp | 2 +- tests/functional/lang/parse-fail-dup-attrs-6.err.exp | 1 - 9 files changed, 14 insertions(+), 27 deletions(-) delete mode 100644 tests/functional/lang/eval-fail-antiquoted-path.err.exp delete mode 100644 tests/functional/lang/eval-fail-bad-antiquote-1.err.exp delete mode 100644 tests/functional/lang/eval-fail-bad-antiquote-2.err.exp delete mode 100644 tests/functional/lang/eval-fail-bad-antiquote-3.err.exp delete mode 100644 tests/functional/lang/parse-fail-dup-attrs-6.err.exp diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh index f4760eced..c3acef5ee 100755 --- a/tests/functional/lang.sh +++ b/tests/functional/lang.sh @@ -68,7 +68,7 @@ for i in lang/eval-fail-*.nix; do echo "evaluating $i (should fail)"; i=$(basename "$i" .nix) if - expectStderr 1 nix-instantiate --show-trace "lang/$i.nix" \ + expectStderr 1 nix-instantiate --eval --strict --show-trace "lang/$i.nix" \ | sed "s!$(pwd)!/pwd!g" > "lang/$i.err" then diffAndAccept "$i" err err.exp diff --git a/tests/functional/lang/eval-fail-antiquoted-path.err.exp b/tests/functional/lang/eval-fail-antiquoted-path.err.exp deleted file mode 100644 index 425deba42..000000000 --- a/tests/functional/lang/eval-fail-antiquoted-path.err.exp +++ /dev/null @@ -1 +0,0 @@ -error: getting attributes of path ‘PWD/lang/fnord’: No such file or directory diff --git a/tests/functional/lang/eval-fail-bad-antiquote-1.err.exp b/tests/functional/lang/eval-fail-bad-antiquote-1.err.exp deleted file mode 100644 index cf94f53bc..000000000 --- a/tests/functional/lang/eval-fail-bad-antiquote-1.err.exp +++ /dev/null @@ -1,10 +0,0 @@ -error: - … while evaluating a path segment - - at /pwd/lang/eval-fail-bad-antiquote-1.nix:1:2: - - 1| "${x: x}" - | ^ - 2| - - error: cannot coerce a function to a string diff --git a/tests/functional/lang/eval-fail-bad-antiquote-2.err.exp b/tests/functional/lang/eval-fail-bad-antiquote-2.err.exp deleted file mode 100644 index c8fe39d12..000000000 --- a/tests/functional/lang/eval-fail-bad-antiquote-2.err.exp +++ /dev/null @@ -1 +0,0 @@ -error: operation 'addToStoreFromDump' is not supported by store 'dummy' diff --git a/tests/functional/lang/eval-fail-bad-antiquote-3.err.exp b/tests/functional/lang/eval-fail-bad-antiquote-3.err.exp deleted file mode 100644 index fbefbc826..000000000 --- a/tests/functional/lang/eval-fail-bad-antiquote-3.err.exp +++ /dev/null @@ -1,10 +0,0 @@ -error: - … while evaluating a path segment - - at /pwd/lang/eval-fail-bad-antiquote-3.nix:1:3: - - 1| ''${x: x}'' - | ^ - 2| - - error: cannot coerce a function to a string diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp index c8fe39d12..dea119ae8 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp @@ -1 +1 @@ -error: operation 'addToStoreFromDump' is not supported by store 'dummy' +error: getting status of '/pwd/lang/fnord': No such file or directory diff --git a/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp b/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp index e01f8e6d0..c5fa67523 100644 --- a/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp +++ b/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp @@ -1,4 +1,14 @@ -error: dynamic attribute 'b' already defined at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:11 +error: + … while evaluating the attribute 'set' + + at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:3: + + 1| { + 2| set = { "${"" + "b"}" = 1; }; + | ^ + 3| set = { "${"b" + ""}" = 2; }; + + error: dynamic attribute 'b' already defined at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:11 at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:3:11: diff --git a/tests/functional/lang/eval-fail-nonexist-path.err.exp b/tests/functional/lang/eval-fail-nonexist-path.err.exp index c8fe39d12..dea119ae8 100644 --- a/tests/functional/lang/eval-fail-nonexist-path.err.exp +++ b/tests/functional/lang/eval-fail-nonexist-path.err.exp @@ -1 +1 @@ -error: operation 'addToStoreFromDump' is not supported by store 'dummy' +error: getting status of '/pwd/lang/fnord': No such file or directory diff --git a/tests/functional/lang/parse-fail-dup-attrs-6.err.exp b/tests/functional/lang/parse-fail-dup-attrs-6.err.exp deleted file mode 100644 index 74823fc25..000000000 --- a/tests/functional/lang/parse-fail-dup-attrs-6.err.exp +++ /dev/null @@ -1 +0,0 @@ -error: attribute ‘services.ssh’ at (string):3:3 already defined at (string):2:3 From b3fd7db63f78ec736238016745338277703ad1d5 Mon Sep 17 00:00:00 2001 From: Vladimir Kryachko Date: Mon, 16 Oct 2023 13:00:49 -0400 Subject: [PATCH 385/909] Detect cycles in flake follows. This change results in an error thrown as opposed to segfaulting due to stack overflow. Fixes #9144 --- src/libexpr/flake/lockfile.cc | 22 ++++++++++++++--- tests/functional/flakes/follow-paths.sh | 32 ++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 3c202967a..68443211b 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -45,16 +45,26 @@ StorePath LockedNode::computeStorePath(Store & store) const return lockedRef.input.computeStorePath(store); } -std::shared_ptr LockFile::findInput(const InputPath & path) -{ + +static std::shared_ptr doFind(const ref& root, const InputPath & path, std::vector& visited) { auto pos = root; + auto pathS = printInputPath(path); + auto found = std::find(visited.cbegin(), visited.cend(), pathS); + + if(found != visited.end()) { + std::vector cycle(found, visited.cend()); + cycle.push_back(pathS); + throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle)); + } + visited.push_back(pathS); + for (auto & elem : path) { if (auto i = get(pos->inputs, elem)) { if (auto node = std::get_if<0>(&*i)) pos = *node; else if (auto follows = std::get_if<1>(&*i)) { - if (auto p = findInput(*follows)) + if (auto p = doFind(root, *follows, visited)) pos = ref(p); else return {}; @@ -66,6 +76,12 @@ std::shared_ptr LockFile::findInput(const InputPath & path) return pos; } +std::shared_ptr LockFile::findInput(const InputPath & path) +{ + std::vector visited; + return doFind(root, path, visited); +} + LockFile::LockFile(const nlohmann::json & json, const Path & path) { auto version = json.value("version", 0); diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh index dc97027ac..8573b5511 100644 --- a/tests/functional/flakes/follow-paths.sh +++ b/tests/functional/flakes/follow-paths.sh @@ -167,7 +167,7 @@ nix flake lock "$flakeFollowsA" 2>&1 | grep "warning: input 'B' has an override # # The message was # error: input 'B/D' follows a non-existent input 'B/C/D' -# +# # Note that for `B` to resolve its follow for `D`, it needs `C/D`, for which it needs to resolve the follow on `C` first. flakeFollowsOverloadA="$TEST_ROOT/follows/overload/flakeA" flakeFollowsOverloadB="$TEST_ROOT/follows/overload/flakeA/flakeB" @@ -230,3 +230,33 @@ git -C "$flakeFollowsOverloadA" add flake.nix flakeB/flake.nix \ nix flake metadata "$flakeFollowsOverloadA" nix flake update "$flakeFollowsOverloadA" nix flake lock "$flakeFollowsOverloadA" + +# Now test follow cycle detection +# We construct the following follows graph: +# +# foo +# / ^ +# / \ +# v \ +# bar -> baz +# The message was +# error: follow cycle detected: [baz -> foo -> bar -> baz] +flakeFollowCycle="$TEST_ROOT/follows/followCycle" + +# Test following path flakerefs. +mkdir -p "$flakeFollowCycle" + +cat > $flakeFollowCycle/flake.nix <&1 && fail "nix flake lock should have failed." || true) +echo $checkRes | grep -F "error: follow cycle detected: [baz -> foo -> bar -> baz]" From add066cc7bdd1357ccf471b6ab6e68c470e6f604 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Mon, 16 Oct 2023 19:32:47 +0100 Subject: [PATCH 386/909] Fix broken move --- src/libutil/config.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 38d406e8a..8672edaa8 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -68,6 +68,7 @@ void AbstractConfig::warnUnknownSettings() void AbstractConfig::reapplyUnknownSettings() { auto unknownSettings2 = std::move(unknownSettings); + unknownSettings = {}; for (auto & s : unknownSettings2) set(s.first, s.second); } From d6066c90f87d9ba8ed1ddd8e2a3bd199f8498d6f Mon Sep 17 00:00:00 2001 From: Vladimir Kryachko Date: Mon, 16 Oct 2023 15:47:28 -0400 Subject: [PATCH 387/909] Don't convert InputPaths to strings prematurely. --- src/libexpr/flake/lockfile.cc | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 68443211b..f3ea9063f 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -2,8 +2,10 @@ #include "store-api.hh" #include "url-parts.hh" +#include #include +#include #include namespace nix::flake { @@ -46,18 +48,18 @@ StorePath LockedNode::computeStorePath(Store & store) const } -static std::shared_ptr doFind(const ref& root, const InputPath & path, std::vector& visited) { +static std::shared_ptr doFind(const ref& root, const InputPath & path, std::vector& visited) { auto pos = root; - auto pathS = printInputPath(path); - auto found = std::find(visited.cbegin(), visited.cend(), pathS); + auto found = std::find(visited.cbegin(), visited.cend(), path); if(found != visited.end()) { - std::vector cycle(found, visited.cend()); - cycle.push_back(pathS); + std::vector cycle; + std::transform(found, visited.cend(), std::back_inserter(cycle), printInputPath); + cycle.push_back(printInputPath(path)); throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle)); } - visited.push_back(pathS); + visited.push_back(path); for (auto & elem : path) { if (auto i = get(pos->inputs, elem)) { @@ -78,7 +80,7 @@ static std::shared_ptr doFind(const ref& root, const InputPath & pat std::shared_ptr LockFile::findInput(const InputPath & path) { - std::vector visited; + std::vector visited; return doFind(root, path, visited); } From abf7df2b376a1cdbc00527bc662344fd9232ce05 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Mon, 16 Oct 2023 21:47:33 +0100 Subject: [PATCH 388/909] Fix moves that accidentally copy anyway --- src/libexpr/eval.cc | 4 ++-- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/outputs-spec.cc | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index b18eb5266..5280614a1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -539,7 +539,7 @@ EvalState::EvalState( auto r = resolveSearchPathPath(i.path); if (!r) continue; - auto path = *std::move(r); + auto path = std::move(*r); if (store->isInStore(path)) { try { @@ -1035,7 +1035,7 @@ std::string EvalState::mkOutputStringRaw( /* In practice, this is testing for the case of CA derivations, or dynamic derivations. */ return optStaticOutputPath - ? store->printStorePath(*std::move(optStaticOutputPath)) + ? store->printStorePath(std::move(*optStaticOutputPath)) /* Downstream we would substitute this for an actual path once we build the floating CA derivation */ : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b, xpSettings).render(); diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index dc4d91079..360c6b70b 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -561,7 +561,7 @@ void DerivationGoal::inputsRealised() attempt = fullDrv.tryResolve(worker.store); } assert(attempt); - Derivation drvResolved { *std::move(attempt) }; + Derivation drvResolved { std::move(*attempt) }; auto pathResolved = writeDerivation(worker.store, drvResolved); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 817b20b2f..6a02f5ad4 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2521,7 +2521,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() ValidPathInfo newInfo0 { worker.store, outputPathName(drv->name, outputName), - *std::move(optCA), + std::move(*optCA), Hash::dummy, }; if (*scratchPath != newInfo0.path) { diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc index d943bc111..21c069223 100644 --- a/src/libstore/outputs-spec.cc +++ b/src/libstore/outputs-spec.cc @@ -63,7 +63,7 @@ std::optional> ExtendedOutputsS auto specOpt = OutputsSpec::parseOpt(s.substr(found + 1)); if (!specOpt) return std::nullopt; - return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { *std::move(specOpt) } }; + return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { std::move(*specOpt) } }; } From 54b350d517932ef11de4b3d638d67ffa5cec1634 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Mon, 16 Oct 2023 21:48:02 +0100 Subject: [PATCH 389/909] Drop some moves that would happen anyway but forbid NRVO where appicable --- src/libexpr/eval.cc | 2 +- src/libexpr/paths.cc | 2 +- src/libstore/content-address.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 5280614a1..511a5f434 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2290,7 +2290,7 @@ BackedStringView EvalState::coerceToString( && (!v2->isList() || v2->listSize() != 0)) result += " "; } - return std::move(result); + return result; } } diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index 1d690b722..b6a696f47 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -4,7 +4,7 @@ namespace nix { SourcePath EvalState::rootPath(CanonPath path) { - return std::move(path); + return path; } } diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index e290a8d38..ae91b859b 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -83,7 +83,7 @@ static std::pair parseContentAddressMethodPrefix if (!hashTypeRaw) throw UsageError("content address hash must be in form ':', but found: %s", wholeInput); HashType hashType = parseHashType(*hashTypeRaw); - return std::move(hashType); + return hashType; }; // Switch on prefix From 564922939404db110bcdfa71319470533df54cb4 Mon Sep 17 00:00:00 2001 From: Artturin Date: Mon, 11 Sep 2023 00:19:21 +0300 Subject: [PATCH 390/909] Bindmount files instead of hardlinking or copying to chroot https://github.com/nixos/nix/commit/16591eb3cccf86da8cd3f20c56e2dd847576ff5e#diff-19f999107b609d37cfb22c58e7f0bc1cf76edf1180e238dd6389e03cc279b604 (2013) added support for files to doBind This is work towards allowing users to change the location of chrootRootDir, to, for example, a tmpfs. inspired by trofi on matrix > It looks like build sandbox created by nix-daemon runs on the same filesystem, as /nix/store including things like /tmp which makes all small temporary files hit the disk. Is it intentional? If it is is there an easy way to redirect chroot's root to be tmpfs? dirsInChroot -> pathsInChroot --- src/libstore/build/local-derivation-goal.cc | 64 +++++++-------------- src/libstore/build/local-derivation-goal.hh | 4 +- 2 files changed, 23 insertions(+), 45 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 64b55ca6a..a3ebfc681 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -581,7 +581,7 @@ void LocalDerivationGoal::startBuilder() /* Allow a user-configurable set of directories from the host file system. */ - dirsInChroot.clear(); + pathsInChroot.clear(); for (auto i : settings.sandboxPaths.get()) { if (i.empty()) continue; @@ -592,19 +592,19 @@ void LocalDerivationGoal::startBuilder() } size_t p = i.find('='); if (p == std::string::npos) - dirsInChroot[i] = {i, optional}; + pathsInChroot[i] = {i, optional}; else - dirsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; + pathsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; } if (hasPrefix(worker.store.storeDir, tmpDirInSandbox)) { throw Error("`sandbox-build-dir` must not contain the storeDir"); } - dirsInChroot[tmpDirInSandbox] = tmpDir; + pathsInChroot[tmpDirInSandbox] = tmpDir; /* Add the closure of store paths to the chroot. */ StorePathSet closure; - for (auto & i : dirsInChroot) + for (auto & i : pathsInChroot) try { if (worker.store.isInStore(i.second.source)) worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); @@ -615,7 +615,7 @@ void LocalDerivationGoal::startBuilder() } for (auto & i : closure) { auto p = worker.store.printStorePath(i); - dirsInChroot.insert_or_assign(p, p); + pathsInChroot.insert_or_assign(p, p); } PathSet allowedPaths = settings.allowedImpureHostPrefixes; @@ -643,7 +643,7 @@ void LocalDerivationGoal::startBuilder() /* Allow files in __impureHostDeps to be missing; e.g. macOS 11+ has no /usr/lib/libSystem*.dylib */ - dirsInChroot[i] = {i, true}; + pathsInChroot[i] = {i, true}; } #if __linux__ @@ -711,15 +711,12 @@ void LocalDerivationGoal::startBuilder() for (auto & i : inputPaths) { auto p = worker.store.printStorePath(i); Path r = worker.store.toRealPath(p); - if (S_ISDIR(lstat(r).st_mode)) - dirsInChroot.insert_or_assign(p, r); - else - linkOrCopy(r, chrootRootDir + p); + pathsInChroot.insert_or_assign(p, r); } /* If we're repairing, checking or rebuilding part of a multiple-outputs derivation, it's possible that we're - rebuilding a path that is in settings.dirsInChroot + rebuilding a path that is in settings.sandbox-paths (typically the dependencies of /bin/sh). Throw them out. */ for (auto & i : drv->outputsAndOptPaths(worker.store)) { @@ -729,7 +726,7 @@ void LocalDerivationGoal::startBuilder() is already in the sandbox, so we don't need to worry about removing it. */ if (i.second.second) - dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); + pathsInChroot.erase(worker.store.printStorePath(*i.second.second)); } if (cgroup) { @@ -787,9 +784,9 @@ void LocalDerivationGoal::startBuilder() } else { auto p = line.find('='); if (p == std::string::npos) - dirsInChroot[line] = line; + pathsInChroot[line] = line; else - dirsInChroot[line.substr(0, p)] = line.substr(p + 1); + pathsInChroot[line.substr(0, p)] = line.substr(p + 1); } } } @@ -1779,7 +1776,7 @@ void LocalDerivationGoal::runChild() /* Set up a nearly empty /dev, unless the user asked to bind-mount the host /dev. */ Strings ss; - if (dirsInChroot.find("/dev") == dirsInChroot.end()) { + if (pathsInChroot.find("/dev") == pathsInChroot.end()) { createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); @@ -1814,34 +1811,15 @@ void LocalDerivationGoal::runChild() ss.push_back(path); if (settings.caFile != "") - dirsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true); + pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true); } - for (auto & i : ss) dirsInChroot.emplace(i, i); + for (auto & i : ss) pathsInChroot.emplace(i, i); /* Bind-mount all the directories from the "host" filesystem that we want in the chroot environment. */ - auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - debug("bind mounting '%1%' to '%2%'", source, target); - struct stat st; - if (stat(source.c_str(), &st) == -1) { - if (optional && errno == ENOENT) - return; - else - throw SysError("getting attributes of path '%1%'", source); - } - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); - }; - - for (auto & i : dirsInChroot) { + for (auto & i : pathsInChroot) { if (i.second.source == "/proc") continue; // backwards compatibility #if HAVE_EMBEDDED_SANDBOX_SHELL @@ -1882,7 +1860,7 @@ void LocalDerivationGoal::runChild() if /dev/ptx/ptmx exists). */ if (pathExists("/dev/pts/ptmx") && !pathExists(chrootRootDir + "/dev/ptmx") - && !dirsInChroot.count("/dev/pts")) + && !pathsInChroot.count("/dev/pts")) { if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) { @@ -2017,7 +1995,7 @@ void LocalDerivationGoal::runChild() /* We build the ancestry before adding all inputPaths to the store because we know they'll all have the same parents (the store), and there might be lots of inputs. This isn't particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : dirsInChroot) { + for (auto & i : pathsInChroot) { Path cur = i.first; while (cur.compare("/") != 0) { cur = dirOf(cur); @@ -2025,7 +2003,7 @@ void LocalDerivationGoal::runChild() } } - /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost + /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost path component this time, since it's typically /nix/store and we care about that. */ Path cur = worker.store.storeDir; while (cur.compare("/") != 0) { @@ -2036,7 +2014,7 @@ void LocalDerivationGoal::runChild() /* Add all our input paths to the chroot */ for (auto & i : inputPaths) { auto p = worker.store.printStorePath(i); - dirsInChroot[p] = p; + pathsInChroot[p] = p; } /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ @@ -2067,7 +2045,7 @@ void LocalDerivationGoal::runChild() without file-write* allowed, access() incorrectly returns EPERM */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : dirsInChroot) { + for (auto & i : pathsInChroot) { if (i.first != i.second.source) throw Error( "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 0a05081c7..05c2c3a56 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -86,8 +86,8 @@ struct LocalDerivationGoal : public DerivationGoal : source(source), optional(optional) { } }; - typedef map DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; + typedef map PathsInChroot; // maps target path to source path + PathsInChroot pathsInChroot; typedef map Environment; Environment env; From 630c2545d13cd8f6de4d678f19e89dfca375f62e Mon Sep 17 00:00:00 2001 From: Artturin Date: Thu, 14 Sep 2023 04:18:18 +0300 Subject: [PATCH 391/909] remove linkOrCopy and use bindmounts for files in addDependency --- src/libstore/build/local-derivation-goal.cc | 62 +++++++-------------- 1 file changed, 21 insertions(+), 41 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index a3ebfc681..204acab11 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -387,26 +387,6 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() } -#if __linux__ -static void linkOrCopy(const Path & from, const Path & to) -{ - if (link(from.c_str(), to.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum link count on a - file (e.g. 32000 of ext3), which is quite possible after a - 'nix-store --optimise'. FIXME: actually, why don't we just - bind-mount in this case? - - It can also fail with EPERM in BeegFS v7 and earlier versions - or fail with EXDEV in OpenAFS - which don't allow hard-links to other directories */ - if (errno != EMLINK && errno != EPERM && errno != EXDEV) - throw SysError("linking '%s' to '%s'", to, from); - copyPath(from, to); - } -} -#endif - - void LocalDerivationGoal::startBuilder() { if ((buildUser && buildUser->getUIDCount() != 1) @@ -1559,34 +1539,34 @@ void LocalDerivationGoal::addDependency(const StorePath & path) auto st = lstat(source); - if (S_ISDIR(st.st_mode)) { + /* Bind-mount the path into the sandbox. This requires + entering its mount namespace, which is not possible + in multithreaded programs. So we do this in a + child process.*/ + Pid child(startProcess([&]() { - /* Bind-mount the path into the sandbox. This requires - entering its mount namespace, which is not possible - in multithreaded programs. So we do this in a - child process.*/ - Pid child(startProcess([&]() { + if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1)) + throw SysError("entering sandbox user namespace"); - if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1)) - throw SysError("entering sandbox user namespace"); - - if (setns(sandboxMountNamespace.get(), 0) == -1) - throw SysError("entering sandbox mount namespace"); + if (setns(sandboxMountNamespace.get(), 0) == -1) + throw SysError("entering sandbox mount namespace"); + if (S_ISDIR(st.st_mode)) createDirs(target); + else { + createDirs(dirOf(target)); + writeFile(target, ""); + } - if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) - throw SysError("bind mount from '%s' to '%s' failed", source, target); + if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) + throw SysError("bind mount from '%s' to '%s' failed", source, target); - _exit(0); - })); + _exit(0); + })); - int status = child.wait(); - if (status != 0) - throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); - - } else - linkOrCopy(source, target); + int status = child.wait(); + if (status != 0) + throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); #else throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", From 11e47e7dfbca2f330b693665a4ee38302d1f6ad2 Mon Sep 17 00:00:00 2001 From: Artturin Date: Thu, 14 Sep 2023 04:36:25 +0300 Subject: [PATCH 392/909] factor out doBind from runChild --- src/libstore/build/local-derivation-goal.cc | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 204acab11..9592df43a 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -386,6 +386,26 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() cleanupPostOutputsRegisteredModeCheck(); } +#if __linux__ +static void doBind(const Path & source, const Path & target, bool optional = false) { + debug("bind mounting '%1%' to '%2%'", source, target); + struct stat st; + if (stat(source.c_str(), &st) == -1) { + if (optional && errno == ENOENT) + return; + else + throw SysError("getting attributes of path '%1%'", source); + } + if (S_ISDIR(st.st_mode)) + createDirs(target); + else { + createDirs(dirOf(target)); + writeFile(target, ""); + } + if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) + throw SysError("bind mount from '%1%' to '%2%' failed", source, target); +}; +#endif void LocalDerivationGoal::startBuilder() { From b8dfa3d53bda6793fe2c2566ae6e63e7c83718d8 Mon Sep 17 00:00:00 2001 From: Artturin Date: Thu, 14 Sep 2023 04:36:40 +0300 Subject: [PATCH 393/909] use doBind in addDependency --- src/libstore/build/local-derivation-goal.cc | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9592df43a..fd12b66b9 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1552,13 +1552,12 @@ void LocalDerivationGoal::addDependency(const StorePath & path) Path source = worker.store.Store::toRealPath(path); Path target = chrootRootDir + worker.store.printStorePath(path); - debug("bind-mounting %s -> %s", target, source); if (pathExists(target)) + // There is a similar debug message in doBind, so only run it in this block to not have double messages. + debug("bind-mounting %s -> %s", target, source); throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); - auto st = lstat(source); - /* Bind-mount the path into the sandbox. This requires entering its mount namespace, which is not possible in multithreaded programs. So we do this in a @@ -1571,15 +1570,7 @@ void LocalDerivationGoal::addDependency(const StorePath & path) if (setns(sandboxMountNamespace.get(), 0) == -1) throw SysError("entering sandbox mount namespace"); - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - - if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) - throw SysError("bind mount from '%s' to '%s' failed", source, target); + doBind(source, target); _exit(0); })); From dcc5f801f4fd88e08182b13304cdfa1b3aed8ece Mon Sep 17 00:00:00 2001 From: vicky1999 Date: Tue, 17 Oct 2023 09:39:59 +0530 Subject: [PATCH 394/909] Store info command help updates --- src/nix/{ping-store.cc => store-info.cc} | 8 ++++---- src/nix/{ping-store.md => store-info.md} | 6 +++--- tests/functional/{store-ping.sh => store-info.sh} | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) rename src/nix/{ping-store.cc => store-info.cc} (91%) rename src/nix/{ping-store.md => store-info.md} (82%) rename tests/functional/{store-ping.sh => store-info.sh} (69%) diff --git a/src/nix/ping-store.cc b/src/nix/store-info.cc similarity index 91% rename from src/nix/ping-store.cc rename to src/nix/store-info.cc index 247d9c6bb..a7c595761 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/store-info.cc @@ -17,7 +17,7 @@ struct CmdPingStore : StoreCommand, MixJSON std::string doc() override { return - #include "ping-store.md" + #include "store-info.md" ; } @@ -50,11 +50,11 @@ struct CmdInfoStore : CmdPingStore { void run(nix::ref store) override { - warn("'nix store info' is a deprecated alias for 'nix store ping'"); + warn("'nix store ping' is a deprecated alias for 'nix store info'"); CmdPingStore::run(store); } }; -static auto rCmdPingStore = registerCommand2({"store", "ping"}); -static auto rCmdInfoStore = registerCommand2({"store", "info"}); +static auto rCmdPingStore = registerCommand2({"store", "info"}); +static auto rCmdInfoStore = registerCommand2({"store", "ping"}); diff --git a/src/nix/ping-store.md b/src/nix/store-info.md similarity index 82% rename from src/nix/ping-store.md rename to src/nix/store-info.md index 8c846791b..f86efd722 100644 --- a/src/nix/ping-store.md +++ b/src/nix/store-info.md @@ -5,19 +5,19 @@ R""( * Test whether connecting to a remote Nix store via SSH works: ```console - # nix store ping --store ssh://mac1 + # nix store info --store ssh://mac1 ``` * Test whether a URL is a valid binary cache: ```console - # nix store ping --store https://cache.nixos.org + # nix store info --store https://cache.nixos.org ``` * Test whether the Nix daemon is up and running: ```console - # nix store ping --store daemon + # nix store info --store daemon ``` # Description diff --git a/tests/functional/store-ping.sh b/tests/functional/store-info.sh similarity index 69% rename from tests/functional/store-ping.sh rename to tests/functional/store-info.sh index 9846c7d3d..c002e50be 100644 --- a/tests/functional/store-ping.sh +++ b/tests/functional/store-info.sh @@ -1,7 +1,7 @@ source common.sh -STORE_INFO=$(nix store ping 2>&1) -STORE_INFO_JSON=$(nix store ping --json) +STORE_INFO=$(nix store info 2>&1) +STORE_INFO_JSON=$(nix store info --json) echo "$STORE_INFO" | grep "Store URL: ${NIX_REMOTE}" @@ -11,7 +11,7 @@ if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then [[ "$(echo "$STORE_INFO_JSON" | jq -r ".version")" == "$DAEMON_VERSION" ]] fi -expect 127 NIX_REMOTE=unix:$PWD/store nix store ping || \ - fail "nix store ping on a non-existent store should fail" +expect 127 NIX_REMOTE=unix:$PWD/store nix store info || \ + fail "nix store info on a non-existent store should fail" [[ "$(echo "$STORE_INFO_JSON" | jq -r ".url")" == "${NIX_REMOTE:-local}" ]] From f62b5500ff96cc46e610da13dcf72c134935642d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Oct 2023 14:52:34 +0200 Subject: [PATCH 395/909] fetchTree: Require the flakes experimental feature for the URL syntax --- src/libexpr/primops/fetchTree.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 0335e5f36..3431ff013 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -148,6 +148,11 @@ static void fetchTree( attrs.emplace("url", fixGitURL(url)); input = fetchers::Input::fromAttrs(std::move(attrs)); } else { + if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) + state.debugThrowLastTrace(EvalError({ + .msg = hintfmt("passing a string argument to 'fetchTree' requires the 'flakes' experimental feature"), + .errPos = state.positions[pos] + })); input = fetchers::Input::fromURL(url); } } @@ -180,6 +185,10 @@ static RegisterPrimOp primop_fetchTree({ *input* must be a [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references), either in attribute set representation or in the URL-like syntax. The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is enabled. + > **Note** + > + > The URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. + Here are some examples of how to use `fetchTree`: - Fetch a GitHub repository using the attribute set representation: From 3470cd68c46334d9d2c55e890f7fe50fa9ebe35c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Oct 2023 14:57:29 +0200 Subject: [PATCH 396/909] Mark some fetchers as experimental --- src/libfetchers/github.cc | 5 +++++ src/libfetchers/path.cc | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 291f457f0..6c5c792ec 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -223,6 +223,11 @@ struct GitArchiveInputScheme : InputScheme return {result.tree.storePath, input}; } + + std::optional experimentalFeature() override + { + return Xp::Flakes; + } }; struct GitHubInputScheme : GitArchiveInputScheme diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 01f1be978..092975f5d 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -125,6 +125,11 @@ struct PathInputScheme : InputScheme return {std::move(*storePath), input}; } + + std::optional experimentalFeature() override + { + return Xp::Flakes; + } }; static auto rPathInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); From ff684260950cd31efffbe9804494d42bc226df59 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 17 Oct 2023 11:15:36 -0400 Subject: [PATCH 397/909] Name the protocol version types This makes the code clearer, and will help us replace them with proper structs and get rid of the macros later. --- src/libstore/daemon.cc | 10 +++++----- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/remote-store-connection.hh | 2 +- src/libstore/serve-protocol.hh | 7 +++++++ src/libstore/worker-protocol.hh | 7 +++++++ src/nix-store/nix-store.cc | 2 +- 6 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 8cbf6f044..fbff64e89 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -45,9 +45,9 @@ struct TunnelLogger : public Logger Sync state_; - unsigned int clientVersion; + WorkerProto::Version clientVersion; - TunnelLogger(FdSink & to, unsigned int clientVersion) + TunnelLogger(FdSink & to, WorkerProto::Version clientVersion) : to(to), clientVersion(clientVersion) { } void enqueueMsg(const std::string & s) @@ -261,7 +261,7 @@ struct ClientSettings } }; -static std::vector readDerivedPaths(Store & store, unsigned int clientVersion, WorkerProto::ReadConn conn) +static std::vector readDerivedPaths(Store & store, WorkerProto::Version clientVersion, WorkerProto::ReadConn conn) { std::vector reqs; if (GET_PROTOCOL_MINOR(clientVersion) >= 30) { @@ -274,7 +274,7 @@ static std::vector readDerivedPaths(Store & store, unsigned int cli } static void performOp(TunnelLogger * logger, ref store, - TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, + TrustedFlag trusted, RecursiveFlag recursive, WorkerProto::Version clientVersion, Source & from, BufferedSink & to, WorkerProto::Op op) { WorkerProto::ReadConn rconn { .from = from }; @@ -1017,7 +1017,7 @@ void processConnection( if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); to << WORKER_MAGIC_2 << PROTOCOL_VERSION; to.flush(); - unsigned int clientVersion = readInt(from); + WorkerProto::Version clientVersion = readInt(from); if (clientVersion < 0x10a) throw Error("the Nix client version is too old"); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 703ded0b2..efced20d0 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -45,7 +45,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor std::unique_ptr sshConn; FdSink to; FdSource from; - int remoteVersion; + ServeProto::Version remoteVersion; bool good = true; /** diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh index ce4740a9c..8738e4d95 100644 --- a/src/libstore/remote-store-connection.hh +++ b/src/libstore/remote-store-connection.hh @@ -30,7 +30,7 @@ struct RemoteStore::Connection * sides support. (If the maximum doesn't exist, we would fail to * establish a connection and produce a value of this type.) */ - unsigned int daemonVersion; + WorkerProto::Version daemonVersion; /** * Whether the remote side trusts us or not. diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index e2345d450..dddb7c54d 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -30,6 +30,13 @@ struct ServeProto */ enum struct Command : uint64_t; + /** + * Version type for the protocol. + * + * @todo Convert to struct with separate major vs minor fields. + */ + using Version = unsigned int; + /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index c84060103..e9fc8d957 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -47,6 +47,13 @@ struct WorkerProto */ enum struct Op : uint64_t; + /** + * Version type for the protocol. + * + * @todo Convert to struct with separate major vs minor fields. + */ + using Version = unsigned int; + /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 9b6c80a75..ea53e49b0 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -818,7 +818,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch"); out << SERVE_MAGIC_2 << SERVE_PROTOCOL_VERSION; out.flush(); - unsigned int clientVersion = readInt(in); + ServeProto::Version clientVersion = readInt(in); ServeProto::ReadConn rconn { .from = in }; ServeProto::WriteConn wconn { .to = out }; From e36c9175f49c0fac4f0511d4cae5ab991fd9cb7d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Mar 2022 04:40:49 +0000 Subject: [PATCH 398/909] Add protocol versions to `{Worker,Serve}Proto::*Conn` This will allow us to factor out logic, which is currently scattered inline, into several reusable instances The tests are also updated to support versioning. Currently all Worker and Serve protocol tests are using the minimum version, since no version-specific serialisers have been created yet. But in subsequent commits when that changes, we will test individual versions to ensure complete coverage. --- src/libstore/daemon.cc | 15 ++++- src/libstore/legacy-ssh-store.cc | 2 + src/libstore/path-info.cc | 10 +++- src/libstore/remote-store-connection.hh | 2 + src/libstore/serve-protocol.hh | 8 +-- src/libstore/tests/common-protocol.cc | 73 +++++++++++++++++++++---- src/libstore/tests/protocol.hh | 45 ++++++++------- src/libstore/tests/serve-protocol.cc | 38 +++++++++---- src/libstore/tests/worker-protocol.cc | 49 ++++++++++++----- src/libstore/worker-protocol.hh | 8 +-- src/nix-store/nix-store.cc | 10 +++- 11 files changed, 185 insertions(+), 75 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index fbff64e89..d61e97a64 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -277,8 +277,14 @@ static void performOp(TunnelLogger * logger, ref store, TrustedFlag trusted, RecursiveFlag recursive, WorkerProto::Version clientVersion, Source & from, BufferedSink & to, WorkerProto::Op op) { - WorkerProto::ReadConn rconn { .from = from }; - WorkerProto::WriteConn wconn { .to = to }; + WorkerProto::ReadConn rconn { + .from = from, + .version = clientVersion, + }; + WorkerProto::WriteConn wconn { + .to = to, + .version = clientVersion, + }; switch (op) { @@ -1052,7 +1058,10 @@ void processConnection( auto temp = trusted ? store->isTrustedClient() : std::optional { NotTrusted }; - WorkerProto::WriteConn wconn { .to = to }; + WorkerProto::WriteConn wconn { + .to = to, + .version = clientVersion, + }; WorkerProto::write(*store, wconn, temp); } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index efced20d0..9143be72e 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -60,6 +60,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor { return ServeProto::ReadConn { .from = from, + .version = remoteVersion, }; } @@ -75,6 +76,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor { return ServeProto::WriteConn { .to = to, + .version = remoteVersion, }; } }; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index ccb57104f..a9231ce8d 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -141,7 +141,10 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned ValidPathInfo info(path, narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); info.references = WorkerProto::Serialise::read(store, - WorkerProto::ReadConn { .from = source }); + WorkerProto::ReadConn { + .from = source, + .version = format, + }); source >> info.registrationTime >> info.narSize; if (format >= 16) { source >> info.ultimate; @@ -163,7 +166,10 @@ void ValidPathInfo::write( sink << (deriver ? store.printStorePath(*deriver) : "") << narHash.to_string(Base16, false); WorkerProto::write(store, - WorkerProto::WriteConn { .to = sink }, + WorkerProto::WriteConn { + .to = sink, + .version = format, + }, references); sink << registrationTime << narSize; if (format >= 16) { diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh index 8738e4d95..e4a9cacb9 100644 --- a/src/libstore/remote-store-connection.hh +++ b/src/libstore/remote-store-connection.hh @@ -70,6 +70,7 @@ struct RemoteStore::Connection { return WorkerProto::ReadConn { .from = from, + .version = daemonVersion, }; } @@ -85,6 +86,7 @@ struct RemoteStore::Connection { return WorkerProto::WriteConn { .to = to, + .version = daemonVersion, }; } diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index dddb7c54d..a627c6ad6 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -40,23 +40,19 @@ struct ServeProto /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. - * - * This currently is just a `Source &`, but more fields will be added - * later. */ struct ReadConn { Source & from; + Version version; }; /** * A unidirectional write connection, to be used by the write half of the * canonical serializers below. - * - * This currently is just a `Sink &`, but more fields will be added - * later. */ struct WriteConn { Sink & to; + Version version; }; /** diff --git a/src/libstore/tests/common-protocol.cc b/src/libstore/tests/common-protocol.cc index de57211f0..61c2cb70c 100644 --- a/src/libstore/tests/common-protocol.cc +++ b/src/libstore/tests/common-protocol.cc @@ -13,10 +13,71 @@ namespace nix { const char commonProtoDir[] = "common-protocol"; -using CommonProtoTest = ProtoTest; +class CommonProtoTest : public ProtoTest +{ +public: + /** + * Golden test for `T` reading + */ + template + void readTest(PathView testStem, T value) + { + if (testAccept()) + { + GTEST_SKIP() << "Cannot read golden master because another test is also updating it"; + } + else + { + auto expected = readFile(goldenMaster(testStem)); + + T got = ({ + StringSource from { expected }; + CommonProto::Serialise::read( + *store, + CommonProto::ReadConn { .from = from }); + }); + + ASSERT_EQ(got, value); + } + } + + /** + * Golden test for `T` write + */ + template + void writeTest(PathView testStem, const T & value) + { + auto file = goldenMaster(testStem); + + StringSink to; + CommonProto::write( + *store, + CommonProto::WriteConn { .to = to }, + value); + + if (testAccept()) + { + createDirs(dirOf(file)); + writeFile(file, to.s); + GTEST_SKIP() << "Updating golden master"; + } + else + { + auto expected = readFile(file); + ASSERT_EQ(to.s, expected); + } + } +}; + +#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + TEST_F(CommonProtoTest, NAME ## _read) { \ + readTest(STEM, VALUE); \ + } \ + TEST_F(CommonProtoTest, NAME ## _write) { \ + writeTest(STEM, VALUE); \ + } CHARACTERIZATION_TEST( - CommonProtoTest, string, "string", (std::tuple { @@ -28,7 +89,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, storePath, "store-path", (std::tuple { @@ -37,7 +97,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, contentAddress, "content-address", (std::tuple { @@ -56,7 +115,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, drvOutput, "drv-output", (std::tuple { @@ -71,7 +129,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, realisation, "realisation", (std::tuple { @@ -103,7 +160,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, vector, "vector", (std::tuple, std::vector, std::vector, std::vector>> { @@ -114,7 +170,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, set, "set", (std::tuple, std::set, std::set, std::set>> { @@ -125,7 +180,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, optionalStorePath, "optional-store-path", (std::tuple, std::optional> { @@ -136,7 +190,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, optionalContentAddress, "optional-content-address", (std::tuple, std::optional> { diff --git a/src/libstore/tests/protocol.hh b/src/libstore/tests/protocol.hh index 86a900757..496915745 100644 --- a/src/libstore/tests/protocol.hh +++ b/src/libstore/tests/protocol.hh @@ -9,26 +9,23 @@ namespace nix { template class ProtoTest : public LibStoreTest { - /** - * Read this as simply `using S = Inner::Serialise;`. - * - * See `LengthPrefixedProtoHelper::S` for the same trick, and its - * rationale. - */ - template using S = typename Proto::template Serialise; - -public: +protected: Path unitTestData = getUnitTestData() + "/libstore/" + protocolDir; Path goldenMaster(std::string_view testStem) { return unitTestData + "/" + testStem + ".bin"; } +}; +template +class VersionedProtoTest : public ProtoTest +{ +public: /** * Golden test for `T` reading */ template - void readTest(PathView testStem, T value) + void readTest(PathView testStem, typename Proto::Version version, T value) { if (testAccept()) { @@ -36,13 +33,16 @@ public: } else { - auto expected = readFile(goldenMaster(testStem)); + auto expected = readFile(ProtoTest::goldenMaster(testStem)); T got = ({ StringSource from { expected }; - S::read( - *store, - typename Proto::ReadConn { .from = from }); + Proto::template Serialise::read( + *LibStoreTest::store, + typename Proto::ReadConn { + .from = from, + .version = version, + }); }); ASSERT_EQ(got, value); @@ -53,14 +53,17 @@ public: * Golden test for `T` write */ template - void writeTest(PathView testStem, const T & value) + void writeTest(PathView testStem, typename Proto::Version version, const T & value) { - auto file = goldenMaster(testStem); + auto file = ProtoTest::goldenMaster(testStem); StringSink to; Proto::write( - *store, - typename Proto::WriteConn { .to = to }, + *LibStoreTest::store, + typename Proto::WriteConn { + .to = to, + .version = version, + }, value); if (testAccept()) @@ -77,12 +80,12 @@ public: } }; -#define CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VALUE) \ +#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ TEST_F(FIXTURE, NAME ## _read) { \ - readTest(STEM, VALUE); \ + readTest(STEM, VERSION, VALUE); \ } \ TEST_F(FIXTURE, NAME ## _write) { \ - writeTest(STEM, VALUE); \ + writeTest(STEM, VERSION, VALUE); \ } } diff --git a/src/libstore/tests/serve-protocol.cc b/src/libstore/tests/serve-protocol.cc index 5d7df56cf..ba798ab1c 100644 --- a/src/libstore/tests/serve-protocol.cc +++ b/src/libstore/tests/serve-protocol.cc @@ -11,14 +11,22 @@ namespace nix { -const char commonProtoDir[] = "serve-protocol"; +const char serveProtoDir[] = "serve-protocol"; -using ServeProtoTest = ProtoTest; +struct ServeProtoTest : VersionedProtoTest +{ + /** + * For serializers that don't care about the minimum version, we + * used the oldest one: 1.0. + */ + ServeProto::Version defaultVersion = 1 << 8 | 0; +}; -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, string, "string", + defaultVersion, (std::tuple { "", "hi", @@ -27,19 +35,21 @@ CHARACTERIZATION_TEST( "oh no \0\0\0 what was that!", })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, storePath, "store-path", + defaultVersion, (std::tuple { StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, contentAddress, "content-address", + defaultVersion, (std::tuple { ContentAddress { .method = TextIngestionMethod {}, @@ -55,10 +65,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, drvOutput, "drv-output", + defaultVersion, (std::tuple { { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), @@ -70,10 +81,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, realisation, "realisation", + defaultVersion, (std::tuple { Realisation { .id = DrvOutput { @@ -102,10 +114,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, vector, "vector", + defaultVersion, (std::tuple, std::vector, std::vector, std::vector>> { { }, { "" }, @@ -113,10 +126,11 @@ CHARACTERIZATION_TEST( { {}, { "" }, { "", "1", "2" } }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, set, "set", + defaultVersion, (std::tuple, std::set, std::set, std::set>> { { }, { "" }, @@ -124,10 +138,11 @@ CHARACTERIZATION_TEST( { {}, { "" }, { "", "1", "2" } }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, optionalStorePath, "optional-store-path", + defaultVersion, (std::tuple, std::optional> { std::nullopt, std::optional { @@ -135,10 +150,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, optionalContentAddress, "optional-content-address", + defaultVersion, (std::tuple, std::optional> { std::nullopt, std::optional { diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index 59d7ff96d..63fecce96 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -14,12 +14,21 @@ namespace nix { const char workerProtoDir[] = "worker-protocol"; -using WorkerProtoTest = ProtoTest; +struct WorkerProtoTest : VersionedProtoTest +{ + /** + * For serializers that don't care about the minimum version, we + * used the oldest one: 1.0. + */ + WorkerProto::Version defaultVersion = 1 << 8 | 0; +}; -CHARACTERIZATION_TEST( + +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, string, "string", + defaultVersion, (std::tuple { "", "hi", @@ -28,19 +37,21 @@ CHARACTERIZATION_TEST( "oh no \0\0\0 what was that!", })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, storePath, "store-path", + defaultVersion, (std::tuple { StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, contentAddress, "content-address", + defaultVersion, (std::tuple { ContentAddress { .method = TextIngestionMethod {}, @@ -56,10 +67,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, derivedPath, "derived-path", + defaultVersion, (std::tuple { DerivedPath::Opaque { .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, @@ -72,10 +84,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, drvOutput, "drv-output", + defaultVersion, (std::tuple { { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), @@ -87,10 +100,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, realisation, "realisation", + defaultVersion, (std::tuple { Realisation { .id = DrvOutput { @@ -119,10 +133,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult, "build-result", + defaultVersion, ({ using namespace std::literals::chrono_literals; std::tuple t { @@ -177,10 +192,11 @@ CHARACTERIZATION_TEST( t; })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, keyedBuildResult, "keyed-build-result", + defaultVersion, ({ using namespace std::literals::chrono_literals; std::tuple t { @@ -213,20 +229,22 @@ CHARACTERIZATION_TEST( t; })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, optionalTrustedFlag, "optional-trusted-flag", + defaultVersion, (std::tuple, std::optional, std::optional> { std::nullopt, std::optional { Trusted }, std::optional { NotTrusted }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, vector, "vector", + defaultVersion, (std::tuple, std::vector, std::vector, std::vector>> { { }, { "" }, @@ -234,10 +252,11 @@ CHARACTERIZATION_TEST( { {}, { "" }, { "", "1", "2" } }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, set, "set", + defaultVersion, (std::tuple, std::set, std::set, std::set>> { { }, { "" }, @@ -245,10 +264,11 @@ CHARACTERIZATION_TEST( { {}, { "" }, { "", "1", "2" } }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, optionalStorePath, "optional-store-path", + defaultVersion, (std::tuple, std::optional> { std::nullopt, std::optional { @@ -256,10 +276,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, optionalContentAddress, "optional-content-address", + defaultVersion, (std::tuple, std::optional> { std::nullopt, std::optional { diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index e9fc8d957..2d55d926a 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -57,23 +57,19 @@ struct WorkerProto /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. - * - * This currently is just a `Source &`, but more fields will be added - * later. */ struct ReadConn { Source & from; + Version version; }; /** * A unidirectional write connection, to be used by the write half of the * canonical serializers below. - * - * This currently is just a `Sink &`, but more fields will be added - * later. */ struct WriteConn { Sink & to; + Version version; }; /** diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index ea53e49b0..11d2e86e3 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -820,8 +820,14 @@ static void opServe(Strings opFlags, Strings opArgs) out.flush(); ServeProto::Version clientVersion = readInt(in); - ServeProto::ReadConn rconn { .from = in }; - ServeProto::WriteConn wconn { .to = out }; + ServeProto::ReadConn rconn { + .from = in, + .version = clientVersion, + }; + ServeProto::WriteConn wconn { + .to = out, + .version = clientVersion, + }; auto getBuildSettings = [&]() { // FIXME: changing options here doesn't work if we're From a0f071f1d3b05140fa1c97de5a7612d1b938ffbf Mon Sep 17 00:00:00 2001 From: vicky1999 Date: Wed, 18 Oct 2023 00:12:10 +0530 Subject: [PATCH 399/909] store info sh renamed --- tests/functional/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 643351163..3679349f8 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -113,7 +113,7 @@ nix_tests = \ pass-as-file.sh \ nix-profile.sh \ suggestions.sh \ - store-ping.sh \ + store-info.sh \ fetchClosure.sh \ completions.sh \ flakes/show.sh \ From 891dfb435943f68f165a0c5921f0af8e9e7362e8 Mon Sep 17 00:00:00 2001 From: vicky1999 Date: Wed, 18 Oct 2023 00:14:11 +0530 Subject: [PATCH 400/909] updated store ping to store info in files --- doc/manual/src/advanced-topics/distributed-builds.md | 4 ++-- doc/manual/src/release-notes/rl-2.15.md | 2 +- doc/manual/src/release-notes/rl-2.7.md | 2 +- tests/functional/legacy-ssh-store.sh | 4 ++-- tests/functional/local-store.sh | 4 ++-- tests/functional/nix-copy-ssh-ng.sh | 2 +- tests/functional/remote-store.sh | 8 ++++---- tests/installer/default.nix | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index 73a113d35..507c5ecb7 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -12,14 +12,14 @@ machine is accessible via SSH and that it has Nix installed. You can test whether connecting to the remote Nix instance works, e.g. ```console -$ nix store ping --store ssh://mac +$ nix store info --store ssh://mac ``` will try to connect to the machine named `mac`. It is possible to specify an SSH identity file as part of the remote store URI, e.g. ```console -$ nix store ping --store ssh://mac?ssh-key=/home/alice/my-key +$ nix store info --store ssh://mac?ssh-key=/home/alice/my-key ``` Since builds should be non-interactive, the key should not have a diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md index 133121999..4faf0b143 100644 --- a/doc/manual/src/release-notes/rl-2.15.md +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -44,7 +44,7 @@ (The store always had to check whether it trusts the client, but now the client is informed of the store's decision.) This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores. - `nix store ping` and `nix doctor` now display this information. + `nix store info` and `nix doctor` now display this information. * The new command `nix derivation add` allows adding derivations to the store without involving the Nix language. It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store. diff --git a/doc/manual/src/release-notes/rl-2.7.md b/doc/manual/src/release-notes/rl-2.7.md index 2f3879422..dd649e166 100644 --- a/doc/manual/src/release-notes/rl-2.7.md +++ b/doc/manual/src/release-notes/rl-2.7.md @@ -24,7 +24,7 @@ [repository](https://github.com/NixOS/bundlers) has various bundlers implemented. -* `nix store ping` now reports the version of the remote Nix daemon. +* `nix store info` now reports the version of the remote Nix daemon. * `nix flake {init,new}` now display information about which files have been created. diff --git a/tests/functional/legacy-ssh-store.sh b/tests/functional/legacy-ssh-store.sh index 71b716b84..894efccd4 100644 --- a/tests/functional/legacy-ssh-store.sh +++ b/tests/functional/legacy-ssh-store.sh @@ -1,4 +1,4 @@ source common.sh -# Check that store ping trusted doesn't yet work with ssh:// -nix --store ssh://localhost?remote-store=$TEST_ROOT/other-store store ping --json | jq -e 'has("trusted") | not' +# Check that store info trusted doesn't yet work with ssh:// +nix --store ssh://localhost?remote-store=$TEST_ROOT/other-store store info --json | jq -e 'has("trusted") | not' diff --git a/tests/functional/local-store.sh b/tests/functional/local-store.sh index 89502f864..f7c8eb3f1 100644 --- a/tests/functional/local-store.sh +++ b/tests/functional/local-store.sh @@ -18,5 +18,5 @@ PATH2=$(nix path-info --store "$PWD/x" $CORRECT_PATH) PATH3=$(nix path-info --store "local?root=$PWD/x" $CORRECT_PATH) [ $CORRECT_PATH == $PATH3 ] -# Ensure store ping trusted works with local store -nix --store ./x store ping --json | jq -e '.trusted' +# Ensure store info trusted works with local store +nix --store ./x store info --json | jq -e '.trusted' diff --git a/tests/functional/nix-copy-ssh-ng.sh b/tests/functional/nix-copy-ssh-ng.sh index 45e53c9c0..463b5e0c4 100644 --- a/tests/functional/nix-copy-ssh-ng.sh +++ b/tests/functional/nix-copy-ssh-ng.sh @@ -9,7 +9,7 @@ rm -rf "$remoteRoot" outPath=$(nix-build --no-out-link dependencies.nix) -nix store ping --store "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" +nix store info --store "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" # Regression test for https://github.com/NixOS/nix/issues/6253 nix copy --to "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath --no-check-sigs & diff --git a/tests/functional/remote-store.sh b/tests/functional/remote-store.sh index 50e6f24b9..5c7bfde46 100644 --- a/tests/functional/remote-store.sh +++ b/tests/functional/remote-store.sh @@ -5,17 +5,17 @@ clearStore # Ensure "fake ssh" remote store works just as legacy fake ssh would. nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store doctor -# Ensure that store ping trusted works with ssh-ng:// -nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store store ping --json | jq -e '.trusted' +# Ensure that store info trusted works with ssh-ng:// +nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store store info --json | jq -e '.trusted' startDaemon if isDaemonNewer "2.15pre0"; then # Ensure that ping works trusted with new daemon - nix store ping --json | jq -e '.trusted' + nix store info --json | jq -e '.trusted' else # And the the field is absent with the old daemon - nix store ping --json | jq -e 'has("trusted") | not' + nix store info --json | jq -e 'has("trusted") | not' fi # Test import-from-derivation through the daemon. diff --git a/tests/installer/default.nix b/tests/installer/default.nix index 49cfd2bcc..238c6ac8e 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -213,7 +213,7 @@ let source /etc/bashrc || true nix-env --version - nix --extra-experimental-features nix-command store ping + nix --extra-experimental-features nix-command store info out=\$(nix-build --no-substitute -E 'derivation { name = "foo"; system = "x86_64-linux"; builder = "/bin/sh"; args = ["-c" "echo foobar > \$out"]; }') [[ \$(cat \$out) = foobar ]] From a2f0ba6a6dbb79efbb83ebe92b287f79b3f3af91 Mon Sep 17 00:00:00 2001 From: Vladimir Kryachko Date: Mon, 16 Oct 2023 16:51:49 -0400 Subject: [PATCH 401/909] Fix transitive input locking. Fixes reproducibility issue described in #9143 Fixes #9143 --- src/libexpr/flake/flake.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index a6212c12f..5d9d60655 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -351,10 +351,13 @@ LockedFlake lockFlake( debug("old lock file: %s", oldLockFile); std::map overrides; + std::set explicitCliOverrides; std::set overridesUsed, updatesUsed; - for (auto & i : lockFlags.inputOverrides) + for (auto & i : lockFlags.inputOverrides) { overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); + explicitCliOverrides.insert(i.first); + } LockFile newLockFile; @@ -425,6 +428,7 @@ LockedFlake lockFlake( ancestors? */ auto i = overrides.find(inputPath); bool hasOverride = i != overrides.end(); + bool hasCliOverride = explicitCliOverrides.find(inputPath) != explicitCliOverrides.end(); if (hasOverride) { overridesUsed.insert(inputPath); // Respect the “flakeness” of the input even if we @@ -460,7 +464,7 @@ LockedFlake lockFlake( if (oldLock && oldLock->originalRef == *input.ref - && !hasOverride) + && !hasCliOverride) { debug("keeping existing input '%s'", inputPathS); @@ -541,7 +545,7 @@ LockedFlake lockFlake( nuked the next time we update the lock file. That is, overrides are sticky unless you use --no-write-lock-file. */ - auto ref = input2.ref ? *input2.ref : *input.ref; + auto ref = (input2.ref && explicitCliOverrides.contains(inputPath)) ? *input2.ref : *input.ref; if (input.isFlake) { Path localPath = parentPath; From 311e2ad024441950cb1300e56c9745259deebdda Mon Sep 17 00:00:00 2001 From: Vladimir Kryachko Date: Wed, 18 Oct 2023 10:37:06 -0400 Subject: [PATCH 402/909] Address review comments --- src/libexpr/flake/flake.cc | 2 +- tests/functional/flakes/follow-paths.sh | 76 +++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 5d9d60655..2c7e12ec9 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -428,7 +428,7 @@ LockedFlake lockFlake( ancestors? */ auto i = overrides.find(inputPath); bool hasOverride = i != overrides.end(); - bool hasCliOverride = explicitCliOverrides.find(inputPath) != explicitCliOverrides.end(); + bool hasCliOverride = explicitCliOverrides.contains(inputPath); if (hasOverride) { overridesUsed.insert(inputPath); // Respect the “flakeness” of the input even if we diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh index 8573b5511..7f4e8bf5d 100644 --- a/tests/functional/flakes/follow-paths.sh +++ b/tests/functional/flakes/follow-paths.sh @@ -260,3 +260,79 @@ EOF checkRes=$(nix flake lock "$flakeFollowCycle" 2>&1 && fail "nix flake lock should have failed." || true) echo $checkRes | grep -F "error: follow cycle detected: [baz -> foo -> bar -> baz]" + + +# Test transitive input url locking +# This tests the following lockfile issue: https://github.com/NixOS/nix/issues/9143 +# +# We construct the following graph, where p->q means p has input q. +# +# A -> B -> C +# +# And override B/C to flake D, first in A's flake.nix and then with --override-input. +# +# A -> B -> D +flakeFollowsCustomUrlA="$TEST_ROOT/follows/custom-url/flakeA" +flakeFollowsCustomUrlB="$TEST_ROOT/follows/custom-url/flakeA/flakeB" +flakeFollowsCustomUrlC="$TEST_ROOT/follows/custom-url/flakeA/flakeB/flakeC" +flakeFollowsCustomUrlD="$TEST_ROOT/follows/custom-url/flakeA/flakeB/flakeD" + + +createGitRepo "$flakeFollowsCustomUrlA" +mkdir -p "$flakeFollowsCustomUrlB" +mkdir -p "$flakeFollowsCustomUrlC" +mkdir -p "$flakeFollowsCustomUrlD" + +cat > "$flakeFollowsCustomUrlD/flake.nix" < "$flakeFollowsCustomUrlC/flake.nix" < "$flakeFollowsCustomUrlB/flake.nix" < "$flakeFollowsCustomUrlA/flake.nix" < Date: Wed, 18 Oct 2023 15:32:31 +0200 Subject: [PATCH 403/909] Introduce FSInputAccessor and use it Backported from the lazy-trees branch. Note that this doesn't yet use the access control features of FSInputAccessor. --- src/libexpr/attr-path.cc | 2 +- src/libexpr/eval.cc | 35 +++-- src/libexpr/eval.hh | 3 + src/libexpr/flake/flake.cc | 6 +- src/libexpr/nixexpr.hh | 7 +- src/libexpr/parser.y | 9 +- src/libexpr/paths.cc | 3 +- src/libexpr/primops.cc | 18 +-- src/libexpr/tests/json.cc | 2 +- src/libexpr/tests/libexpr.hh | 15 ++- src/libexpr/value.hh | 17 ++- src/libfetchers/fs-input-accessor.cc | 141 ++++++++++++++++++++ src/libfetchers/fs-input-accessor.hh | 33 +++++ src/libfetchers/input-accessor.cc | 189 ++++++++++++++++++++------- src/libfetchers/input-accessor.hh | 105 ++++++++++++--- src/libstore/store-api.cc | 14 +- src/libstore/store-api.hh | 15 ++- src/nix/main.cc | 14 +- 18 files changed, 502 insertions(+), 126 deletions(-) create mode 100644 src/libfetchers/fs-input-accessor.cc create mode 100644 src/libfetchers/fs-input-accessor.hh diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index ab654c1b0..d12345710 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -132,7 +132,7 @@ std::pair findPackageFilename(EvalState & state, Value & v if (colon == std::string::npos) fail(); std::string filename(fn, 0, colon); auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos)); - return {CanonPath(fn.substr(0, colon)), lineno}; + return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno}; } catch (std::invalid_argument & e) { fail(); abort(); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 511a5f434..87791fa52 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -12,6 +12,7 @@ #include "function-trace.hh" #include "profiles.hh" #include "print.hh" +#include "fs-input-accessor.hh" #include #include @@ -503,6 +504,18 @@ EvalState::EvalState( , sOutputSpecified(symbols.create("outputSpecified")) , repair(NoRepair) , emptyBindings(0) + , rootFS( + makeFSInputAccessor( + CanonPath::root, + evalSettings.restrictEval || evalSettings.pureEval + ? std::optional>(std::set()) + : std::nullopt, + [](const CanonPath & path) -> RestrictedPathError { + auto modeInformation = evalSettings.pureEval + ? "in pure evaluation mode (use '--impure' to override)" + : "in restricted mode"; + throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); + })) , derivationInternal(rootPath(CanonPath("/builtin/derivation.nix"))) , store(store) , buildStore(buildStore ? buildStore : store) @@ -518,6 +531,8 @@ EvalState::EvalState( , baseEnv(allocEnv(128)) , staticBaseEnv{std::make_shared(false, nullptr)} { + rootFS->allowPath(CanonPath::root); // FIXME + countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; assert(gcInitialised); @@ -599,7 +614,7 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_) */ Path abspath = canonPath(path_.path.abs()); - if (hasPrefix(abspath, corepkgsPrefix)) return CanonPath(abspath); + if (hasPrefix(abspath, corepkgsPrefix)) return rootPath(CanonPath(abspath)); for (auto & i : *allowedPaths) { if (isDirOrInDir(abspath, i)) { @@ -617,7 +632,7 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_) /* Resolve symlinks. */ debug("checking access to '%s'", abspath); - SourcePath path = CanonPath(canonPath(abspath, true)); + SourcePath path = rootPath(CanonPath(canonPath(abspath, true))); for (auto & i : *allowedPaths) { if (isDirOrInDir(path.path.abs(), i)) { @@ -649,12 +664,12 @@ void EvalState::checkURI(const std::string & uri) /* If the URI is a path, then check it against allowedPaths as well. */ if (hasPrefix(uri, "/")) { - checkSourcePath(CanonPath(uri)); + checkSourcePath(rootPath(CanonPath(uri))); return; } if (hasPrefix(uri, "file://")) { - checkSourcePath(CanonPath(std::string(uri, 7))); + checkSourcePath(rootPath(CanonPath(std::string(uri, 7)))); return; } @@ -950,7 +965,7 @@ void Value::mkStringMove(const char * s, const NixStringContext & context) void Value::mkPath(const SourcePath & path) { - mkPath(makeImmutableString(path.path.abs())); + mkPath(&*path.accessor, makeImmutableString(path.path.abs())); } @@ -2037,7 +2052,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) else if (firstType == nPath) { if (!context.empty()) state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow(); - v.mkPath(CanonPath(canonPath(str()))); + v.mkPath(state.rootPath(CanonPath(canonPath(str())))); } else v.mkStringMove(c_str(), context); } @@ -2236,7 +2251,7 @@ BackedStringView EvalState::coerceToString( !canonicalizePath && !copyToStore ? // FIXME: hack to preserve path literals that end in a // slash, as in /foo/${x}. - v._path + v._path.path : copyToStore ? store->printStorePath(copyPathToStore(context, v.path())) : std::string(v.path().path.abs()); @@ -2329,7 +2344,7 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (path == "" || path[0] != '/') error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow(); - return CanonPath(path); + return rootPath(CanonPath(path)); } @@ -2429,7 +2444,9 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v return v1.string_view().compare(v2.string_view()) == 0; case nPath: - return strcmp(v1._path, v2._path) == 0; + return + v1._path.accessor == v2._path.accessor + && strcmp(v1._path.path, v2._path.path) == 0; case nNull: return true; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index c95558952..4bb0a6a0b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -24,6 +24,7 @@ class EvalState; class StorePath; struct SingleDerivedPath; enum RepairFlag : bool; +struct FSInputAccessor; /** @@ -211,6 +212,8 @@ public: Bindings emptyBindings; + const ref rootFS; + const SourcePath derivationInternal; /** diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index a6212c12f..2f91ad433 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -223,9 +223,9 @@ static Flake getFlake( throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); Value vInfo; - state.evalFile(CanonPath(flakeFile), vInfo, true); // FIXME: symlink attack + state.evalFile(state.rootPath(CanonPath(flakeFile)), vInfo, true); // FIXME: symlink attack - expectType(state, nAttrs, vInfo, state.positions.add({CanonPath(flakeFile)}, 1, 1)); + expectType(state, nAttrs, vInfo, state.positions.add({state.rootPath(CanonPath(flakeFile))}, 1, 1)); if (auto description = vInfo.attrs->get(state.sDescription)) { expectType(state, nString, *description->value, description->pos); @@ -741,7 +741,7 @@ void callFlake(EvalState & state, state.vCallFlake = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "call-flake.nix.gen.hh" - , CanonPath::root), **state.vCallFlake); + , state.rootPath(CanonPath::root)), **state.vCallFlake); } state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos); diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index e766996c8..10099d49e 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -20,7 +20,6 @@ MakeError(Abort, EvalError); MakeError(TypeError, EvalError); MakeError(UndefinedVarError, Error); MakeError(MissingArgumentError, EvalError); -MakeError(RestrictedPathError, Error); /** * Position objects. @@ -200,9 +199,13 @@ struct ExprString : Expr struct ExprPath : Expr { + ref accessor; std::string s; Value v; - ExprPath(std::string s) : s(std::move(s)) { v.mkPath(this->s.c_str()); }; + ExprPath(ref accessor, std::string s) : accessor(accessor), s(std::move(s)) + { + v.mkPath(&*accessor, this->s.c_str()); + } Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 70228e1e2..78436c6a2 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -64,6 +64,7 @@ struct StringToken { #include "parser-tab.hh" #include "lexer-tab.hh" +#include "fs-input-accessor.hh" YY_DECL; @@ -520,7 +521,7 @@ path_start /* add back in the trailing '/' to the first segment */ if ($1.p[$1.l-1] == '/' && $1.l > 1) path += "/"; - $$ = new ExprPath(std::move(path)); + $$ = new ExprPath(ref(data->state.rootFS), std::move(path)); } | HPATH { if (evalSettings.pureEval) { @@ -530,7 +531,7 @@ path_start ); } Path path(getHome() + std::string($1.p + 1, $1.l - 1)); - $$ = new ExprPath(std::move(path)); + $$ = new ExprPath(ref(data->state.rootFS), std::move(path)); } ; @@ -756,11 +757,11 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_ auto r = *rOpt; Path res = suffix == "" ? r : concatStrings(r, "/", suffix); - if (pathExists(res)) return CanonPath(canonPath(res)); + if (pathExists(res)) return rootPath(CanonPath(canonPath(res))); } if (hasPrefix(path, "nix/")) - return CanonPath(concatStrings(corepkgsPrefix, path.substr(4))); + return rootPath(CanonPath(concatStrings(corepkgsPrefix, path.substr(4)))); debugThrow(ThrownError({ .msg = hintfmt(evalSettings.pureEval diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index b6a696f47..099607638 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -1,10 +1,11 @@ #include "eval.hh" +#include "fs-input-accessor.hh" namespace nix { SourcePath EvalState::rootPath(CanonPath path) { - return path; + return {rootFS, std::move(path)}; } } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3bed7c5aa..dd3fa12ee 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -202,7 +202,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "imported-drv-to-derivation.nix.gen.hh" - , CanonPath::root), **state.vImportedDrvToDerivation); + , state.rootPath(CanonPath::root)), **state.vImportedDrvToDerivation); } state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh"); @@ -213,7 +213,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v else if (path2 == corepkgsPrefix + "fetchurl.nix") { state.eval(state.parseExprFromString( #include "fetchurl.nix.gen.hh" - , CanonPath::root), v); + , state.rootPath(CanonPath::root)), v); } else { @@ -599,7 +599,8 @@ struct CompareValues case nString: return v1->string_view().compare(v2->string_view()) < 0; case nPath: - return strcmp(v1->_path, v2->_path) < 0; + // FIXME: handle accessor? + return strcmp(v1->_path.path, v2->_path.path) < 0; case nList: // Lexicographic comparison for (size_t i = 0;; i++) { @@ -2203,7 +2204,7 @@ static void addPath( path = evalSettings.pureEval && expectedHash ? path - : state.checkSourcePath(CanonPath(path)).path.abs(); + : state.checkSourcePath(state.rootPath(CanonPath(path))).path.abs(); PathFilter filter = filterFun ? ([&](const Path & path) { auto st = lstat(path); @@ -2236,9 +2237,10 @@ static void addPath( }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - StorePath dstPath = settings.readOnlyMode - ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first - : state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs); + // FIXME + if (method != FileIngestionMethod::Recursive) + throw Error("'recursive = false' is not implemented"); + auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, &filter, state.repair); if (expectedHash && expectedStorePath != dstPath) state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); state.allowAndSetStorePathString(dstPath, v); @@ -4447,7 +4449,7 @@ void EvalState::createBaseEnv() // the parser needs two NUL bytes as terminators; one of them // is implied by being a C string. "\0"; - eval(parse(code, sizeof(code), derivationInternal, {CanonPath::root}, staticBaseEnv), *vDerivation); + eval(parse(code, sizeof(code), derivationInternal, rootPath(CanonPath::root), staticBaseEnv), *vDerivation); } diff --git a/src/libexpr/tests/json.cc b/src/libexpr/tests/json.cc index 7586bdd9b..f4cc118d6 100644 --- a/src/libexpr/tests/json.cc +++ b/src/libexpr/tests/json.cc @@ -62,7 +62,7 @@ namespace nix { // not supported by store 'dummy'" thrown in the test body. TEST_F(JSONValueTest, DISABLED_Path) { Value v; - v.mkPath("test"); + v.mkPath(state.rootPath(CanonPath("/test"))); ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\""); } } /* namespace nix */ diff --git a/src/libexpr/tests/libexpr.hh b/src/libexpr/tests/libexpr.hh index 97b05ac46..968431446 100644 --- a/src/libexpr/tests/libexpr.hh +++ b/src/libexpr/tests/libexpr.hh @@ -103,14 +103,17 @@ namespace nix { } MATCHER_P(IsPathEq, p, fmt("Is a path equal to \"%1%\"", p)) { - if (arg.type() != nPath) { - *result_listener << "Expected a path got " << arg.type(); - return false; - } else if (std::string_view(arg._path) != p) { - *result_listener << "Expected a path that equals \"" << p << "\" but got: " << arg.c_str(); + if (arg.type() != nPath) { + *result_listener << "Expected a path got " << arg.type(); + return false; + } else { + auto path = arg.path(); + if (path.path != CanonPath(p)) { + *result_listener << "Expected a path that equals \"" << p << "\" but got: " << path.path; return false; } - return true; + } + return true; } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 7a1165cac..622e613ea 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -189,7 +189,12 @@ public: const char * c_str; const char * * context; // must be in sorted order } string; - const char * _path; + + struct { + InputAccessor * accessor; + const char * path; + } _path; + Bindings * attrs; struct { size_t size; @@ -286,11 +291,12 @@ public: void mkPath(const SourcePath & path); - inline void mkPath(const char * path) + inline void mkPath(InputAccessor * accessor, const char * path) { clearValue(); internalType = tPath; - _path = path; + _path.accessor = accessor; + _path.path = path; } inline void mkNull() @@ -437,7 +443,10 @@ public: SourcePath path() const { assert(internalType == tPath); - return SourcePath{CanonPath(_path)}; + return SourcePath { + .accessor = ref(_path.accessor->shared_from_this()), + .path = CanonPath(CanonPath::unchecked_t(), _path.path) + }; } std::string_view string_view() const diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc new file mode 100644 index 000000000..a35955465 --- /dev/null +++ b/src/libfetchers/fs-input-accessor.cc @@ -0,0 +1,141 @@ +#include "fs-input-accessor.hh" +#include "store-api.hh" + +namespace nix { + +struct FSInputAccessorImpl : FSInputAccessor +{ + CanonPath root; + std::optional> allowedPaths; + MakeNotAllowedError makeNotAllowedError; + + FSInputAccessorImpl( + const CanonPath & root, + std::optional> && allowedPaths, + MakeNotAllowedError && makeNotAllowedError) + : root(root) + , allowedPaths(std::move(allowedPaths)) + , makeNotAllowedError(std::move(makeNotAllowedError)) + { + } + + std::string readFile(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + checkAllowed(absPath); + return nix::readFile(absPath.abs()); + } + + bool pathExists(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + return isAllowed(absPath) && nix::pathExists(absPath.abs()); + } + + Stat lstat(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + checkAllowed(absPath); + auto st = nix::lstat(absPath.abs()); + return Stat { + .type = + S_ISREG(st.st_mode) ? tRegular : + S_ISDIR(st.st_mode) ? tDirectory : + S_ISLNK(st.st_mode) ? tSymlink : + tMisc, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR + }; + } + + DirEntries readDirectory(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + checkAllowed(absPath); + DirEntries res; + for (auto & entry : nix::readDirectory(absPath.abs())) { + std::optional type; + switch (entry.type) { + case DT_REG: type = Type::tRegular; break; + case DT_LNK: type = Type::tSymlink; break; + case DT_DIR: type = Type::tDirectory; break; + } + if (isAllowed(absPath + entry.name)) + res.emplace(entry.name, type); + } + return res; + } + + std::string readLink(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + checkAllowed(absPath); + return nix::readLink(absPath.abs()); + } + + CanonPath makeAbsPath(const CanonPath & path) + { + return root + path; + } + + void checkAllowed(const CanonPath & absPath) override + { + if (!isAllowed(absPath)) + throw makeNotAllowedError + ? makeNotAllowedError(absPath) + : RestrictedPathError("access to path '%s' is forbidden", absPath); + } + + bool isAllowed(const CanonPath & absPath) + { + if (!absPath.isWithin(root)) + return false; + + if (allowedPaths) { + auto p = absPath.removePrefix(root); + if (!p.isAllowed(*allowedPaths)) + return false; + } + + return true; + } + + void allowPath(CanonPath path) override + { + if (allowedPaths) + allowedPaths->insert(std::move(path)); + } + + bool hasAccessControl() override + { + return (bool) allowedPaths; + } + + std::optional getPhysicalPath(const CanonPath & path) override + { + return makeAbsPath(path); + } +}; + +ref makeFSInputAccessor( + const CanonPath & root, + std::optional> && allowedPaths, + MakeNotAllowedError && makeNotAllowedError) +{ + return make_ref(root, std::move(allowedPaths), std::move(makeNotAllowedError)); +} + +ref makeStorePathAccessor( + ref store, + const StorePath & storePath, + MakeNotAllowedError && makeNotAllowedError) +{ + return makeFSInputAccessor(CanonPath(store->toRealPath(storePath)), {}, std::move(makeNotAllowedError)); +} + +SourcePath getUnfilteredRootPath(CanonPath path) +{ + static auto rootFS = makeFSInputAccessor(CanonPath::root); + return {rootFS, path}; +} + +} diff --git a/src/libfetchers/fs-input-accessor.hh b/src/libfetchers/fs-input-accessor.hh new file mode 100644 index 000000000..19a5211c8 --- /dev/null +++ b/src/libfetchers/fs-input-accessor.hh @@ -0,0 +1,33 @@ +#pragma once + +#include "input-accessor.hh" + +namespace nix { + +class StorePath; +class Store; + +struct FSInputAccessor : InputAccessor +{ + virtual void checkAllowed(const CanonPath & absPath) = 0; + + virtual void allowPath(CanonPath path) = 0; + + virtual bool hasAccessControl() = 0; +}; + +typedef std::function MakeNotAllowedError; + +ref makeFSInputAccessor( + const CanonPath & root, + std::optional> && allowedPaths = {}, + MakeNotAllowedError && makeNotAllowedError = {}); + +ref makeStorePathAccessor( + ref store, + const StorePath & storePath, + MakeNotAllowedError && makeNotAllowedError = {}); + +SourcePath getUnfilteredRootPath(CanonPath path); + +} diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index f37a8058b..fec16e99f 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -3,12 +3,149 @@ namespace nix { +static std::atomic nextNumber{0}; + +InputAccessor::InputAccessor() + : number(++nextNumber) +{ +} + +// FIXME: merge with archive.cc. +void InputAccessor::dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter) +{ + auto dumpContents = [&](const CanonPath & path) + { + // FIXME: pipe + auto s = readFile(path); + sink << "contents" << s.size(); + sink(s); + writePadding(s.size(), sink); + }; + + std::function dump; + + dump = [&](const CanonPath & path) { + checkInterrupt(); + + auto st = lstat(path); + + sink << "("; + + if (st.type == tRegular) { + sink << "type" << "regular"; + if (st.isExecutable) + sink << "executable" << ""; + dumpContents(path); + } + + else if (st.type == tDirectory) { + sink << "type" << "directory"; + + /* If we're on a case-insensitive system like macOS, undo + the case hack applied by restorePath(). */ + std::map unhacked; + for (auto & i : readDirectory(path)) + if (/* archiveSettings.useCaseHack */ false) { // FIXME + std::string name(i.first); + size_t pos = i.first.find(caseHackSuffix); + if (pos != std::string::npos) { + debug("removing case hack suffix from '%s'", path + i.first); + name.erase(pos); + } + if (!unhacked.emplace(name, i.first).second) + throw Error("file name collision in between '%s' and '%s'", + (path + unhacked[name]), + (path + i.first)); + } else + unhacked.emplace(i.first, i.first); + + for (auto & i : unhacked) + if (filter((path + i.first).abs())) { + sink << "entry" << "(" << "name" << i.first << "node"; + dump(path + i.second); + sink << ")"; + } + } + + else if (st.type == tSymlink) + sink << "type" << "symlink" << "target" << readLink(path); + + else throw Error("file '%s' has an unsupported type", path); + + sink << ")"; + }; + + sink << narVersionMagic1; + dump(path); +} + +Hash InputAccessor::hashPath( + const CanonPath & path, + PathFilter & filter, + HashType ht) +{ + HashSink sink(ht); + dumpPath(path, sink, filter); + return sink.finish().first; +} + +StorePath InputAccessor::fetchToStore( + ref store, + const CanonPath & path, + std::string_view name, + PathFilter * filter, + RepairFlag repair) +{ + Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path))); + + auto source = sinkToSource([&](Sink & sink) { + dumpPath(path, sink, filter ? *filter : defaultPathFilter); + }); + + auto storePath = + settings.readOnlyMode + ? store->computeStorePathFromDump(*source, name).first + : store->addToStoreFromDump(*source, name, FileIngestionMethod::Recursive, htSHA256, repair); + + return storePath; +} + +std::optional InputAccessor::maybeLstat(const CanonPath & path) +{ + // FIXME: merge these into one operation. + if (!pathExists(path)) + return {}; + return lstat(path); +} + +std::string InputAccessor::showPath(const CanonPath & path) +{ + return path.abs(); +} + +SourcePath InputAccessor::root() +{ + return {ref(shared_from_this()), CanonPath::root}; +} + std::ostream & operator << (std::ostream & str, const SourcePath & path) { str << path.to_string(); return str; } +StorePath SourcePath::fetchToStore( + ref store, + std::string_view name, + PathFilter * filter, + RepairFlag repair) const +{ + return accessor->fetchToStore(store, path, name, filter, repair); +} + std::string_view SourcePath::baseName() const { return path.baseName().value_or("source"); @@ -18,60 +155,12 @@ SourcePath SourcePath::parent() const { auto p = path.parent(); assert(p); - return std::move(*p); -} - -InputAccessor::Stat SourcePath::lstat() const -{ - auto st = nix::lstat(path.abs()); - return InputAccessor::Stat { - .type = - S_ISREG(st.st_mode) ? InputAccessor::tRegular : - S_ISDIR(st.st_mode) ? InputAccessor::tDirectory : - S_ISLNK(st.st_mode) ? InputAccessor::tSymlink : - InputAccessor::tMisc, - .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR - }; -} - -std::optional SourcePath::maybeLstat() const -{ - // FIXME: merge these into one operation. - if (!pathExists()) - return {}; - return lstat(); -} - -InputAccessor::DirEntries SourcePath::readDirectory() const -{ - InputAccessor::DirEntries res; - for (auto & entry : nix::readDirectory(path.abs())) { - std::optional type; - switch (entry.type) { - case DT_REG: type = InputAccessor::Type::tRegular; break; - case DT_LNK: type = InputAccessor::Type::tSymlink; break; - case DT_DIR: type = InputAccessor::Type::tDirectory; break; - } - res.emplace(entry.name, type); - } - return res; -} - -StorePath SourcePath::fetchToStore( - ref store, - std::string_view name, - PathFilter * filter, - RepairFlag repair) const -{ - return - settings.readOnlyMode - ? store->computeStorePathForPath(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter).first - : store->addToStore(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter, repair); + return {accessor, std::move(*p)}; } SourcePath SourcePath::resolveSymlinks() const { - SourcePath res(CanonPath::root); + auto res = accessor->root(); int linksAllowed = 1024; diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 5a2f17f62..a0f08e295 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -5,14 +5,29 @@ #include "archive.hh" #include "canon-path.hh" #include "repair-flag.hh" +#include "hash.hh" namespace nix { +MakeError(RestrictedPathError, Error); + +struct SourcePath; class StorePath; class Store; -struct InputAccessor +struct InputAccessor : public std::enable_shared_from_this { + const size_t number; + + InputAccessor(); + + virtual ~InputAccessor() + { } + + virtual std::string readFile(const CanonPath & path) = 0; + + virtual bool pathExists(const CanonPath & path) = 0; + enum Type { tRegular, tSymlink, tDirectory, /** @@ -32,9 +47,63 @@ struct InputAccessor bool isExecutable = false; // regular files only }; + virtual Stat lstat(const CanonPath & path) = 0; + + std::optional maybeLstat(const CanonPath & path); + typedef std::optional DirEntry; typedef std::map DirEntries; + + virtual DirEntries readDirectory(const CanonPath & path) = 0; + + virtual std::string readLink(const CanonPath & path) = 0; + + virtual void dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter = defaultPathFilter); + + Hash hashPath( + const CanonPath & path, + PathFilter & filter = defaultPathFilter, + HashType ht = htSHA256); + + StorePath fetchToStore( + ref store, + const CanonPath & path, + std::string_view name = "source", + PathFilter * filter = nullptr, + RepairFlag repair = NoRepair); + + /* Return a corresponding path in the root filesystem, if + possible. This is only possible for inputs that are + materialized in the root filesystem. */ + virtual std::optional getPhysicalPath(const CanonPath & path) + { return std::nullopt; } + + bool operator == (const InputAccessor & x) const + { + return number == x.number; + } + + bool operator < (const InputAccessor & x) const + { + return number < x.number; + } + + void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); + + virtual std::string showPath(const CanonPath & path); + + SourcePath root(); + + /* Return the maximum last-modified time of the files in this + tree, if available. */ + virtual std::optional getLastModified() + { + return std::nullopt; + } }; /** @@ -45,12 +114,9 @@ struct InputAccessor */ struct SourcePath { + ref accessor; CanonPath path; - SourcePath(CanonPath path) - : path(std::move(path)) - { } - std::string_view baseName() const; /** @@ -64,39 +130,42 @@ struct SourcePath * return its contents; otherwise throw an error. */ std::string readFile() const - { return nix::readFile(path.abs()); } + { return accessor->readFile(path); } /** * Return whether this `SourcePath` denotes a file (of any type) * that exists */ bool pathExists() const - { return nix::pathExists(path.abs()); } + { return accessor->pathExists(path); } /** * Return stats about this `SourcePath`, or throw an exception if * it doesn't exist. */ - InputAccessor::Stat lstat() const; + InputAccessor::Stat lstat() const + { return accessor->lstat(path); } /** * Return stats about this `SourcePath`, or std::nullopt if it * doesn't exist. */ - std::optional maybeLstat() const; + std::optional maybeLstat() const + { return accessor->maybeLstat(path); } /** * If this `SourcePath` denotes a directory (not a symlink), * return its directory entries; otherwise throw an error. */ - InputAccessor::DirEntries readDirectory() const; + InputAccessor::DirEntries readDirectory() const + { return accessor->readDirectory(path); } /** * If this `SourcePath` denotes a symlink, return its target; * otherwise throw an error. */ std::string readLink() const - { return nix::readLink(path.abs()); } + { return accessor->readLink(path); } /** * Dump this `SourcePath` to `sink` as a NAR archive. @@ -104,7 +173,7 @@ struct SourcePath void dumpPath( Sink & sink, PathFilter & filter = defaultPathFilter) const - { return nix::dumpPath(path.abs(), sink, filter); } + { return accessor->dumpPath(path, sink, filter); } /** * Copy this `SourcePath` to the Nix store. @@ -120,7 +189,7 @@ struct SourcePath * it has a physical location. */ std::optional getPhysicalPath() const - { return path; } + { return accessor->getPhysicalPath(path); } std::string to_string() const { return path.abs(); } @@ -129,7 +198,7 @@ struct SourcePath * Append a `CanonPath` to this path. */ SourcePath operator + (const CanonPath & x) const - { return {path + x}; } + { return {accessor, path + x}; } /** * Append a single component `c` to this path. `c` must not @@ -137,21 +206,21 @@ struct SourcePath * and `c`. */ SourcePath operator + (std::string_view c) const - { return {path + c}; } + { return {accessor, path + c}; } bool operator == (const SourcePath & x) const { - return path == x.path; + return std::tie(accessor, path) == std::tie(x.accessor, x.path); } bool operator != (const SourcePath & x) const { - return path != x.path; + return std::tie(accessor, path) != std::tie(x.accessor, x.path); } bool operator < (const SourcePath & x) const { - return path < x.path; + return std::tie(accessor, path) < std::tie(x.accessor, x.path); } /** diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 28689e100..e761c0e96 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -225,12 +225,16 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA } -std::pair Store::computeStorePathForPath(std::string_view name, - const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const +std::pair Store::computeStorePathFromDump( + Source & dump, + std::string_view name, + FileIngestionMethod method, + HashType hashAlgo, + const StorePathSet & references) const { - Hash h = method == FileIngestionMethod::Recursive - ? hashPath(hashAlgo, srcPath, filter).first - : hashFile(hashAlgo, srcPath); + HashSink sink(hashAlgo); + dump.drainInto(sink); + auto h = sink.finish().first; FixedOutputInfo caInfo { .method = method, .hash = h, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 4e233bac3..bd69999f8 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -292,14 +292,15 @@ public: StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; /** - * Preparatory part of addToStore(). - * - * @return the store path to which srcPath is to be copied - * and the cryptographic hash of the contents of srcPath. + * Read-only variant of addToStoreFromDump(). It returns the store + * path to which a NAR or flat file would be written. */ - std::pair computeStorePathForPath(std::string_view name, - const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, - HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; + std::pair computeStorePathFromDump( + Source & dump, + std::string_view name, + FileIngestionMethod method = FileIngestionMethod::Recursive, + HashType hashAlgo = htSHA256, + const StorePathSet & references = {}) const; /** * Preparatory part of addTextToStore(). diff --git a/src/nix/main.cc b/src/nix/main.cc index 879baa5f5..6cd7ace51 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -204,30 +204,30 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) auto vGenerateManpage = state.allocValue(); state.eval(state.parseExprFromString( #include "generate-manpage.nix.gen.hh" - , CanonPath::root), *vGenerateManpage); + , state.rootPath(CanonPath::root)), *vGenerateManpage); auto vUtils = state.allocValue(); state.cacheFile( - CanonPath("/utils.nix"), CanonPath("/utils.nix"), + state.rootPath(CanonPath("/utils.nix")), state.rootPath(CanonPath("/utils.nix")), state.parseExprFromString( #include "utils.nix.gen.hh" - , CanonPath::root), + , state.rootPath(CanonPath::root)), *vUtils); auto vSettingsInfo = state.allocValue(); state.cacheFile( - CanonPath("/generate-settings.nix"), CanonPath("/generate-settings.nix"), + state.rootPath(CanonPath("/generate-settings.nix")), state.rootPath(CanonPath("/generate-settings.nix")), state.parseExprFromString( #include "generate-settings.nix.gen.hh" - , CanonPath::root), + , state.rootPath(CanonPath::root)), *vSettingsInfo); auto vStoreInfo = state.allocValue(); state.cacheFile( - CanonPath("/generate-store-info.nix"), CanonPath("/generate-store-info.nix"), + state.rootPath(CanonPath("/generate-store-info.nix")), state.rootPath(CanonPath("/generate-store-info.nix")), state.parseExprFromString( #include "generate-store-info.nix.gen.hh" - , CanonPath::root), + , state.rootPath(CanonPath::root)), *vStoreInfo); auto vDump = state.allocValue(); From df73c6eb8cb14fe11f2d5b27e2e28135c7c5dbaf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Oct 2023 17:34:58 +0200 Subject: [PATCH 404/909] Introduce MemoryInputAccessor and use it for corepkgs MemoryInputAccessor is an in-memory virtual filesystem that returns files like . This removes the need for special hacks to handle those files. --- doc/manual/generate-manpage.nix | 2 +- doc/manual/local.mk | 2 +- src/libexpr/eval.cc | 77 ++++++++++++++++-------- src/libexpr/eval.hh | 32 +++++----- src/libexpr/flake/flake.cc | 10 +-- src/libexpr/parser.y | 5 +- src/libexpr/primops.cc | 29 +++------ src/libexpr/tests/error_traces.cc | 14 ++--- src/libfetchers/memory-input-accessor.cc | 54 +++++++++++++++++ src/libfetchers/memory-input-accessor.hh | 15 +++++ src/nix/main.cc | 34 ++++------- 11 files changed, 176 insertions(+), 98 deletions(-) create mode 100644 src/libfetchers/memory-input-accessor.cc create mode 100644 src/libfetchers/memory-input-accessor.hh diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index d81eac182..14136016d 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -2,7 +2,7 @@ let inherit (builtins) attrNames attrValues fromJSON listToAttrs mapAttrs groupBy concatStringsSep concatMap length lessThan replaceStrings sort; - inherit (import ./utils.nix) attrsToList concatStrings optionalString filterAttrs trim squash unique; + inherit (import ) attrsToList concatStrings optionalString filterAttrs trim squash unique; showStoreDocs = import ./generate-store-info.nix; in diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 702fb1ff8..2e9f08678 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -32,7 +32,7 @@ dummy-env = env -i \ NIX_STATE_DIR=/dummy \ NIX_CONFIG='cores = 0' -nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw +nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw # re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution define process-includes diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 87791fa52..a28bb2101 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -13,6 +13,7 @@ #include "profiles.hh" #include "print.hh" #include "fs-input-accessor.hh" +#include "memory-input-accessor.hh" #include #include @@ -516,7 +517,16 @@ EvalState::EvalState( : "in restricted mode"; throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); })) - , derivationInternal(rootPath(CanonPath("/builtin/derivation.nix"))) + , corepkgsFS(makeMemoryInputAccessor()) + , internalFS(makeMemoryInputAccessor()) + , derivationInternal{corepkgsFS->addFile( + CanonPath("derivation-internal.nix"), + #include "primops/derivation.nix.gen.hh" + )} + , callFlakeInternal{internalFS->addFile( + CanonPath("call-flake.nix"), + #include "flake/call-flake.nix.gen.hh" + )} , store(store) , buildStore(buildStore ? buildStore : store) , debugRepl(nullptr) @@ -531,7 +541,8 @@ EvalState::EvalState( , baseEnv(allocEnv(128)) , staticBaseEnv{std::make_shared(false, nullptr)} { - rootFS->allowPath(CanonPath::root); // FIXME + // For now, don't rely on FSInputAccessor for access control. + rootFS->allowPath(CanonPath::root); countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; @@ -570,6 +581,11 @@ EvalState::EvalState( } } + corepkgsFS->addFile( + CanonPath("fetchurl.nix"), + #include "fetchurl.nix.gen.hh" + ); + createBaseEnv(); } @@ -600,6 +616,7 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & SourcePath EvalState::checkSourcePath(const SourcePath & path_) { + if (path_.accessor != rootFS) return path_; if (!allowedPaths) return path_; auto i = resolvedPaths.find(path_.path.abs()); @@ -614,8 +631,6 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_) */ Path abspath = canonPath(path_.path.abs()); - if (hasPrefix(abspath, corepkgsPrefix)) return rootPath(CanonPath(abspath)); - for (auto & i : *allowedPaths) { if (isDirOrInDir(abspath, i)) { found = true; @@ -1180,24 +1195,6 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial if (!e) e = parseExprFromFile(checkSourcePath(resolvedPath)); - cacheFile(path, resolvedPath, e, v, mustBeTrivial); -} - - -void EvalState::resetFileCache() -{ - fileEvalCache.clear(); - fileParseCache.clear(); -} - - -void EvalState::cacheFile( - const SourcePath & path, - const SourcePath & resolvedPath, - Expr * e, - Value & v, - bool mustBeTrivial) -{ fileParseCache[resolvedPath] = e; try { @@ -1226,6 +1223,13 @@ void EvalState::cacheFile( } +void EvalState::resetFileCache() +{ + fileEvalCache.clear(); + fileParseCache.clear(); +} + + void EvalState::eval(Expr * e, Value & v) { e->eval(*this, baseEnv, v); @@ -2341,10 +2345,31 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx) { - auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); - if (path == "" || path[0] != '/') - error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow(); - return rootPath(CanonPath(path)); + try { + forceValue(v, pos); + + if (v.type() == nString) { + copyContext(v, context); + auto s = v.string_view(); + if (!hasPrefix(s, "/")) + error("string '%s' doesn't represent an absolute path", s).atPos(pos).debugThrow(); + return rootPath(CanonPath(s)); + } + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } + + if (v.type() == nPath) + return v.path(); + + if (v.type() == nAttrs) { + auto i = v.attrs->find(sOutPath); + if (i != v.attrs->end()) + return coerceToPath(pos, *i->value, context, errorCtx); + } + + error("cannot coerce %1% to a path", showType(v)).withTrace(pos, errorCtx).debugThrow(); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 4bb0a6a0b..048dff42b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -25,6 +25,7 @@ class StorePath; struct SingleDerivedPath; enum RepairFlag : bool; struct FSInputAccessor; +struct MemoryInputAccessor; /** @@ -212,10 +213,26 @@ public: Bindings emptyBindings; + /** + * The accessor for the root filesystem. + */ const ref rootFS; + /** + * The in-memory filesystem for paths. + */ + const ref corepkgsFS; + + /** + * In-memory filesystem for internal, non-user-callable Nix + * expressions like call-flake.nix. + */ + const ref internalFS; + const SourcePath derivationInternal; + const SourcePath callFlakeInternal; + /** * Store used to materialise .drv files. */ @@ -226,7 +243,6 @@ public: */ const ref buildStore; - RootValue vCallFlake = nullptr; RootValue vImportedDrvToDerivation = nullptr; /** @@ -408,16 +424,6 @@ public: */ void evalFile(const SourcePath & path, Value & v, bool mustBeTrivial = false); - /** - * Like `evalFile`, but with an already parsed expression. - */ - void cacheFile( - const SourcePath & path, - const SourcePath & resolvedPath, - Expr * e, - Value & v, - bool mustBeTrivial = false); - void resetFileCache(); /** @@ -427,7 +433,7 @@ public: SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); /** - * Try to resolve a search path value (not the optinal key part) + * Try to resolve a search path value (not the optional key part) * * If the specified search path element is a URI, download it. * @@ -832,8 +838,6 @@ struct InvalidPathError : EvalError #endif }; -static const std::string corepkgsPrefix{"/__corepkgs__/"}; - template void ErrorBuilder::debugThrow() { diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 2f91ad433..acc84ea4f 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -737,14 +737,10 @@ void callFlake(EvalState & state, vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir); - if (!state.vCallFlake) { - state.vCallFlake = allocRootValue(state.allocValue()); - state.eval(state.parseExprFromString( - #include "call-flake.nix.gen.hh" - , state.rootPath(CanonPath::root)), **state.vCallFlake); - } + auto vCallFlake = state.allocValue(); + state.evalFile(state.callFlakeInternal, *vCallFlake); - state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos); + state.callFunction(*vCallFlake, *vLocks, *vTmp1, noPos); state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos); state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos); } diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 78436c6a2..19812cc49 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -64,7 +64,6 @@ struct StringToken { #include "parser-tab.hh" #include "lexer-tab.hh" -#include "fs-input-accessor.hh" YY_DECL; @@ -650,6 +649,8 @@ formal #include "fetchers.hh" #include "store-api.hh" #include "flake/flake.hh" +#include "fs-input-accessor.hh" +#include "memory-input-accessor.hh" namespace nix { @@ -761,7 +762,7 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_ } if (hasPrefix(path, "nix/")) - return rootPath(CanonPath(concatStrings(corepkgsPrefix, path.substr(4)))); + return {corepkgsFS, CanonPath(path.substr(3))}; debugThrow(ThrownError({ .msg = hintfmt(evalSettings.pureEval diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index dd3fa12ee..f1c24a75c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -121,13 +121,15 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, co auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path"); try { - StringMap rewrites = state.realiseContext(context); - - auto realPath = state.rootPath(CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context))); + if (!context.empty()) { + auto rewrites = state.realiseContext(context); + auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context); + return {path.accessor, CanonPath(realPath)}; + } return flags.checkForPureEval - ? state.checkSourcePath(realPath) - : realPath; + ? state.checkSourcePath(path) + : path; } catch (Error & e) { e.addTrace(state.positions[pos], "while realising the context of path '%s'", path); throw; @@ -210,12 +212,6 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh"); } - else if (path2 == corepkgsPrefix + "fetchurl.nix") { - state.eval(state.parseExprFromString( - #include "fetchurl.nix.gen.hh" - , state.rootPath(CanonPath::root)), v); - } - else { if (!vScope) state.evalFile(path, v); @@ -1486,7 +1482,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, })); NixStringContext context; - auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")).path; + auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'")).path; /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so e.g. nix-push does the right thing. */ @@ -2257,7 +2253,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg { NixStringContext context; auto path = state.coerceToPath(pos, *args[1], context, - "while evaluating the second argument (the path to filter) passed to builtins.filterSource"); + "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); } @@ -4444,12 +4440,7 @@ void EvalState::createBaseEnv() /* Note: we have to initialize the 'derivation' constant *after* building baseEnv/staticBaseEnv because it uses 'builtins'. */ - char code[] = - #include "primops/derivation.nix.gen.hh" - // the parser needs two NUL bytes as terminators; one of them - // is implied by being a C string. - "\0"; - eval(parse(code, sizeof(code), derivationInternal, rootPath(CanonPath::root), staticBaseEnv), *vDerivation); + evalFile(derivationInternal, *vDerivation); } diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc index 285651256..6da8a5954 100644 --- a/src/libexpr/tests/error_traces.cc +++ b/src/libexpr/tests/error_traces.cc @@ -295,7 +295,7 @@ namespace nix { TEST_F(ErrorTraceTest, toPath) { ASSERT_TRACE2("toPath []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("cannot coerce %s to a path", "a list"), hintfmt("while evaluating the first argument passed to builtins.toPath")); ASSERT_TRACE2("toPath \"foo\"", @@ -309,8 +309,8 @@ namespace nix { TEST_F(ErrorTraceTest, storePath) { ASSERT_TRACE2("storePath true", TypeError, - hintfmt("cannot coerce %s to a string", "a Boolean"), - hintfmt("while evaluating the first argument passed to builtins.storePath")); + hintfmt("cannot coerce %s to a path", "a Boolean"), + hintfmt("while evaluating the first argument passed to 'builtins.storePath'")); } @@ -318,7 +318,7 @@ namespace nix { TEST_F(ErrorTraceTest, pathExists) { ASSERT_TRACE2("pathExists []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("cannot coerce %s to a path", "a list"), hintfmt("while realising the context of a path")); ASSERT_TRACE2("pathExists \"zorglub\"", @@ -377,13 +377,13 @@ namespace nix { TEST_F(ErrorTraceTest, filterSource) { ASSERT_TRACE2("filterSource [] []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), - hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource")); + hintfmt("cannot coerce %s to a path", "a list"), + hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] \"foo\"", EvalError, hintfmt("string '%s' doesn't represent an absolute path", "foo"), - hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource")); + hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] ./.", TypeError, diff --git a/src/libfetchers/memory-input-accessor.cc b/src/libfetchers/memory-input-accessor.cc new file mode 100644 index 000000000..817d063ba --- /dev/null +++ b/src/libfetchers/memory-input-accessor.cc @@ -0,0 +1,54 @@ +#include "memory-input-accessor.hh" + +namespace nix { + +struct MemoryInputAccessorImpl : MemoryInputAccessor +{ + std::map files; + + std::string readFile(const CanonPath & path) override + { + auto i = files.find(path); + if (i == files.end()) + throw Error("file '%s' does not exist", path); + return i->second; + } + + bool pathExists(const CanonPath & path) override + { + auto i = files.find(path); + return i != files.end(); + } + + Stat lstat(const CanonPath & path) override + { + auto i = files.find(path); + if (i != files.end()) + return Stat { .type = tRegular, .isExecutable = false }; + throw Error("file '%s' does not exist", path); + } + + DirEntries readDirectory(const CanonPath & path) override + { + return {}; + } + + std::string readLink(const CanonPath & path) override + { + throw UnimplementedError("MemoryInputAccessor::readLink"); + } + + SourcePath addFile(CanonPath path, std::string && contents) override + { + files.emplace(path, std::move(contents)); + + return {ref(shared_from_this()), std::move(path)}; + } +}; + +ref makeMemoryInputAccessor() +{ + return make_ref(); +} + +} diff --git a/src/libfetchers/memory-input-accessor.hh b/src/libfetchers/memory-input-accessor.hh new file mode 100644 index 000000000..b75b02bfd --- /dev/null +++ b/src/libfetchers/memory-input-accessor.hh @@ -0,0 +1,15 @@ +#include "input-accessor.hh" + +namespace nix { + +/** + * An input accessor for an in-memory file system. + */ +struct MemoryInputAccessor : InputAccessor +{ + virtual SourcePath addFile(CanonPath path, std::string && contents) = 0; +}; + +ref makeMemoryInputAccessor(); + +} diff --git a/src/nix/main.cc b/src/nix/main.cc index 6cd7ace51..ee3878cc7 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -12,6 +12,7 @@ #include "finally.hh" #include "loggers.hh" #include "markdown.hh" +#include "memory-input-accessor.hh" #include #include @@ -206,29 +207,20 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) #include "generate-manpage.nix.gen.hh" , state.rootPath(CanonPath::root)), *vGenerateManpage); - auto vUtils = state.allocValue(); - state.cacheFile( - state.rootPath(CanonPath("/utils.nix")), state.rootPath(CanonPath("/utils.nix")), - state.parseExprFromString( - #include "utils.nix.gen.hh" - , state.rootPath(CanonPath::root)), - *vUtils); + state.corepkgsFS->addFile( + CanonPath("utils.nix"), + #include "utils.nix.gen.hh" + ); - auto vSettingsInfo = state.allocValue(); - state.cacheFile( - state.rootPath(CanonPath("/generate-settings.nix")), state.rootPath(CanonPath("/generate-settings.nix")), - state.parseExprFromString( - #include "generate-settings.nix.gen.hh" - , state.rootPath(CanonPath::root)), - *vSettingsInfo); + state.corepkgsFS->addFile( + CanonPath("/generate-settings.nix"), + #include "generate-settings.nix.gen.hh" + ); - auto vStoreInfo = state.allocValue(); - state.cacheFile( - state.rootPath(CanonPath("/generate-store-info.nix")), state.rootPath(CanonPath("/generate-store-info.nix")), - state.parseExprFromString( - #include "generate-store-info.nix.gen.hh" - , state.rootPath(CanonPath::root)), - *vStoreInfo); + state.corepkgsFS->addFile( + CanonPath("/generate-store-info.nix"), + #include "generate-store-info.nix.gen.hh" + ); auto vDump = state.allocValue(); vDump->mkString(toplevel.dumpCli()); From e9ddf0b4008262542b422c76f60c3fb80982cd74 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 9 Oct 2023 10:53:47 +0800 Subject: [PATCH 405/909] Simplify parseHashTypeOpt Remove redundant "else" after "return". Use std::nullopt to increase readability. --- src/libutil/hash.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 2c36d9d94..6d56c1989 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -389,10 +389,10 @@ Hash compressHash(const Hash & hash, unsigned int newSize) std::optional parseHashTypeOpt(std::string_view s) { if (s == "md5") return htMD5; - else if (s == "sha1") return htSHA1; - else if (s == "sha256") return htSHA256; - else if (s == "sha512") return htSHA512; - else return std::optional {}; + if (s == "sha1") return htSHA1; + if (s == "sha256") return htSHA256; + if (s == "sha512") return htSHA512; + return std::nullopt; } HashType parseHashType(std::string_view s) From aff177d8609c392b4edf59819a086440cca101a7 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 9 Oct 2023 11:00:14 +0800 Subject: [PATCH 406/909] Elaborate the "unknown hash algorithm" error List the allowed hash formats --- src/libexpr/tests/error_traces.cc | 2 +- src/libutil/hash.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc index 285651256..1af601ec8 100644 --- a/src/libexpr/tests/error_traces.cc +++ b/src/libexpr/tests/error_traces.cc @@ -1084,7 +1084,7 @@ namespace nix { ASSERT_TRACE1("hashString \"foo\" \"content\"", UsageError, - hintfmt("unknown hash algorithm '%s'", "foo")); + hintfmt("unknown hash algorithm '%s', expect 'md5', 'sha1', 'sha256', or 'sha512'", "foo")); ASSERT_TRACE2("hashString \"sha256\" {}", TypeError, diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 6d56c1989..251440363 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -401,7 +401,7 @@ HashType parseHashType(std::string_view s) if (opt_h) return *opt_h; else - throw UsageError("unknown hash algorithm '%1%'", s); + throw UsageError("unknown hash algorithm '%1%', expect 'md5', 'sha1', 'sha256', or 'sha512'", s); } std::string_view printHashType(HashType ht) From 838c70f62116328ce01cb41a01886e4f1b9a727f Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Tue, 10 Oct 2023 18:53:01 +0800 Subject: [PATCH 407/909] treewide: Rename hashBase to hashFormat hashBase is ambiguous, since it's not about the digital bases, but about the format of hashes. Base16, Base32 and Base64 are all character maps for binary encoding. Rename the enum Base to HashFormat. Rename variables of type HashFormat from [hash]Base to hashFormat, including CmdHashBase::hashFormat and CmdToBase::hashFormat. --- src/libstore/store-api.cc | 6 ++--- src/libstore/store-api.hh | 2 +- src/libutil/hash.cc | 8 +++---- src/libutil/hash.hh | 4 ++-- src/libutil/tests/hash.cc | 16 +++++++------- src/nix/hash.cc | 46 +++++++++++++++++++-------------------- 6 files changed, 41 insertions(+), 41 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 28689e100..fa23f976e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -938,7 +938,7 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor json Store::pathInfoToJSON(const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, - Base hashBase, + HashFormat hashFormat, AllowInvalidFlag allowInvalid) { json::array_t jsonList = json::array(); @@ -951,7 +951,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths, jsonPath["path"] = printStorePath(info->path); jsonPath["valid"] = true; - jsonPath["narHash"] = info->narHash.to_string(hashBase, true); + jsonPath["narHash"] = info->narHash.to_string(hashFormat, true); jsonPath["narSize"] = info->narSize; { @@ -993,7 +993,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths, if (!narInfo->url.empty()) jsonPath["url"] = narInfo->url; if (narInfo->fileHash) - jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashBase, true); + jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashFormat, true); if (narInfo->fileSize) jsonPath["downloadSize"] = narInfo->fileSize; if (showClosureSize) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 4e233bac3..f30ccc950 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -676,7 +676,7 @@ public: */ nlohmann::json pathInfoToJSON(const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, - Base hashBase = Base32, + HashFormat hashFormat = Base32, AllowInvalidFlag allowInvalid = DisallowInvalid); /** diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 251440363..401a65ec3 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -115,14 +115,14 @@ std::string printHash16or32(const Hash & hash) } -std::string Hash::to_string(Base base, bool includeType) const +std::string Hash::to_string(HashFormat hashFormat, bool includeType) const { std::string s; - if (base == SRI || includeType) { + if (hashFormat == SRI || includeType) { s += printHashType(type); - s += base == SRI ? '-' : ':'; + s += hashFormat == SRI ? '-' : ':'; } - switch (base) { + switch (hashFormat) { case Base16: s += printHash16(*this); break; diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index c3aa5cd81..73c2183ef 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -23,7 +23,7 @@ extern std::set hashTypes; extern const std::string base32Chars; -enum Base : int { Base64, Base32, Base16, SRI }; +enum HashFormat : int { Base64, Base32, Base16, SRI }; struct Hash @@ -114,7 +114,7 @@ public: * or base-64. By default, this is prefixed by the hash type * (e.g. "sha256:"). */ - std::string to_string(Base base, bool includeType) const; + std::string to_string(HashFormat hashFormat, bool includeType) const; std::string gitRev() const { diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc index e4e928b3b..d14e5b26d 100644 --- a/src/libutil/tests/hash.cc +++ b/src/libutil/tests/hash.cc @@ -18,28 +18,28 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc1321 auto s1 = ""; auto hash = hashString(HashType::htMD5, s1); - ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); } TEST(hashString, testKnownMD5Hashes2) { // values taken from: https://tools.ietf.org/html/rfc1321 auto s2 = "abc"; auto hash = hashString(HashType::htMD5, s2); - ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); } TEST(hashString, testKnownSHA1Hashes1) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abc"; auto hash = hashString(HashType::htSHA1, s); - ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); } TEST(hashString, testKnownSHA1Hashes2) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; auto hash = hashString(HashType::htSHA1, s); - ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); } TEST(hashString, testKnownSHA256Hashes1) { @@ -47,7 +47,7 @@ namespace nix { auto s = "abc"; auto hash = hashString(HashType::htSHA256, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); } @@ -55,7 +55,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; auto hash = hashString(HashType::htSHA256, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); } @@ -63,7 +63,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abc"; auto hash = hashString(HashType::htSHA512, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9" "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" "454d4423643ce80e2a9ac94fa54ca49f"); @@ -74,7 +74,7 @@ namespace nix { auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; auto hash = hashString(HashType::htSHA512, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1" "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" "c7d329eeb6dd26545e96e55b874be909"); diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 9feca9345..6af6bd19f 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -11,7 +11,7 @@ using namespace nix; struct CmdHashBase : Command { FileIngestionMethod mode; - Base base = SRI; + HashFormat hashFormat = SRI; bool truncate = false; HashType ht = htSHA256; std::vector paths; @@ -22,25 +22,25 @@ struct CmdHashBase : Command addFlag({ .longName = "sri", .description = "Print the hash in SRI format.", - .handler = {&base, SRI}, + .handler = {&hashFormat, SRI}, }); addFlag({ .longName = "base64", .description = "Print the hash in base-64 format.", - .handler = {&base, Base64}, + .handler = {&hashFormat, Base64}, }); addFlag({ .longName = "base32", .description = "Print the hash in base-32 (Nix-specific) format.", - .handler = {&base, Base32}, + .handler = {&hashFormat, Base32}, }); addFlag({ .longName = "base16", .description = "Print the hash in base-16 format.", - .handler = {&base, Base16}, + .handler = {&hashFormat, Base16}, }); addFlag(Flag::mkHashTypeFlag("type", &ht)); @@ -94,18 +94,18 @@ struct CmdHashBase : Command Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); - logger->cout(h.to_string(base, base == SRI)); + logger->cout(h.to_string(hashFormat, hashFormat == SRI)); } } }; struct CmdToBase : Command { - Base base; + HashFormat hashFormat; std::optional ht; std::vector args; - CmdToBase(Base base) : base(base) + CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat) { addFlag(Flag::mkHashTypeOptFlag("type", &ht)); expectArgs("strings", &args); @@ -114,16 +114,16 @@ struct CmdToBase : Command std::string description() override { return fmt("convert a hash to %s representation", - base == Base16 ? "base-16" : - base == Base32 ? "base-32" : - base == Base64 ? "base-64" : + hashFormat == Base16 ? "base-16" : + hashFormat == Base32 ? "base-32" : + hashFormat == Base64 ? "base-64" : "SRI"); } void run() override { for (auto s : args) - logger->cout(Hash::parseAny(s, ht).to_string(base, base == SRI)); + logger->cout(Hash::parseAny(s, ht).to_string(hashFormat, hashFormat == SRI)); } }; @@ -162,7 +162,7 @@ static int compatNixHash(int argc, char * * argv) { std::optional ht; bool flat = false; - Base base = Base16; + HashFormat hashFormat = Base16; bool truncate = false; enum { opHash, opTo } op = opHash; std::vector ss; @@ -173,10 +173,10 @@ static int compatNixHash(int argc, char * * argv) else if (*arg == "--version") printVersion("nix-hash"); else if (*arg == "--flat") flat = true; - else if (*arg == "--base16") base = Base16; - else if (*arg == "--base32") base = Base32; - else if (*arg == "--base64") base = Base64; - else if (*arg == "--sri") base = SRI; + else if (*arg == "--base16") hashFormat = Base16; + else if (*arg == "--base32") hashFormat = Base32; + else if (*arg == "--base64") hashFormat = Base64; + else if (*arg == "--sri") hashFormat = SRI; else if (*arg == "--truncate") truncate = true; else if (*arg == "--type") { std::string s = getArg(*arg, arg, end); @@ -184,19 +184,19 @@ static int compatNixHash(int argc, char * * argv) } else if (*arg == "--to-base16") { op = opTo; - base = Base16; + hashFormat = Base16; } else if (*arg == "--to-base32") { op = opTo; - base = Base32; + hashFormat = Base32; } else if (*arg == "--to-base64") { op = opTo; - base = Base64; + hashFormat = Base64; } else if (*arg == "--to-sri") { op = opTo; - base = SRI; + hashFormat = SRI; } else if (*arg != "" && arg->at(0) == '-') return false; @@ -209,14 +209,14 @@ static int compatNixHash(int argc, char * * argv) CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); if (!ht.has_value()) ht = htMD5; cmd.ht = ht.value(); - cmd.base = base; + cmd.hashFormat = hashFormat; cmd.truncate = truncate; cmd.paths = ss; cmd.run(); } else { - CmdToBase cmd(base); + CmdToBase cmd(hashFormat); cmd.args = ss; if (ht.has_value()) cmd.ht = ht; cmd.run(); From 5043e6cf4ea537dfe599470797c5b310ab0e94b9 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Wed, 18 Oct 2023 17:08:06 +0800 Subject: [PATCH 408/909] Document HashFormat --- src/libutil/hash.hh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 73c2183ef..0db9bcd6d 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -23,7 +23,21 @@ extern std::set hashTypes; extern const std::string base32Chars; -enum HashFormat : int { Base64, Base32, Base16, SRI }; +/** + * @brief Enumeration representing the hash formats. + */ +enum HashFormat : int { + /// @brief Base 64 encoding. + /// @see [IETF RFC 4648, section 4](https://datatracker.ietf.org/doc/html/rfc4648#section-4). + Base64, + /// @brief Nix-specific base-32 encoding. @see base32Chars + Base32, + /// @brief Lowercase hexadecimal encoding. @see base16Chars + Base16, + /// @brief ":", format of the SRI integrity attribute. + /// @see W3C recommendation [Subresource Intergrity](https://www.w3.org/TR/SRI/). + SRI +}; struct Hash From e026f3e1ae533ab476e58258c7d84814189df9ff Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 13 Oct 2023 09:48:15 +0800 Subject: [PATCH 409/909] treewide: Reference HashFormat members with scope Base* -> HashFormat::Base* --- perl/lib/Nix/Store.xs | 12 +++--- src/libexpr/eval-cache.cc | 2 +- src/libexpr/primops.cc | 4 +- src/libexpr/primops/fetchTree.cc | 4 +- src/libfetchers/fetchers.cc | 4 +- src/libfetchers/git.cc | 4 +- src/libfetchers/github.cc | 8 ++-- src/libfetchers/mercurial.cc | 4 +- src/libfetchers/tarball.cc | 2 +- src/libstore/binary-cache-store.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 6 +-- src/libstore/builtins/fetchurl.cc | 2 +- src/libstore/content-address.cc | 2 +- src/libstore/daemon.cc | 2 +- src/libstore/derivations.cc | 12 +++--- src/libstore/downstream-placeholder.cc | 4 +- src/libstore/export-import.cc | 2 +- src/libstore/gc.cc | 2 +- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-store.cc | 14 +++---- src/libstore/nar-info-disk-cache.cc | 4 +- src/libstore/nar-info.cc | 4 +- src/libstore/optimise-store.cc | 4 +- src/libstore/path-info.cc | 4 +- src/libstore/path.cc | 2 +- src/libstore/realisation.hh | 2 +- src/libstore/remote-store.cc | 2 +- src/libstore/store-api.cc | 6 +-- src/libstore/store-api.hh | 2 +- src/libutil/hash.cc | 16 +++---- src/libutil/hash.hh | 4 +- src/nix-store/nix-store.cc | 8 ++-- src/nix/flake.cc | 8 ++-- src/nix/hash.cc | 46 ++++++++++----------- src/nix/path-info.cc | 2 +- src/nix/prefetch.cc | 4 +- src/nix/verify.cc | 4 +- 37 files changed, 108 insertions(+), 108 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index e96885e4c..08f812b31 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -78,7 +78,7 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true); + auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Base32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -104,7 +104,7 @@ SV * queryPathInfo(char * path, int base32) XPUSHs(&PL_sv_undef); else XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); - auto s = info->narHash.to_string(base32 ? Base32 : Base16, true); + auto s = info->narHash.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); @@ -206,7 +206,7 @@ SV * hashPath(char * algo, int base32, char * path) PPCODE: try { Hash h = hashPath(parseHashType(algo), path).first; - auto s = h.to_string(base32 ? Base32 : Base16, false); + auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -217,7 +217,7 @@ SV * hashFile(char * algo, int base32, char * path) PPCODE: try { Hash h = hashFile(parseHashType(algo), path); - auto s = h.to_string(base32 ? Base32 : Base16, false); + auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -228,7 +228,7 @@ SV * hashString(char * algo, int base32, char * s) PPCODE: try { Hash h = hashString(parseHashType(algo), s); - auto s = h.to_string(base32 ? Base32 : Base16, false); + auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -239,7 +239,7 @@ SV * convertHash(char * algo, char * s, int toBase32) PPCODE: try { auto h = Hash::parseAny(s, parseHashType(algo)); - auto s = h.to_string(toBase32 ? Base32 : Base16, false); + auto s = h.to_string(toBase32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 391b32a77..10fc799a9 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -50,7 +50,7 @@ struct AttrDb Path cacheDir = getCacheDir() + "/nix/eval-cache-v5"; createDirs(cacheDir); - Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; + Path dbPath = cacheDir + "/" + fingerprint.to_string(HashFormat::Base16, false) + ".sqlite"; state->db = SQLite(dbPath); state->db.isCache(); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3bed7c5aa..830579dbf 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1758,7 +1758,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, V auto path = realisePath(state, pos, *args[1]); - v.mkString(hashString(*ht, path.readFile()).to_string(Base16, false)); + v.mkString(hashString(*ht, path.readFile()).to_string(HashFormat::Base16, false)); } static RegisterPrimOp primop_hashFile({ @@ -3760,7 +3760,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, NixStringContext context; // discarded auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); - v.mkString(hashString(*ht, s).to_string(Base16, false)); + v.mkString(hashString(*ht, s).to_string(HashFormat::Base16, false)); } static RegisterPrimOp primop_hashString({ diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 3431ff013..976037ff9 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -31,7 +31,7 @@ void emitTreeAttrs( auto narHash = input.getNarHash(); assert(narHash); - attrs.alloc("narHash").mkString(narHash->to_string(SRI, true)); + attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true)); if (input.getType() == "git") attrs.alloc("submodules").mkBool( @@ -297,7 +297,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v : hashFile(htSHA256, state.store->toRealPath(storePath)); if (hash != *expectedHash) state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", - *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true))); + *url, expectedHash->to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true))); } state.allowAndSetStorePathString(storePath, v); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index e3d3ac562..c54c39cf0 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -147,12 +147,12 @@ std::pair Input::fetch(ref store) const }; auto narHash = store->queryPathInfo(tree.storePath)->narHash; - input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true)); + input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); if (auto prevNarHash = getNarHash()) { if (narHash != *prevNarHash) throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true)); + to_string(), tree.actualPath, prevNarHash->to_string(HashFormat::SRI, true), narHash.to_string(HashFormat::SRI, true)); } if (auto prevLastModified = getLastModified()) { diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7e9f34790..c1a6dce43 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -46,7 +46,7 @@ bool touchCacheFile(const Path & path, time_t touch_time) Path getCachePath(std::string_view key) { return getCacheDir() + "/nix/gitv3/" + - hashString(htSHA256, key).to_string(Base32, false); + hashString(htSHA256, key).to_string(HashFormat::Base32, false); } // Returns the name of the HEAD branch. @@ -417,7 +417,7 @@ struct GitInputScheme : InputScheme auto checkHashType = [&](const std::optional & hash) { if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256)) - throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(Base16, true)); + throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true)); }; auto getLockedAttrs = [&]() diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 6c5c792ec..d7450defe 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -125,7 +125,7 @@ struct GitArchiveInputScheme : InputScheme auto path = owner + "/" + repo; assert(!(ref && rev)); if (ref) path += "/" + *ref; - if (rev) path += "/" + rev->to_string(Base16, false); + if (rev) path += "/" + rev->to_string(HashFormat::Base16, false); return ParsedURL { .scheme = type(), .path = path, @@ -296,7 +296,7 @@ struct GitHubInputScheme : GitArchiveInputScheme : "https://api.%s/repos/%s/%s/tarball/%s"; const auto url = fmt(urlFmt, host, getOwner(input), getRepo(input), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(HashFormat::Base16, false)); return DownloadUrl { url, headers }; } @@ -362,7 +362,7 @@ struct GitLabInputScheme : GitArchiveInputScheme auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(HashFormat::Base16, false)); Headers headers = makeHeadersWithAuthTokens(host); return DownloadUrl { url, headers }; @@ -449,7 +449,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); auto url = fmt("https://%s/%s/%s/archive/%s.tar.gz", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(HashFormat::Base16, false)); Headers headers = makeHeadersWithAuthTokens(host); return DownloadUrl { url, headers }; diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 51fd1ed42..b0d2d1909 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -206,7 +206,7 @@ struct MercurialInputScheme : InputScheme auto checkHashType = [&](const std::optional & hash) { if (hash.has_value() && hash->type != htSHA1) - throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(Base16, true)); + throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(HashFormat::Base16, true)); }; @@ -252,7 +252,7 @@ struct MercurialInputScheme : InputScheme } } - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(HashFormat::Base32, false)); /* If this is a commit hash that we already have, we don't have to pull again. */ diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index e3a41e75e..269a56526 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -250,7 +250,7 @@ struct CurlInputScheme : InputScheme // NAR hashes are preferred over file hashes since tar/zip // files don't have a canonical representation. if (auto narHash = input.getNarHash()) - url.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); + url.query.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true)); return url; } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index b4fea693f..2a91233ec 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -164,7 +164,7 @@ ref BinaryCacheStore::addToStoreCommon( auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; - narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" + narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "zstd" ? ".zst" : diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6a02f5ad4..40a0385af 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1066,7 +1066,7 @@ void LocalDerivationGoal::initTmpDir() { env[i.first] = i.second; } else { auto hash = hashString(htSHA256, i.first); - std::string fn = ".attr-" + hash.to_string(Base32, false); + std::string fn = ".attr-" + hash.to_string(HashFormat::Base32, false); Path p = tmpDir + "/" + fn; writeFile(p, rewriteStrings(i.second, inputRewrites)); chownToBuilder(p); @@ -2583,8 +2583,8 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", worker.store.printStorePath(drvPath), - wanted.to_string(SRI, true), - got.to_string(SRI, true))); + wanted.to_string(HashFormat::SRI, true), + got.to_string(HashFormat::SRI, true))); } if (!newInfo0.references.empty()) delayedException = std::make_exception_ptr( diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 7d7924d77..357800333 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -65,7 +65,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; std::optional ht = parseHashTypeOpt(getAttr("outputHashAlgo")); Hash h = newHashAllowEmpty(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(HashFormat::Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index ae91b859b..52c60154c 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -60,7 +60,7 @@ std::string ContentAddress::render() const + makeFileIngestionPrefix(method); }, }, method.raw) - + this->hash.to_string(Base32, true); + + this->hash.to_string(HashFormat::Base32, true); } /** diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 8cbf6f044..5860dd548 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -334,7 +334,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); - to << hash.to_string(Base16, false); + to << hash.to_string(HashFormat::Base16, false); break; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index fc17e520c..a5ceb29dc 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -542,7 +542,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, [&](const DerivationOutput::CAFixed & dof) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); s += ','; printUnquotedString(s, dof.ca.printMethodAlgo()); - s += ','; printUnquotedString(s, dof.ca.hash.to_string(Base16, false)); + s += ','; printUnquotedString(s, dof.ca.hash.to_string(HashFormat::Base16, false)); }, [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); @@ -775,7 +775,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut auto & dof = std::get(i.second.raw); auto hash = hashString(htSHA256, "fixed:out:" + dof.ca.printMethodAlgo() + ":" - + dof.ca.hash.to_string(Base16, false) + ":" + + dof.ca.hash.to_string(HashFormat::Base16, false) + ":" + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } @@ -820,7 +820,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut const auto h = get(res.hashes, outputName); if (!h) throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); - inputs2[h->to_string(Base16, false)].value.insert(outputName); + inputs2[h->to_string(HashFormat::Base16, false)].value.insert(outputName); } } @@ -925,7 +925,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr [&](const DerivationOutput::CAFixed & dof) { out << store.printStorePath(dof.path(store, drv.name, i.first)) << dof.ca.printMethodAlgo() - << dof.ca.hash.to_string(Base16, false); + << dof.ca.hash.to_string(HashFormat::Base16, false); }, [&](const DerivationOutput::CAFloating & dof) { out << "" @@ -957,7 +957,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr std::string hashPlaceholder(const OutputNameView outputName) { // FIXME: memoize? - return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false); + return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false); } @@ -1162,7 +1162,7 @@ nlohmann::json DerivationOutput::toJSON( [&](const DerivationOutput::CAFixed & dof) { res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); res["hashAlgo"] = dof.ca.printMethodAlgo(); - res["hash"] = dof.ca.hash.to_string(Base16, false); + res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false); // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index 7e3f7548d..ca9f7476e 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -5,7 +5,7 @@ namespace nix { std::string DownstreamPlaceholder::render() const { - return "/" + hash.to_string(Base32, false); + return "/" + hash.to_string(HashFormat::Base32, false); } @@ -31,7 +31,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( xpSettings.require(Xp::DynamicDerivations); auto compressed = compressHash(placeholder.hash, 20); auto clearText = "nix-computed-output:" - + compressed.to_string(Base32, false) + + compressed.to_string(HashFormat::Base32, false) + ":" + std::string { outputName }; return DownstreamPlaceholder { hashString(htSHA256, clearText) diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 87b2f8741..91b7e30db 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -41,7 +41,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) Hash hash = hashSink.currentHash().first; if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); + printStorePath(path), info->narHash.to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true)); teeSink << exportMagic diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 516cbef83..fb7895817 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -43,7 +43,7 @@ static void makeSymlink(const Path & link, const Path & target) void LocalStore::addIndirectRoot(const Path & path) { - std::string hash = hashString(htSHA1, path).to_string(Base32, false); + std::string hash = hashString(htSHA1, path).to_string(HashFormat::Base32, false); Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash)); makeSymlink(realRoot, path); } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 703ded0b2..46c90ee31 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -209,7 +209,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << ServeProto::Command::AddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash.to_string(HashFormat::Base16, false); ServeProto::write(*this, *conn, info.references); conn->to << info.registrationTime diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 17b4ecc73..1c2f6023a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -826,7 +826,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmts->RegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base16, true)) + (info.narHash.to_string(HashFormat::Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -933,7 +933,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmts->UpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16, true)) + (info.narHash.to_string(HashFormat::Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) @@ -1236,7 +1236,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); + printStorePath(info.path), info.narHash.to_string(HashFormat::Base32, true), hashResult.first.to_string(HashFormat::Base32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s", @@ -1252,8 +1252,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (specified.hash != actualHash.hash) { throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), - specified.hash.to_string(Base32, true), - actualHash.hash.to_string(Base32, true)); + specified.hash.to_string(HashFormat::Base32, true), + actualHash.hash.to_string(HashFormat::Base32, true)); } } @@ -1545,7 +1545,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) for (auto & link : readDirectory(linksDir)) { printMsg(lvlTalkative, "checking contents of '%s'", link.name); Path linkPath = linksDir + "/" + link.name; - std::string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); + std::string hash = hashPath(htSHA256, linkPath).first.to_string(HashFormat::Base32, false); if (hash != link.name) { printError("link '%s' was modified! expected hash '%s', got '%s'", linkPath, link.name, hash); @@ -1578,7 +1578,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) if (info->narHash != nullHash && info->narHash != current.first) { printError("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)); + printStorePath(i), info->narHash.to_string(HashFormat::Base32, true), current.first.to_string(HashFormat::Base32, true)); if (repair) repairPath(i); else errors = true; } else { diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index c7176d30f..cdbcf7e74 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -332,9 +332,9 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(HashFormat::Base32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string(Base32, true)) + (info->narHash.to_string(HashFormat::Base32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index d17253741..ee2ddfd81 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -105,10 +105,10 @@ std::string NarInfo::to_string(const Store & store) const assert(compression != ""); res += "Compression: " + compression + "\n"; assert(fileHash && fileHash->type == htSHA256); - res += "FileHash: " + fileHash->to_string(Base32, true) + "\n"; + res += "FileHash: " + fileHash->to_string(HashFormat::Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; assert(narHash.type == htSHA256); - res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; + res += "NarHash: " + narHash.to_string(HashFormat::Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 4a79cf4a1..23c6a41e4 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -146,10 +146,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ Hash hash = hashPath(htSHA256, path).first; - debug("'%1%' has hash '%2%'", path, hash.to_string(Base32, true)); + debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Base32, true)); /* Check if this is a known hash. */ - Path linkPath = linksDir + "/" + hash.to_string(Base32, false); + Path linkPath = linksDir + "/" + hash.to_string(HashFormat::Base32, false); /* Maybe delete the link, if it has been corrupted. */ if (pathExists(linkPath)) { diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index ccb57104f..7ad3247da 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -12,7 +12,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32, true) + ";" + + narHash.to_string(HashFormat::Base32, true) + ";" + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } @@ -161,7 +161,7 @@ void ValidPathInfo::write( if (includePath) sink << store.printStorePath(path); sink << (deriver ? store.printStorePath(*deriver) : "") - << narHash.to_string(Base16, false); + << narHash.to_string(HashFormat::Base16, false); WorkerProto::write(store, WorkerProto::WriteConn { .to = sink }, references); diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 3c6b9fc10..ec3e53232 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -35,7 +35,7 @@ StorePath::StorePath(std::string_view _baseName) } StorePath::StorePath(const Hash & hash, std::string_view _name) - : baseName((hash.to_string(Base32, false) + "-").append(std::string(_name))) + : baseName((hash.to_string(HashFormat::Base32, false) + "-").append(std::string(_name))) { checkName(baseName, name()); } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 559483ce3..4ba2123d8 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -39,7 +39,7 @@ struct DrvOutput { std::string to_string() const; std::string strHash() const - { return drvHash.to_string(Base16, true); } + { return drvHash.to_string(HashFormat::Base16, true); } static DrvOutput parse(const std::string &); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index a639346d1..1704bfbb0 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -541,7 +541,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn->to << WorkerProto::Op::AddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash.to_string(HashFormat::Base16, false); WorkerProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index fa23f976e..0c58cca0c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -154,7 +154,7 @@ StorePath Store::makeStorePath(std::string_view type, StorePath Store::makeStorePath(std::string_view type, const Hash & hash, std::string_view name) const { - return makeStorePath(type, hash.to_string(Base16, true), name); + return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name); } @@ -192,7 +192,7 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf hashString(htSHA256, "fixed:out:" + makeFileIngestionPrefix(info.method) - + info.hash.to_string(Base16, true) + ":"), + + info.hash.to_string(HashFormat::Base16, true) + ":"), name); } } @@ -884,7 +884,7 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths, auto info = queryPathInfo(i); if (showHash) { - s += info->narHash.to_string(Base16, false) + "\n"; + s += info->narHash.to_string(HashFormat::Base16, false) + "\n"; s += fmt("%1%\n", info->narSize); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f30ccc950..2d27f7d51 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -676,7 +676,7 @@ public: */ nlohmann::json pathInfoToJSON(const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, - HashFormat hashFormat = Base32, + HashFormat hashFormat = HashFormat::Base32, AllowInvalidFlag allowInvalid = DisallowInvalid); /** diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 401a65ec3..b58b5f08d 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -111,26 +111,26 @@ static std::string printHash32(const Hash & hash) std::string printHash16or32(const Hash & hash) { assert(hash.type); - return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false); + return hash.to_string(hash.type == htMD5 ? HashFormat::Base16 : HashFormat::Base32, false); } std::string Hash::to_string(HashFormat hashFormat, bool includeType) const { std::string s; - if (hashFormat == SRI || includeType) { + if (hashFormat == HashFormat::SRI || includeType) { s += printHashType(type); - s += hashFormat == SRI ? '-' : ':'; + s += hashFormat == HashFormat::SRI ? '-' : ':'; } switch (hashFormat) { - case Base16: + case HashFormat::Base16: s += printHash16(*this); break; - case Base32: + case HashFormat::Base32: s += printHash32(*this); break; - case Base64: - case SRI: + case HashFormat::Base64: + case HashFormat::SRI: s += base64Encode(std::string_view((const char *) hash, hashSize)); break; } @@ -267,7 +267,7 @@ Hash newHashAllowEmpty(std::string_view hashStr, std::optional ht) if (!ht) throw BadHash("empty hash requires explicit hash type"); Hash h(*ht); - warn("found empty hash, assuming '%s'", h.to_string(SRI, true)); + warn("found empty hash, assuming '%s'", h.to_string(HashFormat::SRI, true)); return h; } else return Hash::parseAny(hashStr, ht); diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 0db9bcd6d..783e30496 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -132,12 +132,12 @@ public: std::string gitRev() const { - return to_string(Base16, false); + return to_string(HashFormat::Base16, false); } std::string gitShortRev() const { - return std::string(to_string(Base16, false), 0, 7); + return std::string(to_string(HashFormat::Base16, false), 0, 7); } static Hash dummy; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 9b6c80a75..5e494bcbf 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -406,7 +406,7 @@ static void opQuery(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(j); if (query == qHash) { assert(info->narHash.type == htSHA256); - cout << fmt("%s\n", info->narHash.to_string(Base32, true)); + cout << fmt("%s\n", info->narHash.to_string(HashFormat::Base32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); } @@ -769,8 +769,8 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) if (current.first != info->narHash) { printError("path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(path), - info->narHash.to_string(Base32, true), - current.first.to_string(Base32, true)); + info->narHash.to_string(HashFormat::Base32, true), + current.first.to_string(HashFormat::Base32, true)); status = 1; } } @@ -892,7 +892,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << info->narHash.to_string(Base32, true) + out << info->narHash.to_string(HashFormat::Base32, true) << renderContentAddress(info->ca) << info->sigs; } catch (InvalidPath &) { diff --git a/src/nix/flake.cc b/src/nix/flake.cc index a77b5fcb8..ceb112c03 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -179,7 +179,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs()); if (auto rev = flake.lockedRef.input.getRev()) - j["revision"] = rev->to_string(Base16, false); + j["revision"] = rev->to_string(HashFormat::Base16, false); if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) j["dirtyRevision"] = *dirtyRev; if (auto revCount = flake.lockedRef.input.getRevCount()) @@ -206,7 +206,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON if (auto rev = flake.lockedRef.input.getRev()) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", - rev->to_string(Base16, false)); + rev->to_string(HashFormat::Base16, false)); if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", @@ -1345,7 +1345,7 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON if (json) { auto res = nlohmann::json::object(); res["storePath"] = store->printStorePath(tree.storePath); - res["hash"] = hash.to_string(SRI, true); + res["hash"] = hash.to_string(HashFormat::SRI, true); res["original"] = fetchers::attrsToJSON(resolvedRef.toAttrs()); res["locked"] = fetchers::attrsToJSON(lockedRef.toAttrs()); logger->cout(res.dump()); @@ -1353,7 +1353,7 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON notice("Downloaded '%s' to '%s' (hash '%s').", lockedRef.to_string(), store->printStorePath(tree.storePath), - hash.to_string(SRI, true)); + hash.to_string(HashFormat::SRI, true)); } } }; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 6af6bd19f..d6595dcca 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -11,7 +11,7 @@ using namespace nix; struct CmdHashBase : Command { FileIngestionMethod mode; - HashFormat hashFormat = SRI; + HashFormat hashFormat = HashFormat::SRI; bool truncate = false; HashType ht = htSHA256; std::vector paths; @@ -22,25 +22,25 @@ struct CmdHashBase : Command addFlag({ .longName = "sri", .description = "Print the hash in SRI format.", - .handler = {&hashFormat, SRI}, + .handler = {&hashFormat, HashFormat::SRI}, }); addFlag({ .longName = "base64", .description = "Print the hash in base-64 format.", - .handler = {&hashFormat, Base64}, + .handler = {&hashFormat, HashFormat::Base64}, }); addFlag({ .longName = "base32", .description = "Print the hash in base-32 (Nix-specific) format.", - .handler = {&hashFormat, Base32}, + .handler = {&hashFormat, HashFormat::Base32}, }); addFlag({ .longName = "base16", .description = "Print the hash in base-16 format.", - .handler = {&hashFormat, Base16}, + .handler = {&hashFormat, HashFormat::Base16}, }); addFlag(Flag::mkHashTypeFlag("type", &ht)); @@ -94,7 +94,7 @@ struct CmdHashBase : Command Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); - logger->cout(h.to_string(hashFormat, hashFormat == SRI)); + logger->cout(h.to_string(hashFormat, hashFormat == HashFormat::SRI)); } } }; @@ -114,16 +114,16 @@ struct CmdToBase : Command std::string description() override { return fmt("convert a hash to %s representation", - hashFormat == Base16 ? "base-16" : - hashFormat == Base32 ? "base-32" : - hashFormat == Base64 ? "base-64" : + hashFormat == HashFormat::Base16 ? "base-16" : + hashFormat == HashFormat::Base32 ? "base-32" : + hashFormat == HashFormat::Base64 ? "base-64" : "SRI"); } void run() override { for (auto s : args) - logger->cout(Hash::parseAny(s, ht).to_string(hashFormat, hashFormat == SRI)); + logger->cout(Hash::parseAny(s, ht).to_string(hashFormat, hashFormat == HashFormat::SRI)); } }; @@ -133,10 +133,10 @@ struct CmdHash : NixMultiCommand : MultiCommand({ {"file", []() { return make_ref(FileIngestionMethod::Flat);; }}, {"path", []() { return make_ref(FileIngestionMethod::Recursive); }}, - {"to-base16", []() { return make_ref(Base16); }}, - {"to-base32", []() { return make_ref(Base32); }}, - {"to-base64", []() { return make_ref(Base64); }}, - {"to-sri", []() { return make_ref(SRI); }}, + {"to-base16", []() { return make_ref(HashFormat::Base16); }}, + {"to-base32", []() { return make_ref(HashFormat::Base32); }}, + {"to-base64", []() { return make_ref(HashFormat::Base64); }}, + {"to-sri", []() { return make_ref(HashFormat::SRI); }}, }) { } @@ -162,7 +162,7 @@ static int compatNixHash(int argc, char * * argv) { std::optional ht; bool flat = false; - HashFormat hashFormat = Base16; + HashFormat hashFormat = HashFormat::Base16; bool truncate = false; enum { opHash, opTo } op = opHash; std::vector ss; @@ -173,10 +173,10 @@ static int compatNixHash(int argc, char * * argv) else if (*arg == "--version") printVersion("nix-hash"); else if (*arg == "--flat") flat = true; - else if (*arg == "--base16") hashFormat = Base16; - else if (*arg == "--base32") hashFormat = Base32; - else if (*arg == "--base64") hashFormat = Base64; - else if (*arg == "--sri") hashFormat = SRI; + else if (*arg == "--base16") hashFormat = HashFormat::Base16; + else if (*arg == "--base32") hashFormat = HashFormat::Base32; + else if (*arg == "--base64") hashFormat = HashFormat::Base64; + else if (*arg == "--sri") hashFormat = HashFormat::SRI; else if (*arg == "--truncate") truncate = true; else if (*arg == "--type") { std::string s = getArg(*arg, arg, end); @@ -184,19 +184,19 @@ static int compatNixHash(int argc, char * * argv) } else if (*arg == "--to-base16") { op = opTo; - hashFormat = Base16; + hashFormat = HashFormat::Base16; } else if (*arg == "--to-base32") { op = opTo; - hashFormat = Base32; + hashFormat = HashFormat::Base32; } else if (*arg == "--to-base64") { op = opTo; - hashFormat = Base64; + hashFormat = HashFormat::Base64; } else if (*arg == "--to-sri") { op = opTo; - hashFormat = SRI; + hashFormat = HashFormat::SRI; } else if (*arg != "" && arg->at(0) == '-') return false; diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 613c5b191..c16864d30 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -90,7 +90,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON std::cout << store->pathInfoToJSON( // FIXME: preserve order? StorePathSet(storePaths.begin(), storePaths.end()), - true, showClosureSize, SRI, AllowInvalid).dump(); + true, showClosureSize, HashFormat::SRI, AllowInvalid).dump(); } else { diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index b67d381ca..3ed7946a8 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -310,13 +310,13 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON if (json) { auto res = nlohmann::json::object(); res["storePath"] = store->printStorePath(storePath); - res["hash"] = hash.to_string(SRI, true); + res["hash"] = hash.to_string(HashFormat::SRI, true); logger->cout(res.dump()); } else { notice("Downloaded '%s' to '%s' (hash '%s').", url, store->printStorePath(storePath), - hash.to_string(SRI, true)); + hash.to_string(HashFormat::SRI, true)); } } }; diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 0b306cc11..adaa33c0c 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -108,8 +108,8 @@ struct CmdVerify : StorePathsCommand act2.result(resCorruptedPath, store->printStorePath(info->path)); printError("path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(info->path), - info->narHash.to_string(Base32, true), - hash.first.to_string(Base32, true)); + info->narHash.to_string(HashFormat::Base32, true), + hash.first.to_string(HashFormat::Base32, true)); } } From 231b0fca6df668f904dfb86ef858db556caebdff Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 13 Oct 2023 18:48:36 +0800 Subject: [PATCH 410/909] Migrate HashFormat to scoped enumeration (enum struct) --- src/libutil/hash.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 783e30496..d2bf1cb23 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -26,7 +26,7 @@ extern const std::string base32Chars; /** * @brief Enumeration representing the hash formats. */ -enum HashFormat : int { +enum struct HashFormat : int { /// @brief Base 64 encoding. /// @see [IETF RFC 4648, section 4](https://datatracker.ietf.org/doc/html/rfc4648#section-4). Base64, From 6b47635180c740b7478f187b3ae9e35b36deeed7 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 9 Oct 2023 11:03:16 +0800 Subject: [PATCH 411/909] Add helper function parseHashFormat[Opt] printHashFormat Add hash format analogy of parseHashTypeOpt, parseHashType, and printHashType. Co-authored-by: Valentin Gagarin --- src/libutil/hash.cc | 35 +++++++++++++++++++++++++++++++++++ src/libutil/hash.hh | 15 +++++++++++++++ src/libutil/tests/hash.cc | 15 +++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index b58b5f08d..e297c245b 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -386,6 +386,41 @@ Hash compressHash(const Hash & hash, unsigned int newSize) } +std::optional parseHashFormatOpt(std::string_view hashFormatName) +{ + if (hashFormatName == "base16") return HashFormat::Base16; + if (hashFormatName == "base32") return HashFormat::Base32; + if (hashFormatName == "base64") return HashFormat::Base64; + if (hashFormatName == "sri") return HashFormat::SRI; + return std::nullopt; +} + +HashFormat parseHashFormat(std::string_view hashFormatName) +{ + auto opt_f = parseHashFormatOpt(hashFormatName); + if (opt_f) + return *opt_f; + throw UsageError("unknown hash format '%1%', expect 'base16', 'base32', 'base64', or 'sri'", hashFormatName); +} + +std::string_view printHashFormat(HashFormat HashFormat) +{ + switch (HashFormat) { + case HashFormat::Base64: + return "base64"; + case HashFormat::Base32: + return "base32"; + case HashFormat::Base16: + return "base16"; + case HashFormat::SRI: + return "sri"; + default: + // illegal hash base enum value internally, as opposed to external input + // which should be validated with nice error message. + assert(false); + } +} + std::optional parseHashTypeOpt(std::string_view s) { if (s == "md5") return htMD5; diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index d2bf1cb23..cab3e6eca 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -181,6 +181,21 @@ HashResult hashPath(HashType ht, const Path & path, */ Hash compressHash(const Hash & hash, unsigned int newSize); +/** + * Parse a string representing a hash format. + */ +HashFormat parseHashFormat(std::string_view hashFormatName); + +/** + * std::optional version of parseHashFormat that doesn't throw error. + */ +std::optional parseHashFormatOpt(std::string_view hashFormatName); + +/** + * The reverse of parseHashFormat. + */ +std::string_view printHashFormat(HashFormat hashFormat); + /** * Parse a string representing a hash type. */ diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc index d14e5b26d..9a5ebbb30 100644 --- a/src/libutil/tests/hash.cc +++ b/src/libutil/tests/hash.cc @@ -79,6 +79,21 @@ namespace nix { "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" "c7d329eeb6dd26545e96e55b874be909"); } + + /* ---------------------------------------------------------------------------- + * parseHashFormat, parseHashFormatOpt, printHashFormat + * --------------------------------------------------------------------------*/ + + TEST(hashFormat, testRoundTripPrintParse) { + for (const HashFormat hashFormat: { HashFormat::Base64, HashFormat::Base32, HashFormat::Base16, HashFormat::SRI}) { + ASSERT_EQ(parseHashFormat(printHashFormat(hashFormat)), hashFormat); + ASSERT_EQ(*parseHashFormatOpt(printHashFormat(hashFormat)), hashFormat); + } + } + + TEST(hashFormat, testParseHashFormatOptException) { + ASSERT_EQ(parseHashFormatOpt("sha0042"), std::nullopt); + } } namespace rc { From 5088e6563a4e88b7e6615623d3ed655fdebd3345 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li <44064051+ShamrockLee@users.noreply.github.com> Date: Sun, 29 Jan 2023 14:03:06 +0800 Subject: [PATCH 412/909] primops: add builtins.convertHash Co-authored-by: Valentin Gagarin --- doc/manual/src/release-notes/rl-next.md | 2 + src/libexpr/primops.cc | 95 +++++++++++++++++++ .../functional/lang/eval-okay-convertHash.exp | 1 + .../functional/lang/eval-okay-convertHash.nix | 31 ++++++ 4 files changed, 129 insertions(+) create mode 100644 tests/functional/lang/eval-okay-convertHash.exp create mode 100644 tests/functional/lang/eval-okay-convertHash.nix diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index c905f445f..46ce2abac 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -8,4 +8,6 @@ - Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.` or `legacyPackages.`. +- Introduce a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash). + - `builtins.fetchTree` is now marked as stable. diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 830579dbf..14beb069f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3774,6 +3774,101 @@ static RegisterPrimOp primop_hashString({ .fun = prim_hashString, }); +static void prim_convertHash(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos, "while evaluating the first argument passed to builtins.convertHash"); + auto &inputAttrs = args[0]->attrs; + + Bindings::iterator iteratorHash = getAttr(state, state.symbols.create("hash"), inputAttrs, "while locating the attribute 'hash'"); + auto hash = state.forceStringNoCtx(*iteratorHash->value, pos, "while evaluating the attribute 'hash'"); + + Bindings::iterator iteratorHashAlgo = inputAttrs->find(state.symbols.create("hashAlgo")); + std::optional ht = std::nullopt; + if (iteratorHashAlgo != inputAttrs->end()) { + ht = parseHashType(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'")); + } + + Bindings::iterator iteratorToHashFormat = getAttr(state, state.symbols.create("toHashFormat"), args[0]->attrs, "while locating the attribute 'toHashFormat'"); + HashFormat hf = parseHashFormat(state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'")); + + v.mkString(Hash::parseAny(hash, ht).to_string(hf, hf == HashFormat::SRI)); +} + +static RegisterPrimOp primop_convertHash({ + .name = "__convertHash", + .args = {"args"}, + .doc = R"( + Return the specified representation of a hash string, based on the attributes presented in *args*: + + - `hash` + + The hash to be converted. + The hash format is detected automatically. + + - `hashAlgo` + + The algorithm used to create the hash. Must be one of + - `"md5"` + - `"sha1"` + - `"sha256"` + - `"sha512"` + + The attribute may be omitted when `hash` is an [SRI hash](https://www.w3.org/TR/SRI/#the-integrity-attribute) or when the hash is prefixed with the hash algorithm name followed by a colon. + That `:` syntax is supported for backwards compatibility with existing tooling. + + - `toHashFormat` + + The format of the resulting hash. Must be one of + - `"base16"` + - `"base32"` + - `"base64"` + - `"sri"` + + The result hash is the *toHashFormat* representation of the hash *hash*. + + > **Example** + > + > Convert a SHA256 hash in Base16 to SRI: + > + > ```nix + > builtins.convertHash { + > hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + > toHashFormat = "sri"; + > hashAlgo = "sha256"; + > } + > ``` + > + > "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + + > **Example** + > + > Convert a SHA256 hash in SRI to Base16: + > + > ```nix + > builtins.convertHash { + > hash = "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="; + > toHashFormat = "base16"; + > } + > ``` + > + > "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + > **Example** + > + > Convert a hash in the form `:` in Base16 to SRI: + > + > ```nix + > builtins.convertHash { + > hash = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + > toHashFormat = "sri"; + > } + > ``` + > + > "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + )", + .fun = prim_convertHash, +}); + struct RegexCache { // TODO use C++20 transparent comparison when available diff --git a/tests/functional/lang/eval-okay-convertHash.exp b/tests/functional/lang/eval-okay-convertHash.exp new file mode 100644 index 000000000..60e0a3c49 --- /dev/null +++ b/tests/functional/lang/eval-okay-convertHash.exp @@ -0,0 +1 @@ +{ hashesBase16 = [ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]; hashesBase32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesBase64 = [ "1B2M2Y8AsgTpgAmY7PhCfg==" "bGnufyEcZAQZ1TZswHauRg==" "uzQ4+6vUYOptvSfRU+IjOw==" "2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "zVToVowbN88eW62wd5vL84IhIYk=" "bRLhCx0zHa0hDkf9JdTyYIArfnc=" "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; hashesSRI = [ "md5-1B2M2Y8AsgTpgAmY7PhCfg==" "md5-bGnufyEcZAQZ1TZswHauRg==" "md5-uzQ4+6vUYOptvSfRU+IjOw==" "sha1-2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "sha1-zVToVowbN88eW62wd5vL84IhIYk=" "sha1-bRLhCx0zHa0hDkf9JdTyYIArfnc=" "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "sha256-kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "sha256-rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "sha512-nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "sha512-IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; } diff --git a/tests/functional/lang/eval-okay-convertHash.nix b/tests/functional/lang/eval-okay-convertHash.nix new file mode 100644 index 000000000..cf4909aaf --- /dev/null +++ b/tests/functional/lang/eval-okay-convertHash.nix @@ -0,0 +1,31 @@ +let + hashAlgos = [ "md5" "md5" "md5" "sha1" "sha1" "sha1" "sha256" "sha256" "sha256" "sha512" "sha512" "sha512" ]; + hashesBase16 = import ./eval-okay-hashstring.exp; + map2 = f: { fsts, snds }: if fsts == [ ] then [ ] else [ (f (builtins.head fsts) (builtins.head snds)) ] ++ map2 f { fsts = builtins.tail fsts; snds = builtins.tail snds; }; + map2' = f: fsts: snds: map2 f { inherit fsts snds; }; + getOutputHashes = hashes: { + hashesBase16 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base16";}) hashAlgos hashes; + hashesBase32 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";}) hashAlgos hashes; + hashesBase64 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base64";}) hashAlgos hashes; + hashesSRI = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "sri" ;}) hashAlgos hashes; + }; + getOutputHashesColon = hashes: { + hashesBase16 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base16";}) hashAlgos hashes; + hashesBase32 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base32";}) hashAlgos hashes; + hashesBase64 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base64";}) hashAlgos hashes; + hashesSRI = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "sri" ;}) hashAlgos hashes; + }; + outputHashes = getOutputHashes hashesBase16; +in +# map2'` +assert map2' (s1: s2: s1 + s2) [ "a" "b" ] [ "c" "d" ] == [ "ac" "bd" ]; +# hashesBase16 +assert outputHashes.hashesBase16 == hashesBase16; +# standard SRI hashes +assert outputHashes.hashesSRI == (map2' (hashAlgo: hashBody: hashAlgo + "-" + hashBody) hashAlgos outputHashes.hashesBase64); +# without prefix +assert builtins.all (x: getOutputHashes x == outputHashes) (builtins.attrValues outputHashes); +# colon-separated. +# Note that colon prefix must not be applied to the standard SRI. e.g. "sha256:sha256-..." is illegal. +assert builtins.all (x: getOutputHashesColon x == outputHashes) (with outputHashes; [ hashesBase16 hashesBase32 hashesBase64 ]); +outputHashes From d2c0051784e4a338254445f8f680abf1e37e9024 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Oct 2023 23:35:07 +0200 Subject: [PATCH 413/909] Remove obsolete corepkgs references --- doc/manual/local.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 2e9f08678..8bf16e9dd 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -125,7 +125,7 @@ $(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $ @mv $@.tmp $@ $(d)/xp-features.json: $(bindir)/nix - $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp + $(trace-gen) $(dummy-env) $(bindir)/nix __dump-xp-features > $@.tmp @mv $@.tmp $@ $(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix @@ -141,7 +141,7 @@ $(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin @mv $@.tmp $@ $(d)/language.json: $(bindir)/nix - $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-language > $@.tmp + $(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp @mv $@.tmp $@ # Generate the HTML manual. From 34c559352579b9e1a501ec2fc435f3e21cf6e78f Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 19 Oct 2023 10:57:45 +0200 Subject: [PATCH 414/909] add a link to all maintainer meeting notes linking to the discourse category will by default show a view sorted by most recent post, which makes it hard to find particular meeting notes. this also adds a procedural detail about the notes, to make that more explicit and less dependent on being present in the meetings. --- maintainers/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/maintainers/README.md b/maintainers/README.md index 8eeb47e8b..5be4f9d04 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -50,7 +50,9 @@ The team meets twice a week: 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. -Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw), and published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). +Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw). +Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are easy to find from both places. +[All meeting notes](https://discourse.nixos.org/search?expanded=true&q=Nix%20team%20meeting%20minutes%20%23%20%23dev%3Anix%20in%3Atitle%20order%3Alatest_topic) are published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). ## Project board protocol From 9adac237e70003625b626cef04979ce746e9dd7e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 19 Oct 2023 11:40:27 +0200 Subject: [PATCH 415/909] update link to label GitHub now displays a banner and has a dedicated page[1] for good first issues, but that uses a different label name as we had in place. I renamed the label on GitHub, this is updating the link. [1]: https://github.com/NixOS/nix/contribute --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 73210b303..556365375 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,7 +27,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). 1. Search for related issues that cover what you're going to work on. It could help to mention there that you will work on the issue. - Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly. + Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good%20first%20issue) should be relatively easy to fix and are likely to get merged quickly. Pull requests addressing issues labeled [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) or [RFC](https://github.com/NixOS/nix/labels/RFC) are especially welcomed by maintainers and will receive prioritised review. If there is no relevant issue yet and you're not sure whether your change is likely to be accepted, [open an issue](https://github.com/NixOS/nix/issues/new/choose) yourself. From 36b15d905e8ca6f5eb4f73fb842600b6e091708e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 19 Oct 2023 11:47:10 +0200 Subject: [PATCH 416/909] link to popular issues from the contributing guide this also adds a hint to contributors about making far-reaching changes, complementing the recent update to the maintainers' handbook on how to deal with those. --- CONTRIBUTING.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 73210b303..71970f307 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,6 +30,9 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly. Pull requests addressing issues labeled [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) or [RFC](https://github.com/NixOS/nix/labels/RFC) are especially welcomed by maintainers and will receive prioritised review. + If you are proficient with C++, addressing one of the [popular issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc) will be highly appreciated by maintainers and Nix users all over the world. + For far-reaching changes, please investigate possible blockers and design implications, and coordinate with maintainers before investing too much time in writing code that may not end up getting merged. + If there is no relevant issue yet and you're not sure whether your change is likely to be accepted, [open an issue](https://github.com/NixOS/nix/issues/new/choose) yourself. 2. Check for [pull requests](https://github.com/NixOS/nix/pulls) that might already cover the contribution you are about to make. From 12214fef09151e26abfcaea0095ab157388822e3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 14:19:10 +0200 Subject: [PATCH 417/909] InputAccessor::fetchToStore(): Support arbitrary ingestion methods --- src/libexpr/eval.cc | 2 +- src/libexpr/primops.cc | 5 +---- src/libfetchers/input-accessor.cc | 8 +++++--- src/libfetchers/input-accessor.hh | 3 +++ 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index a28bb2101..3b0fe93e1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2329,7 +2329,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat auto dstPath = i != srcToStore.end() ? i->second : [&]() { - auto dstPath = path.fetchToStore(store, path.baseName(), nullptr, repair); + auto dstPath = path.fetchToStore(store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair); allowPath(dstPath); srcToStore.insert_or_assign(path, dstPath); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f1c24a75c..f416aa639 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2233,10 +2233,7 @@ static void addPath( }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - // FIXME - if (method != FileIngestionMethod::Recursive) - throw Error("'recursive = false' is not implemented"); - auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, &filter, state.repair); + auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, method, &filter, state.repair); if (expectedHash && expectedStorePath != dstPath) state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); state.allowAndSetStorePathString(dstPath, v); diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index fec16e99f..63f1f4ad2 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -96,6 +96,7 @@ StorePath InputAccessor::fetchToStore( ref store, const CanonPath & path, std::string_view name, + FileIngestionMethod method, PathFilter * filter, RepairFlag repair) { @@ -107,8 +108,8 @@ StorePath InputAccessor::fetchToStore( auto storePath = settings.readOnlyMode - ? store->computeStorePathFromDump(*source, name).first - : store->addToStoreFromDump(*source, name, FileIngestionMethod::Recursive, htSHA256, repair); + ? store->computeStorePathFromDump(*source, name, method, htSHA256).first + : store->addToStoreFromDump(*source, name, method, htSHA256, repair); return storePath; } @@ -140,10 +141,11 @@ std::ostream & operator << (std::ostream & str, const SourcePath & path) StorePath SourcePath::fetchToStore( ref store, std::string_view name, + FileIngestionMethod method, PathFilter * filter, RepairFlag repair) const { - return accessor->fetchToStore(store, path, name, filter, repair); + return accessor->fetchToStore(store, path, name, method, filter, repair); } std::string_view SourcePath::baseName() const diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index a0f08e295..003c7226c 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -6,6 +6,7 @@ #include "canon-path.hh" #include "repair-flag.hh" #include "hash.hh" +#include "content-address.hh" namespace nix { @@ -73,6 +74,7 @@ struct InputAccessor : public std::enable_shared_from_this ref store, const CanonPath & path, std::string_view name = "source", + FileIngestionMethod method = FileIngestionMethod::Recursive, PathFilter * filter = nullptr, RepairFlag repair = NoRepair); @@ -181,6 +183,7 @@ struct SourcePath StorePath fetchToStore( ref store, std::string_view name = "source", + FileIngestionMethod method = FileIngestionMethod::Recursive, PathFilter * filter = nullptr, RepairFlag repair = NoRepair) const; From f16af08e8393420f6234ab59bde716dd145703d7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 14:20:50 +0200 Subject: [PATCH 418/909] Fix macOS compilation --- src/libexpr/eval.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 3b0fe93e1..7180b9fbe 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -616,7 +616,9 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & SourcePath EvalState::checkSourcePath(const SourcePath & path_) { - if (path_.accessor != rootFS) return path_; + // Don't check non-rootFS accessors, they're in a different namespace. + if (path_.accessor != ref(rootFS)) return path_; + if (!allowedPaths) return path_; auto i = resolvedPaths.find(path_.path.abs()); From 06c57899e306bd6a19fce5d83b2ffce487067cdb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 14:22:05 +0200 Subject: [PATCH 419/909] Remove FIXME --- src/libexpr/primops.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f416aa639..b971fd052 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -595,7 +595,9 @@ struct CompareValues case nString: return v1->string_view().compare(v2->string_view()) < 0; case nPath: - // FIXME: handle accessor? + // Note: we don't take the accessor into account + // since it's not obvious how to compare them in a + // reproducible way. return strcmp(v1->_path.path, v2->_path.path) < 0; case nList: // Lexicographic comparison From 9bc7b4f463b6a06746bd1658d0170865e3fc8337 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Thu, 19 Oct 2023 14:39:41 +0200 Subject: [PATCH 420/909] doc: generic closure supported key types (#9183) * doc: generic closure supported key types Co-authored-by: Valentin Gagarin --- src/libexpr/primops.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 14beb069f..e5c5eb5b1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -734,6 +734,14 @@ static RegisterPrimOp primop_genericClosure(PrimOp { ``` [ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ] ``` + + `key` can be one of the following types: + - [Number](@docroot@/language/values.md#type-number) + - [Boolean](@docroot@/language/values.md#type-boolean) + - [String](@docroot@/language/values.md#type-string) + - [Path](@docroot@/language/values.md#type-path) + - [List](@docroot@/language/values.md#list) + )", .fun = prim_genericClosure, }); From fb6a3910c4b4ebec2ab2e422a7454b8f3ba71f85 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 14:45:26 +0200 Subject: [PATCH 421/909] Move most of InputAccessor into libutil --- src/libfetchers/input-accessor.cc | 102 ---------------------------- src/libfetchers/input-accessor.hh | 85 +---------------------- src/libutil/source-accessor.cc | 108 ++++++++++++++++++++++++++++++ src/libutil/source-accessor.hh | 96 ++++++++++++++++++++++++++ 4 files changed, 206 insertions(+), 185 deletions(-) create mode 100644 src/libutil/source-accessor.cc create mode 100644 src/libutil/source-accessor.hh diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index 63f1f4ad2..9bddf7c2e 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -3,95 +3,6 @@ namespace nix { -static std::atomic nextNumber{0}; - -InputAccessor::InputAccessor() - : number(++nextNumber) -{ -} - -// FIXME: merge with archive.cc. -void InputAccessor::dumpPath( - const CanonPath & path, - Sink & sink, - PathFilter & filter) -{ - auto dumpContents = [&](const CanonPath & path) - { - // FIXME: pipe - auto s = readFile(path); - sink << "contents" << s.size(); - sink(s); - writePadding(s.size(), sink); - }; - - std::function dump; - - dump = [&](const CanonPath & path) { - checkInterrupt(); - - auto st = lstat(path); - - sink << "("; - - if (st.type == tRegular) { - sink << "type" << "regular"; - if (st.isExecutable) - sink << "executable" << ""; - dumpContents(path); - } - - else if (st.type == tDirectory) { - sink << "type" << "directory"; - - /* If we're on a case-insensitive system like macOS, undo - the case hack applied by restorePath(). */ - std::map unhacked; - for (auto & i : readDirectory(path)) - if (/* archiveSettings.useCaseHack */ false) { // FIXME - std::string name(i.first); - size_t pos = i.first.find(caseHackSuffix); - if (pos != std::string::npos) { - debug("removing case hack suffix from '%s'", path + i.first); - name.erase(pos); - } - if (!unhacked.emplace(name, i.first).second) - throw Error("file name collision in between '%s' and '%s'", - (path + unhacked[name]), - (path + i.first)); - } else - unhacked.emplace(i.first, i.first); - - for (auto & i : unhacked) - if (filter((path + i.first).abs())) { - sink << "entry" << "(" << "name" << i.first << "node"; - dump(path + i.second); - sink << ")"; - } - } - - else if (st.type == tSymlink) - sink << "type" << "symlink" << "target" << readLink(path); - - else throw Error("file '%s' has an unsupported type", path); - - sink << ")"; - }; - - sink << narVersionMagic1; - dump(path); -} - -Hash InputAccessor::hashPath( - const CanonPath & path, - PathFilter & filter, - HashType ht) -{ - HashSink sink(ht); - dumpPath(path, sink, filter); - return sink.finish().first; -} - StorePath InputAccessor::fetchToStore( ref store, const CanonPath & path, @@ -114,19 +25,6 @@ StorePath InputAccessor::fetchToStore( return storePath; } -std::optional InputAccessor::maybeLstat(const CanonPath & path) -{ - // FIXME: merge these into one operation. - if (!pathExists(path)) - return {}; - return lstat(path); -} - -std::string InputAccessor::showPath(const CanonPath & path) -{ - return path.abs(); -} - SourcePath InputAccessor::root() { return {ref(shared_from_this()), CanonPath::root}; diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 003c7226c..524a94cd1 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -1,11 +1,9 @@ #pragma once +#include "source-accessor.hh" #include "ref.hh" #include "types.hh" -#include "archive.hh" -#include "canon-path.hh" #include "repair-flag.hh" -#include "hash.hh" #include "content-address.hh" namespace nix { @@ -16,60 +14,8 @@ struct SourcePath; class StorePath; class Store; -struct InputAccessor : public std::enable_shared_from_this +struct InputAccessor : SourceAccessor, std::enable_shared_from_this { - const size_t number; - - InputAccessor(); - - virtual ~InputAccessor() - { } - - virtual std::string readFile(const CanonPath & path) = 0; - - virtual bool pathExists(const CanonPath & path) = 0; - - enum Type { - tRegular, tSymlink, tDirectory, - /** - Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things. - - Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`. - - Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types. - */ - tMisc - }; - - struct Stat - { - Type type = tMisc; - //uint64_t fileSize = 0; // regular files only - bool isExecutable = false; // regular files only - }; - - virtual Stat lstat(const CanonPath & path) = 0; - - std::optional maybeLstat(const CanonPath & path); - - typedef std::optional DirEntry; - - typedef std::map DirEntries; - - virtual DirEntries readDirectory(const CanonPath & path) = 0; - - virtual std::string readLink(const CanonPath & path) = 0; - - virtual void dumpPath( - const CanonPath & path, - Sink & sink, - PathFilter & filter = defaultPathFilter); - - Hash hashPath( - const CanonPath & path, - PathFilter & filter = defaultPathFilter, - HashType ht = htSHA256); - StorePath fetchToStore( ref store, const CanonPath & path, @@ -78,34 +24,7 @@ struct InputAccessor : public std::enable_shared_from_this PathFilter * filter = nullptr, RepairFlag repair = NoRepair); - /* Return a corresponding path in the root filesystem, if - possible. This is only possible for inputs that are - materialized in the root filesystem. */ - virtual std::optional getPhysicalPath(const CanonPath & path) - { return std::nullopt; } - - bool operator == (const InputAccessor & x) const - { - return number == x.number; - } - - bool operator < (const InputAccessor & x) const - { - return number < x.number; - } - - void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); - - virtual std::string showPath(const CanonPath & path); - SourcePath root(); - - /* Return the maximum last-modified time of the files in this - tree, if available. */ - virtual std::optional getLastModified() - { - return std::nullopt; - } }; /** diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc new file mode 100644 index 000000000..23ec55c89 --- /dev/null +++ b/src/libutil/source-accessor.cc @@ -0,0 +1,108 @@ +#include "source-accessor.hh" +#include "archive.hh" + +namespace nix { + +static std::atomic nextNumber{0}; + +SourceAccessor::SourceAccessor() + : number(++nextNumber) +{ +} + +// FIXME: merge with archive.cc. +void SourceAccessor::dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter) +{ + auto dumpContents = [&](const CanonPath & path) + { + // FIXME: pipe + auto s = readFile(path); + sink << "contents" << s.size(); + sink(s); + writePadding(s.size(), sink); + }; + + std::function dump; + + dump = [&](const CanonPath & path) { + checkInterrupt(); + + auto st = lstat(path); + + sink << "("; + + if (st.type == tRegular) { + sink << "type" << "regular"; + if (st.isExecutable) + sink << "executable" << ""; + dumpContents(path); + } + + else if (st.type == tDirectory) { + sink << "type" << "directory"; + + /* If we're on a case-insensitive system like macOS, undo + the case hack applied by restorePath(). */ + std::map unhacked; + for (auto & i : readDirectory(path)) + if (/* archiveSettings.useCaseHack */ false) { // FIXME + std::string name(i.first); + size_t pos = i.first.find(caseHackSuffix); + if (pos != std::string::npos) { + debug("removing case hack suffix from '%s'", path + i.first); + name.erase(pos); + } + if (!unhacked.emplace(name, i.first).second) + throw Error("file name collision in between '%s' and '%s'", + (path + unhacked[name]), + (path + i.first)); + } else + unhacked.emplace(i.first, i.first); + + for (auto & i : unhacked) + if (filter((path + i.first).abs())) { + sink << "entry" << "(" << "name" << i.first << "node"; + dump(path + i.second); + sink << ")"; + } + } + + else if (st.type == tSymlink) + sink << "type" << "symlink" << "target" << readLink(path); + + else throw Error("file '%s' has an unsupported type", path); + + sink << ")"; + }; + + sink << narVersionMagic1; + dump(path); +} + +Hash SourceAccessor::hashPath( + const CanonPath & path, + PathFilter & filter, + HashType ht) +{ + HashSink sink(ht); + dumpPath(path, sink, filter); + return sink.finish().first; +} + +std::optional SourceAccessor::maybeLstat(const CanonPath & path) +{ + // FIXME: merge these into one operation. + if (!pathExists(path)) + return {}; + return lstat(path); +} + +std::string SourceAccessor::showPath(const CanonPath & path) +{ + return path.abs(); +} + +} diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh new file mode 100644 index 000000000..c2d35d6b5 --- /dev/null +++ b/src/libutil/source-accessor.hh @@ -0,0 +1,96 @@ +#pragma once + +#include "canon-path.hh" +#include "hash.hh" + +namespace nix { + +/** + * A read-only filesystem abstraction. This is used by the Nix + * evaluator and elsewhere for accessing sources in various + * filesystem-like entities (such as the real filesystem, tarballs or + * Git repositories). + */ +struct SourceAccessor +{ + const size_t number; + + SourceAccessor(); + + virtual ~SourceAccessor() + { } + + virtual std::string readFile(const CanonPath & path) = 0; + + virtual bool pathExists(const CanonPath & path) = 0; + + enum Type { + tRegular, tSymlink, tDirectory, + /** + Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things. + + Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`. + + Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types. + */ + tMisc + }; + + struct Stat + { + Type type = tMisc; + //uint64_t fileSize = 0; // regular files only + bool isExecutable = false; // regular files only + }; + + virtual Stat lstat(const CanonPath & path) = 0; + + std::optional maybeLstat(const CanonPath & path); + + typedef std::optional DirEntry; + + typedef std::map DirEntries; + + virtual DirEntries readDirectory(const CanonPath & path) = 0; + + virtual std::string readLink(const CanonPath & path) = 0; + + virtual void dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter = defaultPathFilter); + + Hash hashPath( + const CanonPath & path, + PathFilter & filter = defaultPathFilter, + HashType ht = htSHA256); + + /* Return a corresponding path in the root filesystem, if + possible. This is only possible for filesystems that are + materialized in the root filesystem. */ + virtual std::optional getPhysicalPath(const CanonPath & path) + { return std::nullopt; } + + bool operator == (const SourceAccessor & x) const + { + return number == x.number; + } + + bool operator < (const SourceAccessor & x) const + { + return number < x.number; + } + + void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); + + virtual std::string showPath(const CanonPath & path); + + /* Return the maximum last-modified time of the files in this + tree, if available. */ + virtual std::optional getLastModified() + { + return std::nullopt; + } +}; + +} From 9f572eb0e35d3017a160cef89b7417cf6b4a53e2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 15:07:56 +0200 Subject: [PATCH 422/909] Unify the two implementations of dumpPath() --- src/libutil/archive.cc | 183 +++++++++++++++++++-------------- src/libutil/source-accessor.cc | 72 ------------- 2 files changed, 108 insertions(+), 147 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 0f0cae7c0..e48fc7522 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -14,6 +14,7 @@ #include "archive.hh" #include "util.hh" #include "config.hh" +#include "source-accessor.hh" namespace nix { @@ -36,91 +37,134 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings); PathFilter defaultPathFilter = [](const Path &) { return true; }; -static void dumpContents(const Path & path, off_t size, - Sink & sink) +void SourceAccessor::dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter) { - sink << "contents" << size; + auto dumpContents = [&](const CanonPath & path) + { + /* It would be nice if this was streaming, but we need the + size before the contents. */ + auto s = readFile(path); + sink << "contents" << s.size(); + sink(s); + writePadding(s.size(), sink); + }; - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) throw SysError("opening file '%1%'", path); + std::function dump; - std::vector buf(65536); - size_t left = size; + dump = [&](const CanonPath & path) { + checkInterrupt(); - while (left > 0) { - auto n = std::min(left, buf.size()); - readFull(fd.get(), buf.data(), n); - left -= n; - sink({buf.data(), n}); - } + auto st = lstat(path); - writePadding(size, sink); + sink << "("; + + if (st.type == tRegular) { + sink << "type" << "regular"; + if (st.isExecutable) + sink << "executable" << ""; + dumpContents(path); + } + + else if (st.type == tDirectory) { + sink << "type" << "directory"; + + /* If we're on a case-insensitive system like macOS, undo + the case hack applied by restorePath(). */ + std::map unhacked; + for (auto & i : readDirectory(path)) + if (archiveSettings.useCaseHack) { + std::string name(i.first); + size_t pos = i.first.find(caseHackSuffix); + if (pos != std::string::npos) { + debug("removing case hack suffix from '%s'", path + i.first); + name.erase(pos); + } + if (!unhacked.emplace(name, i.first).second) + throw Error("file name collision in between '%s' and '%s'", + (path + unhacked[name]), + (path + i.first)); + } else + unhacked.emplace(i.first, i.first); + + for (auto & i : unhacked) + if (filter((path + i.first).abs())) { + sink << "entry" << "(" << "name" << i.first << "node"; + dump(path + i.second); + sink << ")"; + } + } + + else if (st.type == tSymlink) + sink << "type" << "symlink" << "target" << readLink(path); + + else throw Error("file '%s' has an unsupported type", path); + + sink << ")"; + }; + + sink << narVersionMagic1; + dump(path); } -static time_t dump(const Path & path, Sink & sink, PathFilter & filter) +struct FSSourceAccessor : SourceAccessor { - checkInterrupt(); + time_t mtime = 0; // most recent mtime seen - auto st = lstat(path); - time_t result = st.st_mtime; - - sink << "("; - - if (S_ISREG(st.st_mode)) { - sink << "type" << "regular"; - if (st.st_mode & S_IXUSR) - sink << "executable" << ""; - dumpContents(path, st.st_size, sink); + std::string readFile(const CanonPath & path) override + { + return nix::readFile(path.abs()); } - else if (S_ISDIR(st.st_mode)) { - sink << "type" << "directory"; + bool pathExists(const CanonPath & path) override + { + return nix::pathExists(path.abs()); + } - /* If we're on a case-insensitive system like macOS, undo - the case hack applied by restorePath(). */ - std::map unhacked; - for (auto & i : readDirectory(path)) - if (archiveSettings.useCaseHack) { - std::string name(i.name); - size_t pos = i.name.find(caseHackSuffix); - if (pos != std::string::npos) { - debug("removing case hack suffix from '%1%'", path + "/" + i.name); - name.erase(pos); - } - if (!unhacked.emplace(name, i.name).second) - throw Error("file name collision in between '%1%' and '%2%'", - (path + "/" + unhacked[name]), - (path + "/" + i.name)); - } else - unhacked.emplace(i.name, i.name); + Stat lstat(const CanonPath & path) override + { + auto st = nix::lstat(path.abs()); + mtime = std::max(mtime, st.st_mtime); + return Stat { + .type = + S_ISREG(st.st_mode) ? tRegular : + S_ISDIR(st.st_mode) ? tDirectory : + S_ISLNK(st.st_mode) ? tSymlink : + tMisc, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR + }; + } - for (auto & i : unhacked) - if (filter(path + "/" + i.first)) { - sink << "entry" << "(" << "name" << i.first << "node"; - auto tmp_mtime = dump(path + "/" + i.second, sink, filter); - if (tmp_mtime > result) { - result = tmp_mtime; - } - sink << ")"; + DirEntries readDirectory(const CanonPath & path) override + { + DirEntries res; + for (auto & entry : nix::readDirectory(path.abs())) { + std::optional type; + switch (entry.type) { + case DT_REG: type = Type::tRegular; break; + case DT_LNK: type = Type::tSymlink; break; + case DT_DIR: type = Type::tDirectory; break; } + res.emplace(entry.name, type); + } + return res; } - else if (S_ISLNK(st.st_mode)) - sink << "type" << "symlink" << "target" << readLink(path); - - else throw Error("file '%1%' has an unsupported type", path); - - sink << ")"; - - return result; -} + std::string readLink(const CanonPath & path) override + { + return nix::readLink(path.abs()); + } +}; time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter) { - sink << narVersionMagic1; - return dump(path, sink, filter); + FSSourceAccessor accessor; + accessor.dumpPath(CanonPath::fromCwd(path), sink, filter); + return accessor.mtime; } void dumpPath(const Path & path, Sink & sink, PathFilter & filter) @@ -141,17 +185,6 @@ static SerialisationError badArchive(const std::string & s) } -#if 0 -static void skipGeneric(Source & source) -{ - if (readString(source) == "(") { - while (readString(source) != ")") - skipGeneric(source); - } -} -#endif - - static void parseContents(ParseSink & sink, Source & source, const Path & path) { uint64_t size = readLongLong(source); diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index 23ec55c89..e3adee5f1 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -10,78 +10,6 @@ SourceAccessor::SourceAccessor() { } -// FIXME: merge with archive.cc. -void SourceAccessor::dumpPath( - const CanonPath & path, - Sink & sink, - PathFilter & filter) -{ - auto dumpContents = [&](const CanonPath & path) - { - // FIXME: pipe - auto s = readFile(path); - sink << "contents" << s.size(); - sink(s); - writePadding(s.size(), sink); - }; - - std::function dump; - - dump = [&](const CanonPath & path) { - checkInterrupt(); - - auto st = lstat(path); - - sink << "("; - - if (st.type == tRegular) { - sink << "type" << "regular"; - if (st.isExecutable) - sink << "executable" << ""; - dumpContents(path); - } - - else if (st.type == tDirectory) { - sink << "type" << "directory"; - - /* If we're on a case-insensitive system like macOS, undo - the case hack applied by restorePath(). */ - std::map unhacked; - for (auto & i : readDirectory(path)) - if (/* archiveSettings.useCaseHack */ false) { // FIXME - std::string name(i.first); - size_t pos = i.first.find(caseHackSuffix); - if (pos != std::string::npos) { - debug("removing case hack suffix from '%s'", path + i.first); - name.erase(pos); - } - if (!unhacked.emplace(name, i.first).second) - throw Error("file name collision in between '%s' and '%s'", - (path + unhacked[name]), - (path + i.first)); - } else - unhacked.emplace(i.first, i.first); - - for (auto & i : unhacked) - if (filter((path + i.first).abs())) { - sink << "entry" << "(" << "name" << i.first << "node"; - dump(path + i.second); - sink << ")"; - } - } - - else if (st.type == tSymlink) - sink << "type" << "symlink" << "target" << readLink(path); - - else throw Error("file '%s' has an unsupported type", path); - - sink << ")"; - }; - - sink << narVersionMagic1; - dump(path); -} - Hash SourceAccessor::hashPath( const CanonPath & path, PathFilter & filter, From 50156302c033323895ababb6c190086cbc5cec46 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 15:20:10 +0200 Subject: [PATCH 423/909] Deduplicate FSSourceAccessor and FSInputAccessor --- src/libfetchers/fs-input-accessor.cc | 31 +++++------------ src/libutil/archive.cc | 52 +--------------------------- src/libutil/source-accessor.cc | 49 ++++++++++++++++++++++++++ src/libutil/source-accessor.hh | 24 +++++++++++++ 4 files changed, 82 insertions(+), 74 deletions(-) diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index a35955465..e40faf03f 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -3,7 +3,7 @@ namespace nix { -struct FSInputAccessorImpl : FSInputAccessor +struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor { CanonPath root; std::optional> allowedPaths; @@ -23,28 +23,20 @@ struct FSInputAccessorImpl : FSInputAccessor { auto absPath = makeAbsPath(path); checkAllowed(absPath); - return nix::readFile(absPath.abs()); + return PosixSourceAccessor::readFile(absPath); } bool pathExists(const CanonPath & path) override { auto absPath = makeAbsPath(path); - return isAllowed(absPath) && nix::pathExists(absPath.abs()); + return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath); } Stat lstat(const CanonPath & path) override { auto absPath = makeAbsPath(path); checkAllowed(absPath); - auto st = nix::lstat(absPath.abs()); - return Stat { - .type = - S_ISREG(st.st_mode) ? tRegular : - S_ISDIR(st.st_mode) ? tDirectory : - S_ISLNK(st.st_mode) ? tSymlink : - tMisc, - .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR - }; + return PosixSourceAccessor::lstat(absPath); } DirEntries readDirectory(const CanonPath & path) override @@ -52,16 +44,9 @@ struct FSInputAccessorImpl : FSInputAccessor auto absPath = makeAbsPath(path); checkAllowed(absPath); DirEntries res; - for (auto & entry : nix::readDirectory(absPath.abs())) { - std::optional type; - switch (entry.type) { - case DT_REG: type = Type::tRegular; break; - case DT_LNK: type = Type::tSymlink; break; - case DT_DIR: type = Type::tDirectory; break; - } - if (isAllowed(absPath + entry.name)) - res.emplace(entry.name, type); - } + for (auto & entry : PosixSourceAccessor::readDirectory(absPath)) + if (isAllowed(absPath + entry.first)) + res.emplace(entry); return res; } @@ -69,7 +54,7 @@ struct FSInputAccessorImpl : FSInputAccessor { auto absPath = makeAbsPath(path); checkAllowed(absPath); - return nix::readLink(absPath.abs()); + return PosixSourceAccessor::readLink(absPath); } CanonPath makeAbsPath(const CanonPath & path) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index e48fc7522..6508ba807 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -110,59 +110,9 @@ void SourceAccessor::dumpPath( } -struct FSSourceAccessor : SourceAccessor -{ - time_t mtime = 0; // most recent mtime seen - - std::string readFile(const CanonPath & path) override - { - return nix::readFile(path.abs()); - } - - bool pathExists(const CanonPath & path) override - { - return nix::pathExists(path.abs()); - } - - Stat lstat(const CanonPath & path) override - { - auto st = nix::lstat(path.abs()); - mtime = std::max(mtime, st.st_mtime); - return Stat { - .type = - S_ISREG(st.st_mode) ? tRegular : - S_ISDIR(st.st_mode) ? tDirectory : - S_ISLNK(st.st_mode) ? tSymlink : - tMisc, - .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR - }; - } - - DirEntries readDirectory(const CanonPath & path) override - { - DirEntries res; - for (auto & entry : nix::readDirectory(path.abs())) { - std::optional type; - switch (entry.type) { - case DT_REG: type = Type::tRegular; break; - case DT_LNK: type = Type::tSymlink; break; - case DT_DIR: type = Type::tDirectory; break; - } - res.emplace(entry.name, type); - } - return res; - } - - std::string readLink(const CanonPath & path) override - { - return nix::readLink(path.abs()); - } -}; - - time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter) { - FSSourceAccessor accessor; + PosixSourceAccessor accessor; accessor.dumpPath(CanonPath::fromCwd(path), sink, filter); return accessor.mtime; } diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index e3adee5f1..d5c8cbcdd 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -33,4 +33,53 @@ std::string SourceAccessor::showPath(const CanonPath & path) return path.abs(); } +std::string PosixSourceAccessor::readFile(const CanonPath & path) +{ + return nix::readFile(path.abs()); +} + +bool PosixSourceAccessor::pathExists(const CanonPath & path) +{ + return nix::pathExists(path.abs()); +} + +SourceAccessor::Stat PosixSourceAccessor::lstat(const CanonPath & path) +{ + auto st = nix::lstat(path.abs()); + mtime = std::max(mtime, st.st_mtime); + return Stat { + .type = + S_ISREG(st.st_mode) ? tRegular : + S_ISDIR(st.st_mode) ? tDirectory : + S_ISLNK(st.st_mode) ? tSymlink : + tMisc, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR + }; +} + +SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path) +{ + DirEntries res; + for (auto & entry : nix::readDirectory(path.abs())) { + std::optional type; + switch (entry.type) { + case DT_REG: type = Type::tRegular; break; + case DT_LNK: type = Type::tSymlink; break; + case DT_DIR: type = Type::tDirectory; break; + } + res.emplace(entry.name, type); + } + return res; +} + +std::string PosixSourceAccessor::readLink(const CanonPath & path) +{ + return nix::readLink(path.abs()); +} + +std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) +{ + return path; +} + } diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index c2d35d6b5..a251ae31c 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -93,4 +93,28 @@ struct SourceAccessor } }; +/** + * A source accessor that uses the Unix filesystem. + */ +struct PosixSourceAccessor : SourceAccessor +{ + /** + * The most recent mtime seen by lstat(). This is a hack to + * support dumpPathAndGetMtime(). Should remove this eventually. + */ + time_t mtime = 0; + + std::string readFile(const CanonPath & path) override; + + bool pathExists(const CanonPath & path) override; + + Stat lstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::optional getPhysicalPath(const CanonPath & path) override; +}; + } From 5be7705ddf9397846ca9391d464af1cc27110f06 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 19:20:21 +0200 Subject: [PATCH 424/909] Remove stuff we don't need yet --- src/libexpr/eval.cc | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7180b9fbe..8235b83c5 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -505,18 +505,7 @@ EvalState::EvalState( , sOutputSpecified(symbols.create("outputSpecified")) , repair(NoRepair) , emptyBindings(0) - , rootFS( - makeFSInputAccessor( - CanonPath::root, - evalSettings.restrictEval || evalSettings.pureEval - ? std::optional>(std::set()) - : std::nullopt, - [](const CanonPath & path) -> RestrictedPathError { - auto modeInformation = evalSettings.pureEval - ? "in pure evaluation mode (use '--impure' to override)" - : "in restricted mode"; - throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); - })) + , rootFS(makeFSInputAccessor(CanonPath::root)) , corepkgsFS(makeMemoryInputAccessor()) , internalFS(makeMemoryInputAccessor()) , derivationInternal{corepkgsFS->addFile( @@ -541,9 +530,6 @@ EvalState::EvalState( , baseEnv(allocEnv(128)) , staticBaseEnv{std::make_shared(false, nullptr)} { - // For now, don't rely on FSInputAccessor for access control. - rootFS->allowPath(CanonPath::root); - countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; assert(gcInitialised); From 87c4f4a972ce732962cc0881d848d8bf979893c2 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Thu, 19 Oct 2023 18:24:13 +0100 Subject: [PATCH 425/909] libutil: Move some non-template implememntations from config.hh to config.cc --- src/libutil/config.cc | 45 +++++++++++++++++++++++++++++++++++++++++++ src/libutil/config.hh | 31 +++++++---------------------- 2 files changed, 52 insertions(+), 24 deletions(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8672edaa8..f5c7fb7d5 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -9,6 +9,10 @@ namespace nix { +Config::Config(StringMap initials) + : AbstractConfig(std::move(initials)) +{ } + bool Config::set(const std::string & name, const std::string & value) { bool append = false; @@ -59,6 +63,10 @@ void Config::addSetting(AbstractSetting * setting) } } +AbstractConfig::AbstractConfig(StringMap initials) + : unknownSettings(std::move(initials)) +{ } + void AbstractConfig::warnUnknownSettings() { for (auto & s : unknownSettings) @@ -199,6 +207,13 @@ AbstractSetting::AbstractSetting( { } +AbstractSetting::~AbstractSetting() +{ + // Check against a gcc miscompilation causing our constructor + // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431). + assert(created == 123); +} + nlohmann::json AbstractSetting::toJSON() { return nlohmann::json(toJSONObject()); @@ -220,6 +235,9 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category) { } + +bool AbstractSetting::isOverridden() const { return overridden; } + template<> std::string BaseSetting::parse(const std::string & str) const { return str; @@ -385,11 +403,33 @@ static Path parsePath(const AbstractSetting & s, const std::string & str) return canonPath(str); } +PathSetting::PathSetting(Config * options, + const Path & def, + const std::string & name, + const std::string & description, + const std::set & aliases) + : BaseSetting(def, true, name, description, aliases) +{ + options->addSetting(this); +} + Path PathSetting::parse(const std::string & str) const { return parsePath(*this, str); } + +OptionalPathSetting::OptionalPathSetting(Config * options, + const std::optional & def, + const std::string & name, + const std::string & description, + const std::set & aliases) + : BaseSetting>(def, true, name, description, aliases) +{ + options->addSetting(this); +} + + std::optional OptionalPathSetting::parse(const std::string & str) const { if (str == "") @@ -398,6 +438,11 @@ std::optional OptionalPathSetting::parse(const std::string & str) const return parsePath(*this, str); } +void OptionalPathSetting::operator =(const std::optional & v) +{ + this->assign(v); +} + bool GlobalConfig::set(const std::string & name, const std::string & value) { for (auto & config : *configRegistrations) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 96c2cd75d..47c7be473 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -52,9 +52,7 @@ class AbstractConfig protected: StringMap unknownSettings; - AbstractConfig(const StringMap & initials = {}) - : unknownSettings(initials) - { } + AbstractConfig(StringMap initials = {}); public: @@ -163,9 +161,7 @@ private: public: - Config(const StringMap & initials = {}) - : AbstractConfig(initials) - { } + Config(StringMap initials = {}); bool set(const std::string & name, const std::string & value) override; @@ -206,12 +202,7 @@ protected: const std::set & aliases, std::optional experimentalFeature = std::nullopt); - virtual ~AbstractSetting() - { - // Check against a gcc miscompilation causing our constructor - // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431). - assert(created == 123); - } + virtual ~AbstractSetting(); virtual void set(const std::string & value, bool append = false) = 0; @@ -229,7 +220,7 @@ protected: virtual void convertToArg(Args & args, const std::string & category); - bool isOverridden() const { return overridden; } + bool isOverridden() const; }; /** @@ -365,11 +356,7 @@ public: const Path & def, const std::string & name, const std::string & description, - const std::set & aliases = {}) - : BaseSetting(def, true, name, description, aliases) - { - options->addSetting(this); - } + const std::set & aliases = {}); Path parse(const std::string & str) const override; @@ -391,15 +378,11 @@ public: const std::optional & def, const std::string & name, const std::string & description, - const std::set & aliases = {}) - : BaseSetting>(def, true, name, description, aliases) - { - options->addSetting(this); - } + const std::set & aliases = {}); std::optional parse(const std::string & str) const override; - void operator =(const std::optional & v) { this->assign(v); } + void operator =(const std::optional & v); }; struct GlobalConfig : public AbstractConfig From 55f06b6f30fa212d48cfd6d121a845863fb2980b Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Thu, 19 Oct 2023 18:24:54 +0100 Subject: [PATCH 426/909] libutil: Remove non-needed constructor --- src/libutil/config.cc | 4 ++-- src/libutil/config.hh | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index f5c7fb7d5..8e06273ee 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -33,9 +33,9 @@ bool Config::set(const std::string & name, const std::string & value) void Config::addSetting(AbstractSetting * setting) { - _settings.emplace(setting->name, Config::SettingData(false, setting)); + _settings.emplace(setting->name, Config::SettingData{false, setting}); for (auto & alias : setting->aliases) - _settings.emplace(alias, Config::SettingData(true, setting)); + _settings.emplace(alias, Config::SettingData{true, setting}); bool set = false; diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 47c7be473..f09a9b0b8 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -148,9 +148,6 @@ public: { bool isAlias; AbstractSetting * setting; - SettingData(bool isAlias, AbstractSetting * setting) - : isAlias(isAlias), setting(setting) - { } }; typedef std::map Settings; From b0f4ac29d3c0534b91fff7b37191d4d7d5a96395 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Thu, 19 Oct 2023 18:25:17 +0100 Subject: [PATCH 427/909] libutil: Use c++ style cast --- src/libutil/config.hh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index f09a9b0b8..38c3ce0c4 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -312,8 +312,7 @@ public: template std::ostream & operator <<(std::ostream & str, const BaseSetting & opt) { - str << (const T &) opt; - return str; + return str << static_cast(opt); } template From 42f26eb42e3b115f39570bb14445d7e5f33220d7 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Fri, 20 Oct 2023 02:45:47 +0200 Subject: [PATCH 428/909] doc: complexity for '?' operator (#9184) Co-authored-by: Valentin Gagarin Co-authored-by: Robert Hensing --- doc/manual/src/language/operators.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index a22c71109..07b43a881 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -63,6 +63,8 @@ The result is a [Boolean] value. [Has attribute]: #has-attribute +After evaluating *attrset* and *attrpath*, the computational complexity is O(log(*n*)) for *n* attributes in the *attrset* + ## Arithmetic Numbers are type-compatible: From e58566a057692bbfbd30a6639248adfcf81df108 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Fri, 20 Oct 2023 05:11:03 +0200 Subject: [PATCH 429/909] doc: add reference to hasAttr in `?` operator (#9185) Co-authored-by: Valentin Gagarin --- doc/manual/src/language/operators.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 07b43a881..cc825b4cf 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -59,6 +59,8 @@ An attribute path is a dot-separated list of [attribute names](./values.md#attri Test whether [attribute set] *attrset* contains the attribute denoted by *attrpath*. The result is a [Boolean] value. +See also: [`builtins.hasAttr`](@docroot@/language/builtins.md#builtins-hasAttr) + [Boolean]: ./values.md#type-boolean [Has attribute]: #has-attribute From 9277eb276bf0a942e88fcf499f6a6b9c262be853 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 19 Oct 2023 16:59:35 +0200 Subject: [PATCH 430/909] libstore: Add apple-virt to system features when available I'm sure that we'll adjust the implementation over time, but this at least discerns between an apple silicon bare metal machine and a tart VM. --- doc/manual/src/release-notes/rl-next.md | 2 ++ src/libstore/globals.cc | 31 +++++++++++++++++++++++++ src/libstore/globals.hh | 4 ++++ 3 files changed, 37 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 46ce2abac..d0b516dfb 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -8,6 +8,8 @@ - Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.` or `legacyPackages.`. +- Nix adds `apple-virt` to the default system features on macOS systems that support virtualization. This is similar to what's done for the `kvm` system feature on Linux hosts. + - Introduce a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash). - `builtins.fetchTree` is now marked as stable. diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 5a4cb1824..9c25d9868 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -24,6 +24,9 @@ #include "config-impl.hh" +#ifdef __APPLE__ +#include +#endif namespace nix { @@ -154,6 +157,29 @@ unsigned int Settings::getDefaultCores() return concurrency; } +#if __APPLE__ +static bool hasVirt() { + + int hasVMM; + int hvSupport; + size_t size; + + size = sizeof(hasVMM); + if (sysctlbyname("kern.hv_vmm_present", &hasVMM, &size, NULL, 0) == 0) { + if (hasVMM) + return false; + } + + // whether the kernel and hardware supports virt + size = sizeof(hvSupport); + if (sysctlbyname("kern.hv_support", &hvSupport, &size, NULL, 0) == 0) { + return hvSupport == 1; + } else { + return false; + } +} +#endif + StringSet Settings::getDefaultSystemFeatures() { /* For backwards compatibility, accept some "features" that are @@ -170,6 +196,11 @@ StringSet Settings::getDefaultSystemFeatures() features.insert("kvm"); #endif + #if __APPLE__ + if (hasVirt()) + features.insert("apple-virt"); + #endif + return features; } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 5ec332417..ff7d3df1e 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -714,6 +714,10 @@ public: System features are user-defined, but Nix sets the following defaults: + - `apple-virt` + + Included on darwin if virtualization is available. + - `kvm` Included by default if `/dev/kvm` is accessible. From bb645c5d02025158c8dbb0c2a60e7cba8251f062 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 20 Oct 2023 10:19:07 +0200 Subject: [PATCH 431/909] system-features doc: kvm is Linux-only --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ff7d3df1e..e90f70f5f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -720,7 +720,7 @@ public: - `kvm` - Included by default if `/dev/kvm` is accessible. + Included on Linux if `/dev/kvm` is accessible. - `nixos-test`, `benchmark`, `big-parallel` From df10dc630fd13082e13400ec46acefe4ad85a715 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 12:36:18 +0200 Subject: [PATCH 432/909] Doxygen Co-authored-by: John Ericson --- src/libutil/source-accessor.hh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index a251ae31c..4074b2e63 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -65,9 +65,11 @@ struct SourceAccessor PathFilter & filter = defaultPathFilter, HashType ht = htSHA256); - /* Return a corresponding path in the root filesystem, if - possible. This is only possible for filesystems that are - materialized in the root filesystem. */ + /** + * Return a corresponding path in the root filesystem, if + * possible. This is only possible for filesystems that are + * materialized in the root filesystem. + */ virtual std::optional getPhysicalPath(const CanonPath & path) { return std::nullopt; } From bacceaea919ddb26d1882fec16462cdcb4e1d740 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 12:40:46 +0200 Subject: [PATCH 433/909] Move getLastModified(), remove setPathDisplay() --- src/libfetchers/input-accessor.hh | 9 +++++++++ src/libutil/source-accessor.hh | 9 --------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 524a94cd1..5dc05a363 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -16,6 +16,15 @@ class Store; struct InputAccessor : SourceAccessor, std::enable_shared_from_this { + /** + * Return the maximum last-modified time of the files in this + * tree, if available. + */ + virtual std::optional getLastModified() + { + return std::nullopt; + } + StorePath fetchToStore( ref store, const CanonPath & path, diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index 4074b2e63..53408eb6c 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -83,16 +83,7 @@ struct SourceAccessor return number < x.number; } - void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); - virtual std::string showPath(const CanonPath & path); - - /* Return the maximum last-modified time of the files in this - tree, if available. */ - virtual std::optional getLastModified() - { - return std::nullopt; - } }; /** From 173abec0bce03016b001c415822793a309c40e0b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 13:04:39 +0200 Subject: [PATCH 434/909] coerceToPath(): Handle __toString, add tests --- src/libexpr/eval.cc | 27 ++++++++++--------- src/libexpr/tests/error_traces.cc | 8 +++--- .../functional/lang/eval-okay-pathexists.nix | 2 ++ 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 8235b83c5..e3ecb987c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2335,29 +2335,32 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext { try { forceValue(v, pos); - - if (v.type() == nString) { - copyContext(v, context); - auto s = v.string_view(); - if (!hasPrefix(s, "/")) - error("string '%s' doesn't represent an absolute path", s).atPos(pos).debugThrow(); - return rootPath(CanonPath(s)); - } } catch (Error & e) { e.addTrace(positions[pos], errorCtx); throw; } + /* Handle path values directly, without coercing to a string. */ if (v.type() == nPath) return v.path(); + /* Similarly, handle __toString where the result may be a path + value. */ if (v.type() == nAttrs) { - auto i = v.attrs->find(sOutPath); - if (i != v.attrs->end()) - return coerceToPath(pos, *i->value, context, errorCtx); + auto i = v.attrs->find(sToString); + if (i != v.attrs->end()) { + Value v1; + callFunction(*i->value, v, v1, pos); + return coerceToPath(pos, v1, context, errorCtx); + } } - error("cannot coerce %1% to a path", showType(v)).withTrace(pos, errorCtx).debugThrow(); + /* Any other value should be coercable to a string, interpreted + relative to the root filesystem. */ + auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); + if (path == "" || path[0] != '/') + error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow(); + return rootPath(CanonPath(path)); } diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc index 6da8a5954..9cd9f0a60 100644 --- a/src/libexpr/tests/error_traces.cc +++ b/src/libexpr/tests/error_traces.cc @@ -295,7 +295,7 @@ namespace nix { TEST_F(ErrorTraceTest, toPath) { ASSERT_TRACE2("toPath []", TypeError, - hintfmt("cannot coerce %s to a path", "a list"), + hintfmt("cannot coerce %s to a string", "a list"), hintfmt("while evaluating the first argument passed to builtins.toPath")); ASSERT_TRACE2("toPath \"foo\"", @@ -309,7 +309,7 @@ namespace nix { TEST_F(ErrorTraceTest, storePath) { ASSERT_TRACE2("storePath true", TypeError, - hintfmt("cannot coerce %s to a path", "a Boolean"), + hintfmt("cannot coerce %s to a string", "a Boolean"), hintfmt("while evaluating the first argument passed to 'builtins.storePath'")); } @@ -318,7 +318,7 @@ namespace nix { TEST_F(ErrorTraceTest, pathExists) { ASSERT_TRACE2("pathExists []", TypeError, - hintfmt("cannot coerce %s to a path", "a list"), + hintfmt("cannot coerce %s to a string", "a list"), hintfmt("while realising the context of a path")); ASSERT_TRACE2("pathExists \"zorglub\"", @@ -377,7 +377,7 @@ namespace nix { TEST_F(ErrorTraceTest, filterSource) { ASSERT_TRACE2("filterSource [] []", TypeError, - hintfmt("cannot coerce %s to a path", "a list"), + hintfmt("cannot coerce %s to a string", "a list"), hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] \"foo\"", diff --git a/tests/functional/lang/eval-okay-pathexists.nix b/tests/functional/lang/eval-okay-pathexists.nix index c5e7a62de..31697f66a 100644 --- a/tests/functional/lang/eval-okay-pathexists.nix +++ b/tests/functional/lang/eval-okay-pathexists.nix @@ -25,5 +25,7 @@ builtins.pathExists (./lib.nix) && builtins.pathExists (builtins.toString ./. + "/../lang/..//") && builtins.pathExists (builtins.toPath (builtins.toString ./lib.nix)) && !builtins.pathExists (builtins.toPath (builtins.toString ./bla.nix)) +&& builtins.pathExists (builtins.toPath { __toString = x: builtins.toString ./lib.nix; }) +&& builtins.pathExists (builtins.toPath { outPath = builtins.toString ./lib.nix; }) && builtins.pathExists ./lib.nix && !builtins.pathExists ./bla.nix From 7a086a32bce3338bdc1ce669a51599a99bc17b9f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 13:32:15 +0200 Subject: [PATCH 435/909] fetchToStore(): Handle flat ingestion method and add test --- src/libfetchers/input-accessor.cc | 5 ++++- tests/functional/lang/eval-okay-path.exp | 2 +- tests/functional/lang/eval-okay-path.nix | 22 +++++++++++++++------- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index 9bddf7c2e..488350849 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -14,7 +14,10 @@ StorePath InputAccessor::fetchToStore( Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path))); auto source = sinkToSource([&](Sink & sink) { - dumpPath(path, sink, filter ? *filter : defaultPathFilter); + if (method == FileIngestionMethod::Recursive) + dumpPath(path, sink, filter ? *filter : defaultPathFilter); + else + sink(readFile(path)); // FIXME: stream }); auto storePath = diff --git a/tests/functional/lang/eval-okay-path.exp b/tests/functional/lang/eval-okay-path.exp index 3ce7f8283..635e2243a 100644 --- a/tests/functional/lang/eval-okay-path.exp +++ b/tests/functional/lang/eval-okay-path.exp @@ -1 +1 @@ -"/nix/store/ya937r4ydw0l6kayq8jkyqaips9c75jm-output" +[ "/nix/store/ya937r4ydw0l6kayq8jkyqaips9c75jm-output" "/nix/store/m7y372g6jb0g4hh1dzmj847rd356fhnz-output" ] diff --git a/tests/functional/lang/eval-okay-path.nix b/tests/functional/lang/eval-okay-path.nix index e67168cf3..599b33541 100644 --- a/tests/functional/lang/eval-okay-path.nix +++ b/tests/functional/lang/eval-okay-path.nix @@ -1,7 +1,15 @@ -builtins.path - { path = ./.; - filter = path: _: baseNameOf path == "data"; - recursive = true; - sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw"; - name = "output"; - } +[ + (builtins.path + { path = ./.; + filter = path: _: baseNameOf path == "data"; + recursive = true; + sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw"; + name = "output"; + }) + (builtins.path + { path = ./data; + recursive = false; + sha256 = "0k4lwj58f2w5yh92ilrwy9917pycipbrdrr13vbb3yd02j09vfxm"; + name = "output"; + }) +] From 57db3be9e448042814d386def2d8af16a2a4c4b9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 16:36:41 +0200 Subject: [PATCH 436/909] SourceAccessor::readFile(): Support reading into a sink --- src/libfetchers/fs-input-accessor.cc | 7 +++- src/libfetchers/input-accessor.cc | 2 +- src/libutil/archive.cc | 15 ++++--- src/libutil/source-accessor.cc | 58 +++++++++++++++++++++++++++- src/libutil/source-accessor.hh | 25 +++++++++++- 5 files changed, 94 insertions(+), 13 deletions(-) diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index e40faf03f..3444c4643 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -19,11 +19,14 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor { } - std::string readFile(const CanonPath & path) override + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override { auto absPath = makeAbsPath(path); checkAllowed(absPath); - return PosixSourceAccessor::readFile(absPath); + PosixSourceAccessor::readFile(absPath, sink, sizeCallback); } bool pathExists(const CanonPath & path) override diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index 488350849..d1d450cf7 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -17,7 +17,7 @@ StorePath InputAccessor::fetchToStore( if (method == FileIngestionMethod::Recursive) dumpPath(path, sink, filter ? *filter : defaultPathFilter); else - sink(readFile(path)); // FIXME: stream + readFile(path, sink); }); auto storePath = diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 6508ba807..0cd54e5db 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -44,12 +44,15 @@ void SourceAccessor::dumpPath( { auto dumpContents = [&](const CanonPath & path) { - /* It would be nice if this was streaming, but we need the - size before the contents. */ - auto s = readFile(path); - sink << "contents" << s.size(); - sink(s); - writePadding(s.size(), sink); + sink << "contents"; + std::optional size; + readFile(path, sink, [&](uint64_t _size) + { + size = _size; + sink << _size; + }); + assert(size); + writePadding(*size, sink); }; std::function dump; diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index d5c8cbcdd..2d03d3d7a 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -10,6 +10,28 @@ SourceAccessor::SourceAccessor() { } +std::string SourceAccessor::readFile(const CanonPath & path) +{ + StringSink sink; + std::optional size; + readFile(path, sink, [&](uint64_t _size) + { + size = _size; + }); + assert(size && *size == sink.s.size()); + return std::move(sink.s); +} + +void SourceAccessor::readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) +{ + auto s = readFile(path); + sizeCallback(s.size()); + sink(s); +} + Hash SourceAccessor::hashPath( const CanonPath & path, PathFilter & filter, @@ -33,9 +55,41 @@ std::string SourceAccessor::showPath(const CanonPath & path) return path.abs(); } -std::string PosixSourceAccessor::readFile(const CanonPath & path) +void PosixSourceAccessor::readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) { - return nix::readFile(path.abs()); + // FIXME: add O_NOFOLLOW since symlinks should be resolved by the + // caller? + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%1%'", path); + + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("statting file"); + + sizeCallback(st.st_size); + + off_t left = st.st_size; + + std::vector buf(64 * 1024); + while (left) { + checkInterrupt(); + ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size())); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading from file '%s'", showPath(path)); + } + else if (rd == 0) + throw SysError("unexpected end-of-file reading '%s'", showPath(path)); + else { + assert(rd <= left); + sink({(char *) buf.data(), (size_t) rd}); + left -= rd; + } + } } bool PosixSourceAccessor::pathExists(const CanonPath & path) diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index 53408eb6c..f3504c9bb 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -5,6 +5,8 @@ namespace nix { +struct Sink; + /** * A read-only filesystem abstraction. This is used by the Nix * evaluator and elsewhere for accessing sources in various @@ -20,7 +22,23 @@ struct SourceAccessor virtual ~SourceAccessor() { } - virtual std::string readFile(const CanonPath & path) = 0; + /** + * Return the contents of a file as a string. + */ + virtual std::string readFile(const CanonPath & path); + + /** + * Write the contents of a file as a sink. `sizeCallback` must be + * called with the size of the file before any data is written to + * the sink. + * + * Note: subclasses of `SourceAccessor` need to implement at least + * one of the `readFile()` variants. + */ + virtual void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback = [](uint64_t size){}); virtual bool pathExists(const CanonPath & path) = 0; @@ -97,7 +115,10 @@ struct PosixSourceAccessor : SourceAccessor */ time_t mtime = 0; - std::string readFile(const CanonPath & path) override; + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override; bool pathExists(const CanonPath & path) override; From bcf5c31950a01d73c17cee301d3dd363c91cd23d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 16:58:33 +0200 Subject: [PATCH 437/909] Add future FIXME --- src/libexpr/eval.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e3ecb987c..d26cde423 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2461,6 +2461,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v case nPath: return + // FIXME: compare accessors by their fingerprint. v1._path.accessor == v2._path.accessor && strcmp(v1._path.path, v2._path.path) == 0; From af302267e5e02c8b373779fac979f20e094e7cfa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 17:18:42 +0200 Subject: [PATCH 438/909] Input::hasAllInfo(): Remove --- src/libfetchers/fetchers.cc | 9 +-------- src/libfetchers/fetchers.hh | 11 ----------- src/libfetchers/git.cc | 9 --------- src/libfetchers/github.cc | 5 ----- src/libfetchers/indirect.cc | 5 ----- src/libfetchers/mercurial.cc | 7 ------- src/libfetchers/path.cc | 5 ----- src/libfetchers/tarball.cc | 6 ------ 8 files changed, 1 insertion(+), 56 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index c54c39cf0..000609f09 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -89,11 +89,6 @@ Attrs Input::toAttrs() const return attrs; } -bool Input::hasAllInfo() const -{ - return getNarHash() && scheme && scheme->hasAllInfo(*this); -} - bool Input::operator ==(const Input & other) const { return attrs == other.attrs; @@ -117,7 +112,7 @@ std::pair Input::fetch(ref store) const /* The tree may already be in the Nix store, or it could be substituted (which is often faster than fetching from the original source). So check that. */ - if (hasAllInfo()) { + if (getNarHash()) { try { auto storePath = computeStorePath(*store); @@ -175,8 +170,6 @@ std::pair Input::fetch(ref store) const input.locked = true; - assert(input.hasAllInfo()); - return {std::move(tree), input}; } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index b55aabb6f..f52175e41 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -79,15 +79,6 @@ public: */ bool isLocked() const { return locked; } - /** - * Check whether the input carries all necessary info required - * for cache insertion and substitution. - * These fields are used to uniquely identify cached trees - * within the "tarball TTL" window without necessarily - * indicating that the input's origin is unchanged. - */ - bool hasAllInfo() const; - bool operator ==(const Input & other) const; bool contains(const Input & other) const; @@ -144,8 +135,6 @@ struct InputScheme virtual ParsedURL toURL(const Input & input) const; - virtual bool hasAllInfo(const Input & input) const = 0; - virtual Input applyOverrides( const Input & input, std::optional ref, diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index c1a6dce43..26b8987d6 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -321,15 +321,6 @@ struct GitInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) const override - { - bool maybeDirty = !input.getRef(); - bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); - return - maybeGetIntAttr(input.attrs, "lastModified") - && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount")); - } - Input applyOverrides( const Input & input, std::optional ref, diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index d7450defe..b824140a6 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -132,11 +132,6 @@ struct GitArchiveInputScheme : InputScheme }; } - bool hasAllInfo(const Input & input) const override - { - return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified"); - } - Input applyOverrides( const Input & _input, std::optional ref, diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index e09d8bfb8..947849802 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -78,11 +78,6 @@ struct IndirectInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) const override - { - return false; - } - Input applyOverrides( const Input & _input, std::optional ref, diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index b0d2d1909..f830a3271 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -98,13 +98,6 @@ struct MercurialInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) const override - { - // FIXME: ugly, need to distinguish between dirty and clean - // default trees. - return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount"); - } - Input applyOverrides( const Input & input, std::optional ref, diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 092975f5d..d829609b5 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -66,11 +66,6 @@ struct PathInputScheme : InputScheme }; } - bool hasAllInfo(const Input & input) const override - { - return true; - } - std::optional getSourcePath(const Input & input) override { return getStrAttr(input.attrs, "path"); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 269a56526..ba47d59a7 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -253,12 +253,6 @@ struct CurlInputScheme : InputScheme url.query.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true)); return url; } - - bool hasAllInfo(const Input & input) const override - { - return true; - } - }; struct FileInputScheme : CurlInputScheme From 3e6b9f9357e4573740a4d9e477c4f4fa69c74fa5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 16 Oct 2023 01:17:33 -0400 Subject: [PATCH 439/909] Remove `prevInfos` as its dead code It is unused since 8e0946e8df968391d1430af8377bdb51204e4666 removed support for the repeat and enforce-determinism options. --- src/libstore/build/local-derivation-goal.hh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 0a05081c7..8191af7a6 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -120,14 +120,6 @@ struct LocalDerivationGoal : public DerivationGoal */ OutputPathMap scratchOutputs; - /** - * Path registration info from the previous round, if we're - * building multiple times. Since this contains the hash, it - * allows us to compare whether two rounds produced the same - * result. - */ - std::map prevInfos; - uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } From 862d16436b7606bba9064e2bf2a4c48bd883bb42 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 16 Oct 2023 01:22:34 -0400 Subject: [PATCH 440/909] Remove the `ValidPathInfo` `==` operator It is dead code. It was added in 8e0946e8df968391d1430af8377bdb51204e4666 as part of the repeated / enforce-determinism feature, but that was removed in 8fdd156a650f9b2ce9ae8cd74edcf16225478292. It is not good because it skips many fields. For testing purposes we will soon want to add a new one that doesn't skip fields, but we want to make sure making == sensitive to those fields won't change how Nix works. Proving in this commit that the old version is dead code achieves that. --- src/libstore/path-info.hh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 221523622..fea6d0e5f 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -72,14 +72,6 @@ struct ValidPathInfo */ std::optional ca; - bool operator == (const ValidPathInfo & i) const - { - return - path == i.path - && narHash == i.narHash - && references == i.references; - } - /** * Return a fingerprint of the store path to be used in binary * cache signatures. It contains the store path, the base-32 From 0f7e9d051340531332b7c6854675006cc8a16f50 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 19:13:35 +0200 Subject: [PATCH 441/909] Input: Remove 'direct' field --- src/libfetchers/fetchers.cc | 5 +++++ src/libfetchers/fetchers.hh | 6 ++++-- src/libfetchers/indirect.cc | 5 +++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 000609f09..5f6167868 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -84,6 +84,11 @@ std::string Input::to_string() const return toURL().to_string(); } +bool Input::isDirect() const +{ + return !scheme || scheme->isDirect(*this); +} + Attrs Input::toAttrs() const { return attrs; diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index f52175e41..4119dd17b 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -35,7 +35,6 @@ struct Input std::shared_ptr scheme; // note: can be null Attrs attrs; bool locked = false; - bool direct = true; /** * path of the parent of this input, used for relative path resolution @@ -71,7 +70,7 @@ public: * Check whether this is a "direct" input, that is, not * one that goes through a registry. */ - bool isDirect() const { return direct; } + bool isDirect() const; /** * Check whether this is a "locked" input, that is, @@ -152,6 +151,9 @@ struct InputScheme * Is this `InputScheme` part of an experimental feature? */ virtual std::optional experimentalFeature(); + + virtual bool isDirect(const Input & input) const + { return true; } }; void registerInputScheme(std::shared_ptr && fetcher); diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 947849802..9a71df3d4 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -41,7 +41,6 @@ struct IndirectInputScheme : InputScheme // FIXME: forbid query params? Input input; - input.direct = false; input.attrs.insert_or_assign("type", "indirect"); input.attrs.insert_or_assign("id", id); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); @@ -63,7 +62,6 @@ struct IndirectInputScheme : InputScheme throw BadURL("'%s' is not a valid flake ID", id); Input input; - input.direct = false; input.attrs = attrs; return input; } @@ -98,6 +96,9 @@ struct IndirectInputScheme : InputScheme { return Xp::Flakes; } + + bool isDirect(const Input & input) const override + { return false; } }; static auto rIndirectInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); From 85e5ac403fa9adb82105664bb6c4df501d4a0a25 Mon Sep 17 00:00:00 2001 From: Arthur Gautier Date: Fri, 20 Oct 2023 10:28:26 -0700 Subject: [PATCH 442/909] docker: publish images to ghcr.io (#8066) * docker: publish images to ghcr.io docker.com announced their intention to remove the free plan used by OSS. The nixos/nix image is essential to various CI runs to build with nix. To provide a continuity plan, this commit pushes the image to ghcr.io as well. Co-authored-by: Sandro --- .github/workflows/ci.yml | 23 +++++++++++++++++++ .../src/installation/installing-docker.md | 8 +++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c9c24dad..afe4dc2e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,6 +101,9 @@ jobs: docker_push_image: needs: [check_secrets, tests] + permissions: + contents: read + packages: write if: >- github.event_name == 'push' && github.ref_name == 'master' && @@ -126,6 +129,9 @@ jobs: - run: docker load -i ./result/image.tar.gz - run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION - run: docker tag nix:$NIX_VERSION nixos/nix:master + # We'll deploy the newly built image to both Docker Hub and Github Container Registry. + # + # Push to Docker Hub first - name: Login to Docker Hub uses: docker/login-action@v3 with: @@ -133,3 +139,20 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - run: docker push nixos/nix:$NIX_VERSION - run: docker push nixos/nix:master + # Push to GitHub Container Registry as well + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Push image + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/nix + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + + docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION + docker tag nix:$NIX_VERSION $IMAGE_ID:master + docker push $IMAGE_ID:$NIX_VERSION + docker push $IMAGE_ID:master diff --git a/doc/manual/src/installation/installing-docker.md b/doc/manual/src/installation/installing-docker.md index 9d6d8f2d9..6f77d6a57 100644 --- a/doc/manual/src/installation/installing-docker.md +++ b/doc/manual/src/installation/installing-docker.md @@ -3,14 +3,14 @@ To run the latest stable release of Nix with Docker run the following command: ```console -$ docker run -ti nixos/nix -Unable to find image 'nixos/nix:latest' locally -latest: Pulling from nixos/nix +$ docker run -ti ghcr.io/nixos/nix +Unable to find image 'ghcr.io/nixos/nix:latest' locally +latest: Pulling from ghcr.io/nixos/nix 5843afab3874: Pull complete b52bf13f109c: Pull complete 1e2415612aa3: Pull complete Digest: sha256:27f6e7f60227e959ee7ece361f75d4844a40e1cc6878b6868fe30140420031ff -Status: Downloaded newer image for nixos/nix:latest +Status: Downloaded newer image for ghcr.io/nixos/nix:latest 35ca4ada6e96:/# nix --version nix (Nix) 2.3.12 35ca4ada6e96:/# exit From 935c9981de03248fd5687c5d0363f572af723144 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 19:50:21 +0200 Subject: [PATCH 443/909] Remove fetchers::Tree and move tarball-related stuff into its own header --- src/libcmd/common-eval-args.cc | 5 ++-- src/libexpr/flake/flake.cc | 34 ++++++++++----------- src/libexpr/flake/flake.hh | 6 ++-- src/libexpr/flake/flakeref.cc | 6 ++-- src/libexpr/flake/flakeref.hh | 2 +- src/libexpr/parser.y | 6 ++-- src/libexpr/primops/fetchMercurial.cc | 6 ++-- src/libexpr/primops/fetchTree.cc | 13 ++++---- src/libfetchers/fetchers.cc | 18 +++++------ src/libfetchers/fetchers.hh | 41 ++----------------------- src/libfetchers/github.cc | 5 ++-- src/libfetchers/registry.cc | 2 +- src/libfetchers/tarball.cc | 7 +++-- src/libfetchers/tarball.hh | 43 +++++++++++++++++++++++++++ src/nix-channel/nix-channel.cc | 2 +- src/nix/flake.cc | 18 +++++------ 16 files changed, 111 insertions(+), 103 deletions(-) create mode 100644 src/libfetchers/tarball.hh diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index e36bda52f..e6df49a7a 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -9,6 +9,7 @@ #include "flake/flakeref.hh" #include "store-api.hh" #include "command.hh" +#include "tarball.hh" namespace nix { @@ -168,14 +169,14 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s) { if (EvalSettings::isPseudoUrl(s)) { auto storePath = fetchers::downloadTarball( - state.store, EvalSettings::resolvePseudoUrl(s), "source", false).tree.storePath; + state.store, EvalSettings::resolvePseudoUrl(s), "source", false).storePath; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); - auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first.storePath; + auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index a6212c12f..b32c7c086 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -15,7 +15,7 @@ using namespace flake; namespace flake { -typedef std::pair FetchedFlake; +typedef std::pair FetchedFlake; typedef std::vector> FlakeCache; static std::optional lookupInFlakeCache( @@ -34,7 +34,7 @@ static std::optional lookupInFlakeCache( return std::nullopt; } -static std::tuple fetchOrSubstituteTree( +static std::tuple fetchOrSubstituteTree( EvalState & state, const FlakeRef & originalRef, bool allowLookup, @@ -61,16 +61,16 @@ static std::tuple fetchOrSubstituteTree( flakeCache.push_back({originalRef, *fetched}); } - auto [tree, lockedRef] = *fetched; + auto [storePath, lockedRef] = *fetched; debug("got tree '%s' from '%s'", - state.store->printStorePath(tree.storePath), lockedRef); + state.store->printStorePath(storePath), lockedRef); - state.allowPath(tree.storePath); + state.allowPath(storePath); - assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store)); + assert(!originalRef.input.getNarHash() || storePath == originalRef.input.computeStorePath(*state.store)); - return {std::move(tree), resolvedRef, lockedRef}; + return {std::move(storePath), resolvedRef, lockedRef}; } static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) @@ -202,21 +202,21 @@ static Flake getFlake( FlakeCache & flakeCache, InputPath lockRootPath) { - auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( + auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( state, originalRef, allowLookup, flakeCache); // Guard against symlink attacks. - auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir, true); + auto flakeDir = canonPath(state.store->toRealPath(storePath) + "/" + lockedRef.subdir, true); auto flakeFile = canonPath(flakeDir + "/flake.nix", true); - if (!isInDir(flakeFile, sourceInfo.actualPath)) + if (!isInDir(flakeFile, state.store->toRealPath(storePath))) throw Error("'flake.nix' file of flake '%s' escapes from '%s'", - lockedRef, state.store->printStorePath(sourceInfo.storePath)); + lockedRef, state.store->printStorePath(storePath)); Flake flake { .originalRef = originalRef, .resolvedRef = resolvedRef, .lockedRef = lockedRef, - .sourceInfo = std::make_shared(std::move(sourceInfo)) + .storePath = storePath, }; if (!pathExists(flakeFile)) @@ -346,7 +346,7 @@ LockedFlake lockFlake( // FIXME: symlink attack auto oldLockFile = LockFile::read( lockFlags.referenceLockFilePath.value_or( - flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock")); + state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir + "/flake.lock")); debug("old lock file: %s", oldLockFile); @@ -574,7 +574,7 @@ LockedFlake lockFlake( oldLock ? std::dynamic_pointer_cast(oldLock) : LockFile::read( - inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root.get_ptr(), + state.store->toRealPath(inputFlake.storePath) + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root.get_ptr(), oldLock ? lockRootPath : inputPath, localPath, false); @@ -598,7 +598,7 @@ LockedFlake lockFlake( }; // Bring in the current ref for relative path resolution if we have it - auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir, true); + auto parentPath = canonPath(state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir, true); computeLocks( flake.inputs, @@ -729,7 +729,7 @@ void callFlake(EvalState & state, emitTreeAttrs( state, - *lockedFlake.flake.sourceInfo, + lockedFlake.flake.storePath, lockedFlake.flake.lockedRef.input, *vRootSrc, false, @@ -893,7 +893,7 @@ Fingerprint LockedFlake::getFingerprint() const // flake.sourceInfo.storePath for the fingerprint. return hashString(htSHA256, fmt("%s;%s;%d;%d;%s", - flake.sourceInfo->storePath.to_string(), + flake.storePath.to_string(), flake.lockedRef.subdir, flake.lockedRef.input.getRevCount().value_or(0), flake.lockedRef.input.getLastModified().value_or(0), diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index c1d1b71e5..d5ad3eade 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -10,8 +10,6 @@ namespace nix { class EvalState; -namespace fetchers { struct Tree; } - namespace flake { struct FlakeInput; @@ -84,7 +82,7 @@ struct Flake */ bool forceDirty = false; std::optional description; - std::shared_ptr sourceInfo; + StorePath storePath; FlakeInputs inputs; /** * 'nixConfig' attribute @@ -193,7 +191,7 @@ void callFlake( void emitTreeAttrs( EvalState & state, - const fetchers::Tree & tree, + const StorePath & storePath, const fetchers::Input & input, Value & v, bool emptyRevFallback = false, diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 93ad33a33..16f45ace7 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -272,10 +272,10 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs) fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); } -std::pair FlakeRef::fetchTree(ref store) const +std::pair FlakeRef::fetchTree(ref store) const { - auto [tree, lockedInput] = input.fetch(store); - return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)}; + auto [storePath, lockedInput] = input.fetch(store); + return {std::move(storePath), FlakeRef(std::move(lockedInput), subdir)}; } std::tuple parseFlakeRefWithFragmentAndExtendedOutputsSpec( diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index ffb2e50de..5d78f49b6 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -63,7 +63,7 @@ struct FlakeRef static FlakeRef fromAttrs(const fetchers::Attrs & attrs); - std::pair fetchTree(ref store) const; + std::pair fetchTree(ref store) const; }; std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 70228e1e2..8b73bd056 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -646,7 +646,7 @@ formal #include "eval.hh" #include "filetransfer.hh" -#include "fetchers.hh" +#include "tarball.hh" #include "store-api.hh" #include "flake/flake.hh" @@ -783,7 +783,7 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa if (EvalSettings::isPseudoUrl(value)) { try { auto storePath = fetchers::downloadTarball( - store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath; + store, EvalSettings::resolvePseudoUrl(value), "source", false).storePath; res = { store->toRealPath(storePath) }; } catch (FileTransferError & e) { logWarning({ @@ -797,7 +797,7 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); debug("fetching flake search path element '%s''", value); - auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; + auto storePath = flakeRef.resolve(store).fetchTree(store).first; res = { store->toRealPath(storePath) }; } diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index b9ff01c16..e76ce455d 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -71,10 +71,10 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a auto input = fetchers::Input::fromAttrs(std::move(attrs)); // FIXME: use name - auto [tree, input2] = input.fetch(state.store); + auto [storePath, input2] = input.fetch(state.store); auto attrs2 = state.buildBindings(8); - state.mkStorePathString(tree.storePath, attrs2.alloc(state.sOutPath)); + state.mkStorePathString(storePath, attrs2.alloc(state.sOutPath)); if (input2.getRef()) attrs2.alloc("branch").mkString(*input2.getRef()); // Backward compatibility: set 'rev' to @@ -86,7 +86,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a attrs2.alloc("revCount").mkInt(*revCount); v.mkAttrs(attrs2); - state.allowPath(tree.storePath); + state.allowPath(storePath); } static RegisterPrimOp r_fetchMercurial({ diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 976037ff9..a99b0e500 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -5,6 +5,7 @@ #include "fetchers.hh" #include "filetransfer.hh" #include "registry.hh" +#include "tarball.hh" #include "url.hh" #include @@ -15,7 +16,7 @@ namespace nix { void emitTreeAttrs( EvalState & state, - const fetchers::Tree & tree, + const StorePath & storePath, const fetchers::Input & input, Value & v, bool emptyRevFallback, @@ -25,7 +26,7 @@ void emitTreeAttrs( auto attrs = state.buildBindings(10); - state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath)); + state.mkStorePathString(storePath, attrs.alloc(state.sOutPath)); // FIXME: support arbitrary input attributes. @@ -165,11 +166,11 @@ static void fetchTree( state.checkURI(input.toURLString()); - auto [tree, input2] = input.fetch(state.store); + auto [storePath, input2] = input.fetch(state.store); - state.allowPath(tree.storePath); + state.allowPath(storePath); - emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false); + emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false); } static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v) @@ -288,7 +289,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v // https://github.com/NixOS/nix/issues/4313 auto storePath = unpack - ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).tree.storePath + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; if (expectedHash) { diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 5f6167868..5688c4dc1 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -109,7 +109,7 @@ bool Input::contains(const Input & other) const return false; } -std::pair Input::fetch(ref store) const +std::pair Input::fetch(ref store) const { if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); @@ -126,7 +126,7 @@ std::pair Input::fetch(ref store) const debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath)); - return {Tree { .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath) }, *this}; + return {std::move(storePath), *this}; } catch (Error & e) { debug("substitution of input '%s' failed: %s", to_string(), e.what()); } @@ -141,18 +141,16 @@ std::pair Input::fetch(ref store) const } }(); - Tree tree { - .actualPath = store->toRealPath(storePath), - .storePath = storePath, - }; - - auto narHash = store->queryPathInfo(tree.storePath)->narHash; + auto narHash = store->queryPathInfo(storePath)->narHash; input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); if (auto prevNarHash = getNarHash()) { if (narHash != *prevNarHash) throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, prevNarHash->to_string(HashFormat::SRI, true), narHash.to_string(HashFormat::SRI, true)); + to_string(), + store->printStorePath(storePath), + prevNarHash->to_string(HashFormat::SRI, true), + narHash.to_string(HashFormat::SRI, true)); } if (auto prevLastModified = getLastModified()) { @@ -175,7 +173,7 @@ std::pair Input::fetch(ref store) const input.locked = true; - return {std::move(tree), input}; + return {std::move(storePath), input}; } Input Input::applyOverrides( diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 4119dd17b..ac605ff8e 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -13,12 +13,6 @@ namespace nix { class Store; } namespace nix::fetchers { -struct Tree -{ - Path actualPath; - StorePath storePath; -}; - struct InputScheme; /** @@ -83,10 +77,10 @@ public: bool contains(const Input & other) const; /** - * Fetch the input into the Nix store, returning the location in - * the Nix store and the locked input. + * Fetch the entire input into the Nix store, returning the + * location in the Nix store and the locked input. */ - std::pair fetch(ref store) const; + std::pair fetch(ref store) const; Input applyOverrides( std::optional ref, @@ -158,33 +152,4 @@ struct InputScheme void registerInputScheme(std::shared_ptr && fetcher); -struct DownloadFileResult -{ - StorePath storePath; - std::string etag; - std::string effectiveUrl; - std::optional immutableUrl; -}; - -DownloadFileResult downloadFile( - ref store, - const std::string & url, - const std::string & name, - bool locked, - const Headers & headers = {}); - -struct DownloadTarballResult -{ - Tree tree; - time_t lastModified; - std::optional immutableUrl; -}; - -DownloadTarballResult downloadTarball( - ref store, - const std::string & url, - const std::string & name, - bool locked, - const Headers & headers = {}); - } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index b824140a6..617fc7468 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -7,6 +7,7 @@ #include "git.hh" #include "fetchers.hh" #include "fetch-settings.hh" +#include "tarball.hh" #include #include @@ -213,10 +214,10 @@ struct GitArchiveInputScheme : InputScheme {"rev", rev->gitRev()}, {"lastModified", uint64_t(result.lastModified)} }, - result.tree.storePath, + result.storePath, true); - return {result.tree.storePath, input}; + return {result.storePath, input}; } std::optional experimentalFeature() override diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 43c03beec..a0fff9ceb 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,5 +1,5 @@ #include "registry.hh" -#include "fetchers.hh" +#include "tarball.hh" #include "util.hh" #include "globals.hh" #include "store-api.hh" diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index ba47d59a7..e1ea9b58b 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -1,3 +1,4 @@ +#include "tarball.hh" #include "fetchers.hh" #include "cache.hh" #include "filetransfer.hh" @@ -133,7 +134,7 @@ DownloadTarballResult downloadTarball( if (cached && !cached->expired) return { - .tree = Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) }, + .storePath = std::move(cached->storePath), .lastModified = (time_t) getIntAttr(cached->infoAttrs, "lastModified"), .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"), }; @@ -174,7 +175,7 @@ DownloadTarballResult downloadTarball( locked); return { - .tree = Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) }, + .storePath = std::move(*unpackedStorePath), .lastModified = lastModified, .immutableUrl = res.immutableUrl, }; @@ -307,7 +308,7 @@ struct TarballInputScheme : CurlInputScheme if (result.lastModified && !input.attrs.contains("lastModified")) input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); - return {result.tree.storePath, std::move(input)}; + return {result.storePath, std::move(input)}; } }; diff --git a/src/libfetchers/tarball.hh b/src/libfetchers/tarball.hh new file mode 100644 index 000000000..9e6b50b31 --- /dev/null +++ b/src/libfetchers/tarball.hh @@ -0,0 +1,43 @@ +#pragma once + +#include "types.hh" +#include "path.hh" + +#include + +namespace nix { +class Store; +} + +namespace nix::fetchers { + +struct DownloadFileResult +{ + StorePath storePath; + std::string etag; + std::string effectiveUrl; + std::optional immutableUrl; +}; + +DownloadFileResult downloadFile( + ref store, + const std::string & url, + const std::string & name, + bool locked, + const Headers & headers = {}); + +struct DownloadTarballResult +{ + StorePath storePath; + time_t lastModified; + std::optional immutableUrl; +}; + +DownloadTarballResult downloadTarball( + ref store, + const std::string & url, + const std::string & name, + bool locked, + const Headers & headers = {}); + +} diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 95f401441..4504441fa 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -4,9 +4,9 @@ #include "filetransfer.hh" #include "store-api.hh" #include "legacy.hh" -#include "fetchers.hh" #include "eval-settings.hh" // for defexpr #include "util.hh" +#include "tarball.hh" #include #include diff --git a/src/nix/flake.cc b/src/nix/flake.cc index ceb112c03..b4e4156c0 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -186,7 +186,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["revCount"] = *revCount; if (auto lastModified = flake.lockedRef.input.getLastModified()) j["lastModified"] = *lastModified; - j["path"] = store->printStorePath(flake.sourceInfo->storePath); + j["path"] = store->printStorePath(flake.storePath); j["locks"] = lockedFlake.lockFile.toJSON(); logger->cout("%s", j.dump()); } else { @@ -202,7 +202,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON *flake.description); logger->cout( ANSI_BOLD "Path:" ANSI_NORMAL " %s", - store->printStorePath(flake.sourceInfo->storePath)); + store->printStorePath(flake.storePath)); if (auto rev = flake.lockedRef.input.getRev()) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", @@ -976,7 +976,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun StorePathSet sources; - sources.insert(flake.flake.sourceInfo->storePath); + sources.insert(flake.flake.storePath); // FIXME: use graph output, handle cycles. std::function traverse; @@ -988,7 +988,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun auto storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) - : (*inputNode)->lockedRef.input.fetch(store).first.storePath; + : (*inputNode)->lockedRef.input.fetch(store).first; if (json) { auto& jsonObj3 = jsonObj2[inputName]; jsonObj3["path"] = store->printStorePath(storePath); @@ -1005,7 +1005,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun if (json) { nlohmann::json jsonRoot = { - {"path", store->printStorePath(flake.flake.sourceInfo->storePath)}, + {"path", store->printStorePath(flake.flake.storePath)}, {"inputs", traverse(*flake.lockFile.root)}, }; logger->cout("%s", jsonRoot); @@ -1339,12 +1339,12 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON { auto originalRef = getFlakeRef(); auto resolvedRef = originalRef.resolve(store); - auto [tree, lockedRef] = resolvedRef.fetchTree(store); - auto hash = store->queryPathInfo(tree.storePath)->narHash; + auto [storePath, lockedRef] = resolvedRef.fetchTree(store); + auto hash = store->queryPathInfo(storePath)->narHash; if (json) { auto res = nlohmann::json::object(); - res["storePath"] = store->printStorePath(tree.storePath); + res["storePath"] = store->printStorePath(storePath); res["hash"] = hash.to_string(HashFormat::SRI, true); res["original"] = fetchers::attrsToJSON(resolvedRef.toAttrs()); res["locked"] = fetchers::attrsToJSON(lockedRef.toAttrs()); @@ -1352,7 +1352,7 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON } else { notice("Downloaded '%s' to '%s' (hash '%s').", lockedRef.to_string(), - store->printStorePath(tree.storePath), + store->printStorePath(storePath), hash.to_string(HashFormat::SRI, true)); } } From 97a0c08873496dbd6f53adb253e159ecf700d616 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 20 Oct 2023 21:17:28 +0200 Subject: [PATCH 444/909] Expand derivation examples (#9048) Also use fancier formatting so the example blocks are easier to discern from the description. Co-authored-by: John Ericson --- doc/manual/src/glossary.md | 1 + doc/manual/src/language/derivations.md | 214 +++++++++++++++++++------ 2 files changed, 162 insertions(+), 53 deletions(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 611d9576f..d49d5e52e 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -207,6 +207,7 @@ - [output]{#gloss-output} A [store object] produced by a [derivation]. + See [the `outputs` argument to the `derivation` function](@docroot@/language/derivations.md#attr-outputs) for details. [output]: #gloss-output diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index d80a7dd48..2aded5527 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -8,8 +8,6 @@ It outputs an attribute set, and produces a [store derivation] as a side effect [store derivation]: @docroot@/glossary.md#gloss-store-derivation - - ## Input attributes ### Required @@ -17,11 +15,22 @@ It outputs an attribute set, and produces a [store derivation] as a side effect - [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) A symbolic name for the derivation. - It is added to the [store derivation]'s [path](@docroot@/glossary.md#gloss-store-path) and its [output paths][output path]. + It is added to the [store path] of the corresponding [store derivation] as well as to its [output paths](@docroot@/glossary.md#gloss-output-path). - Example: `name = "hello";` + [store path]: @docroot@/glossary.md#gloss-store-path + + > **Example** + > + > ```nix + > derivation { + > name = "hello"; + > # ... + > } + > ``` + > + > The store derivation's path will be `/nix/store/-hello.drv`. + > The [output](#attr-outputs) paths will be of the form `/nix/store/-hello[-]` - The store derivation's path will be `/nix/store/-hello.drv`, and the output paths will be of the form `/nix/store/-hello[-]` - [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) The system type on which the [`builder`](#attr-builder) executable is meant to be run. @@ -29,77 +38,175 @@ It outputs an attribute set, and produces a [store derivation] as a side effect A necessary condition for Nix to build derivations locally is that the `system` attribute matches the current [`system` configuration option]. It can automatically [build on other platforms](../advanced-topics/distributed-builds.md) by forwarding build requests to other machines. - Examples: - - `system = "x86_64-linux";` - - `system = builtins.currentSystem;` - - [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. - [`system` configuration option]: @docroot@/command-ref/conf-file.md#conf-system + > **Example** + > + > Declare a derivation to be built on a specific system type: + > + > ```nix + > derivation { + > # ... + > system = "x86_64-linux"; + > # ... + > } + > ``` + + > **Example** + > + > Declare a derivation to be built on the system type that evaluates the expression: + > + > ```nix + > derivation { + > # ... + > system = builtins.currentSystem; + > # ... + > } + > ``` + > + > [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. + - [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string)) Path to an executable that will perform the build. - Examples: + > **Example** + > + > Use the file located at `/bin/bash` as the builder executable: + > + > ```nix + > derivation { + > # ... + > builder = "/bin/bash"; + > # ... + > }; + > ``` - `builder = "/bin/bash";` + - `builder = ./builder.sh;` + > **Example** + > + > Copy a local file to the Nix store for use as the builder executable: + > + > ```nix + > derivation { + > # ... + > builder = ./builder.sh; + > # ... + > }; + > ``` - `builder = "${pkgs.python}/bin/python";` + + + > **Example** + > + > Use a file from another derivation as the builder executable: + > + > ```nix + > let pkgs = import {}; in + > derivation { + > # ... + > builder = "${pkgs.python}/bin/python"; + > # ... + > }; + > ``` ### Optional -- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) Default: `[ ]` +- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) + + Default: `[ ]` Command-line arguments to be passed to the [`builder`](#attr-builder) executable. - Example: `args = [ "-c" "echo hello world > $out" ];` + > **Example** + > + > Pass arguments to Bash to interpret a shell command: + > + > ```nix + > derivation { + > # ... + > builder = "/bin/bash"; + > args = [ "-c" "echo hello world > $out" ]; + > # ... + > }; + > ``` -- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) Default: `[ "out" ]` +- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) + + Default: `[ "out" ]` Symbolic outputs of the derivation. - Each output name is passed to the [`builder`](#attr-builder) executable as an environment variable with its value set to the corresponding [output path]. + Each output name is passed to the [`builder`](#attr-builder) executable as an environment variable with its value set to the corresponding [store path]. - [output path]: @docroot@/glossary.md#gloss-output-path - - By default, a derivation produces a single output path called `out`. - However, derivations can produce multiple output paths. + By default, a derivation produces a single output called `out`. + However, derivations can produce multiple outputs. This allows the associated [store objects](@docroot@/glossary.md#gloss-store-object) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. - Examples: + > **Example** + > + > Imagine a library package that provides a dynamic library, header files, and documentation. + > A program that links against such a library doesn’t need the header files and documentation at runtime, and it doesn’t need the documentation at build time. + > Thus, the library package could specify: + > + > ```nix + > derivation { + > # ... + > outputs = [ "lib" "dev" "doc" ]; + > # ... + > } + > ``` + > + > This will cause Nix to pass environment variables `lib`, `dev`, and `doc` to the builder containing the intended store paths of each output. + > The builder would typically do something like + > + > ```bash + > ./configure \ + > --libdir=$lib/lib \ + > --includedir=$dev/include \ + > --docdir=$doc/share/doc + > ``` + > + > for an Autoconf-style package. - Imagine a library package that provides a dynamic library, header files, and documentation. - A program that links against such a library doesn’t need the header files and documentation at runtime, and it doesn’t need the documentation at build time. - Thus, the library package could specify: + The name of an output is combined with the name of the derivation to create the name part of the output's store path, unless it is `out`, in which case just the name of the derivation is used. - ```nix - derivation { - # ... - outputs = [ "lib" "dev" "doc" ]; - # ... - } - ``` + > **Example** + > + > + > ```nix + > derivation { + > name = "example"; + > outputs = [ "lib" "dev" "doc" "out" ]; + > # ... + > } + > ``` + > + > The store derivation path will be `/nix/store/-example.drv`. + > The output paths will be + > - `/nix/store/-example-lib` + > - `/nix/store/-example-dev` + > - `/nix/store/-example-doc` + > - `/nix/store/-example` - This will cause Nix to pass environment variables `lib`, `dev`, and `doc` to the builder containing the intended store paths of each output. - The builder would typically do something like + You can refer to each output of a derivation by selecting it as an attribute. + The first element of `outputs` determines the *default output* and ends up at the top-level. - ```bash - ./configure \ - --libdir=$lib/lib \ - --includedir=$dev/include \ - --docdir=$doc/share/doc - ``` - - for an Autoconf-style package. - - You can refer to each output of a derivation by selecting it as an attribute, e.g. `myPackage.lib` or `myPackage.doc`. - - The first element of `outputs` determines the *default output*. - Therefore, in the given example, `myPackage` is equivalent to `myPackage.lib`. + > **Example** + > + > Select an output by attribute name: + > + > ```nix + > let + > myPackage = derivation { + > name = "example"; + > outputs = [ "lib" "dev" "doc" "out" ]; + > # ... + > }; + > in myPackage.dev + > ``` + > + > Since `lib` is the first output, `myPackage` is equivalent to `myPackage.lib`. @@ -123,8 +230,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect reside in the Nix store. - A *derivation* causes that derivation to be built prior to the - present derivation; its default output path is put in the - environment variable. + present derivation. The environment variable is set to the [store path] of the derivation's default [output](#attr-outputs). - Lists of the previous types are also allowed. They are simply concatenated, separated by spaces. @@ -132,6 +238,8 @@ It outputs an attribute set, and produces a [store derivation] as a side effect - `true` is passed as the string `1`, `false` and `null` are passed as an empty string. + + ## Builder execution The [`builder`](#attr-builder) is executed as follows: From 96c58550b839ca415a2cef0f7a5ae51f3e86f028 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 16 Oct 2023 01:45:06 -0400 Subject: [PATCH 445/909] Test more derived paths --- src/libstore/tests/worker-protocol.cc | 11 ++++++++++- .../libstore/worker-protocol/derived-path.bin | Bin 120 -> 248 bytes 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index 63fecce96..b10192cc1 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -72,10 +72,19 @@ VERSIONED_CHARACTERIZATION_TEST( derivedPath, "derived-path", defaultVersion, - (std::tuple { + (std::tuple { DerivedPath::Opaque { .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, }, + DerivedPath::Opaque { + .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::All { }, + }, DerivedPath::Built { .drvPath = makeConstantStorePathRef(StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", diff --git a/unit-test-data/libstore/worker-protocol/derived-path.bin b/unit-test-data/libstore/worker-protocol/derived-path.bin index bb1a81ac6d096cddf510a457e7506e6a81ee66a3..0729b2690e2f99dc8122b013e4ceaa4a51ebd3d6 100644 GIT binary patch delta 39 mcmb>U!8pN!(Rd=G8JAv4Q5gdWm`(%= Date: Fri, 26 May 2023 14:48:11 -0400 Subject: [PATCH 446/909] Systematize the worker protocol derived path serialiser It was some ad-hoc functions to account for versions, while the already factored-out serializer just supported the latest version. Now, we can fold that version-specific logic into the factored out one, and so we do. --- src/libstore/daemon.cc | 18 ++-------- src/libstore/remote-store.cc | 33 ++---------------- src/libstore/tests/worker-protocol.cc | 29 +++++++++++++-- src/libstore/worker-protocol.cc | 26 ++++++++++++-- .../worker-protocol/derived-path-1.29.bin | Bin 0 -> 184 bytes ...derived-path.bin => derived-path-1.30.bin} | Bin 6 files changed, 56 insertions(+), 50 deletions(-) create mode 100644 unit-test-data/libstore/worker-protocol/derived-path-1.29.bin rename unit-test-data/libstore/worker-protocol/{derived-path.bin => derived-path-1.30.bin} (100%) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e117b507f..27c0b6431 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -261,18 +261,6 @@ struct ClientSettings } }; -static std::vector readDerivedPaths(Store & store, WorkerProto::Version clientVersion, WorkerProto::ReadConn conn) -{ - std::vector reqs; - if (GET_PROTOCOL_MINOR(clientVersion) >= 30) { - reqs = WorkerProto::Serialise>::read(store, conn); - } else { - for (auto & s : readStrings(conn.from)) - reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath()); - } - return reqs; -} - static void performOp(TunnelLogger * logger, ref store, TrustedFlag trusted, RecursiveFlag recursive, WorkerProto::Version clientVersion, Source & from, BufferedSink & to, WorkerProto::Op op) @@ -538,7 +526,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::BuildPaths: { - auto drvs = readDerivedPaths(*store, clientVersion, rconn); + auto drvs = WorkerProto::Serialise::read(*store, rconn); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode) readInt(from); @@ -563,7 +551,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::BuildPathsWithResults: { - auto drvs = readDerivedPaths(*store, clientVersion, rconn); + auto drvs = WorkerProto::Serialise::read(*store, rconn); BuildMode mode = bmNormal; mode = (BuildMode) readInt(from); @@ -938,7 +926,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryMissing: { - auto targets = readDerivedPaths(*store, clientVersion, rconn); + auto targets = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1704bfbb0..482c2ae01 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -655,33 +655,6 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, } catch (...) { return callback.rethrow(); } } -static void writeDerivedPaths(RemoteStore & store, RemoteStore::Connection & conn, const std::vector & reqs) -{ - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 30) { - WorkerProto::write(store, conn, reqs); - } else { - Strings ss; - for (auto & p : reqs) { - auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); - std::visit(overloaded { - [&](const StorePathWithOutputs & s) { - ss.push_back(s.to_string(store)); - }, - [&](const StorePath & drvPath) { - throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", - store.printStorePath(drvPath), - GET_PROTOCOL_MAJOR(conn.daemonVersion), - GET_PROTOCOL_MINOR(conn.daemonVersion)); - }, - [&](std::monostate) { - throw Error("wanted to build a derivation that is itself a build product, but the legacy 'ssh://' protocol doesn't support that. Try using 'ssh-ng://'"); - }, - }, sOrDrvPath); - } - conn.to << ss; - } -} - void RemoteStore::copyDrvsFromEvalStore( const std::vector & paths, std::shared_ptr evalStore) @@ -711,7 +684,7 @@ void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMod auto conn(getConnection()); conn->to << WorkerProto::Op::BuildPaths; assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); - writeDerivedPaths(*this, *conn, drvPaths); + WorkerProto::write(*this, *conn, drvPaths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) conn->to << buildMode; else @@ -735,7 +708,7 @@ std::vector RemoteStore::buildPathsWithResults( if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) { conn->to << WorkerProto::Op::BuildPathsWithResults; - writeDerivedPaths(*this, *conn, paths); + WorkerProto::write(*this, *conn, paths); conn->to << buildMode; conn.processStderr(); return WorkerProto::Serialise>::read(*this, *conn); @@ -929,7 +902,7 @@ void RemoteStore::queryMissing(const std::vector & targets, // to prevent a deadlock. goto fallback; conn->to << WorkerProto::Op::QueryMissing; - writeDerivedPaths(*this, *conn, targets); + WorkerProto::write(*this, *conn, targets); conn.processStderr(); willBuild = WorkerProto::Serialise::read(*this, *conn); willSubstitute = WorkerProto::Serialise::read(*this, *conn); diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index b10192cc1..b3e857bc4 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -69,9 +69,32 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - derivedPath, - "derived-path", - defaultVersion, + derivedPath_1_29, + "derived-path-1.29", + 1 << 8 | 29, + (std::tuple { + DerivedPath::Opaque { + .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::All { }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::Names { "x", "y" }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + derivedPath_1_30, + "derived-path-1.30", + 1 << 8 | 30, (std::tuple { DerivedPath::Opaque { .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 415e66f16..0f3f20ca5 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -51,12 +51,34 @@ void WorkerProto::Serialise>::write(const Store & sto DerivedPath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { auto s = readString(conn.from); - return DerivedPath::parseLegacy(store, s); + if (GET_PROTOCOL_MINOR(conn.version) >= 30) { + return DerivedPath::parseLegacy(store, s); + } else { + return parsePathWithOutputs(store, s).toDerivedPath(); + } } void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const DerivedPath & req) { - conn.to << req.to_string_legacy(store); + if (GET_PROTOCOL_MINOR(conn.version) >= 30) { + conn.to << req.to_string_legacy(store); + } else { + auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(req); + std::visit(overloaded { + [&](const StorePathWithOutputs & s) { + conn.to << s.to_string(store); + }, + [&](const StorePath & drvPath) { + throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", + store.printStorePath(drvPath), + GET_PROTOCOL_MAJOR(conn.version), + GET_PROTOCOL_MINOR(conn.version)); + }, + [&](std::monostate) { + throw Error("wanted to build a derivation that is itself a build product, but protocols do not support that. Try upgrading the Nix on the other end of this connection"); + }, + }, sOrDrvPath); + } } diff --git a/unit-test-data/libstore/worker-protocol/derived-path-1.29.bin b/unit-test-data/libstore/worker-protocol/derived-path-1.29.bin new file mode 100644 index 0000000000000000000000000000000000000000..05ea7678aa058344a5ca4cb1e9e7333e9cf8fe20 GIT binary patch literal 184 zcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7Eu*F&X-j5{vXwipsz`&B@oVSfNwN F001Y$HlP3i literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/derived-path.bin b/unit-test-data/libstore/worker-protocol/derived-path-1.30.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/derived-path.bin rename to unit-test-data/libstore/worker-protocol/derived-path-1.30.bin From ab822af0dfee341be131ad285a2cba5362a3f2dc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 28 Feb 2021 18:42:46 +0000 Subject: [PATCH 447/909] Factor out serialization for `BuildResult` Worker Protocol: Note that the worker protocol already had a serialization for `BuildResult`; this was added in a4604f19284254ac98f19a13ff7c2216de7fe176. It didn't have any versioning support because at that time reusable seralizers were not away for the protocol version. It could thus only be used for new messages also introduced in that commit. Now that we do support versioning in reusable serializers, we can expand it to support all known versions and use it in many more places. The exist test data becomes the version 1.29 tests: note that those files' contents are unchanged. 1.28 and 1.27 tests are added to cover the older code-paths. The keyered build result test only has 1.29 because the keying was also added in a4604f19284254ac98f19a13ff7c2216de7fe176; the older serializations are always used unkeyed. Serve Protocol: Conversely, no attempt was made to factor out such a serializer for the serve protocol, so our work there in this commit for that protocol proceeds from scratch. --- src/libstore/daemon.cc | 11 +- src/libstore/legacy-ssh-store.cc | 15 +-- src/libstore/remote-store.cc | 15 +-- src/libstore/serve-protocol.cc | 43 +++++++ src/libstore/serve-protocol.hh | 6 + src/libstore/tests/serve-protocol.cc | 113 +++++++++++++++++- src/libstore/tests/worker-protocol.cc | 80 ++++++++++++- src/libstore/worker-protocol.cc | 49 ++++---- src/nix-store/nix-store.cc | 12 +- .../serve-protocol/build-result-2.2.bin | Bin 0 -> 80 bytes .../serve-protocol/build-result-2.3.bin | Bin 0 -> 176 bytes .../build-result-2.6.bin} | Bin .../worker-protocol/build-result-1.27.bin | Bin 0 -> 80 bytes .../worker-protocol/build-result-1.28.bin | Bin 0 -> 648 bytes .../worker-protocol/build-result-1.29.bin | Bin 0 -> 744 bytes ...result.bin => keyed-build-result-1.29.bin} | Bin 16 files changed, 268 insertions(+), 76 deletions(-) create mode 100644 unit-test-data/libstore/serve-protocol/build-result-2.2.bin create mode 100644 unit-test-data/libstore/serve-protocol/build-result-2.3.bin rename unit-test-data/libstore/{worker-protocol/build-result.bin => serve-protocol/build-result-2.6.bin} (100%) create mode 100644 unit-test-data/libstore/worker-protocol/build-result-1.27.bin create mode 100644 unit-test-data/libstore/worker-protocol/build-result-1.28.bin create mode 100644 unit-test-data/libstore/worker-protocol/build-result-1.29.bin rename unit-test-data/libstore/worker-protocol/{keyed-build-result.bin => keyed-build-result-1.29.bin} (100%) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 27c0b6431..19afe4388 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -635,16 +635,7 @@ static void performOp(TunnelLogger * logger, ref store, auto res = store->buildDerivation(drvPath, drv, buildMode); logger->stopWork(); - to << res.status << res.errorMsg; - if (GET_PROTOCOL_MINOR(clientVersion) >= 29) { - to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime; - } - if (GET_PROTOCOL_MINOR(clientVersion) >= 28) { - DrvOutputs builtOutputs; - for (auto & [output, realisation] : res.builtOutputs) - builtOutputs.insert_or_assign(realisation.id, realisation); - WorkerProto::write(*store, wconn, builtOutputs); - } + WorkerProto::write(*store, wconn, res); break; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index d6e24521a..c712f7eb1 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -319,20 +319,7 @@ public: conn->to.flush(); - BuildResult status; - status.status = (BuildResult::Status) readInt(conn->from); - conn->from >> status.errorMsg; - - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) - conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - auto builtOutputs = ServeProto::Serialise::read(*this, *conn); - for (auto && [output, realisation] : builtOutputs) - status.builtOutputs.insert_or_assign( - std::move(output.outputName), - std::move(realisation)); - } - return status; + return ServeProto::Serialise::read(*this, *conn); } void buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) override diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 482c2ae01..ef648e01b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -788,20 +788,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD writeDerivation(conn->to, *this, drv); conn->to << buildMode; conn.processStderr(); - BuildResult res; - res.status = (BuildResult::Status) readInt(conn->from); - conn->from >> res.errorMsg; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { - conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime; - } - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) { - auto builtOutputs = WorkerProto::Serialise::read(*this, *conn); - for (auto && [output, realisation] : builtOutputs) - res.builtOutputs.insert_or_assign( - std::move(output.outputName), - std::move(realisation)); - } - return res; + return WorkerProto::Serialise::read(*this, *conn); } diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index 16a62b5bc..97a0ddf0e 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "path-with-outputs.hh" #include "store-api.hh" +#include "build-result.hh" #include "serve-protocol.hh" #include "serve-protocol-impl.hh" #include "archive.hh" @@ -12,4 +13,46 @@ namespace nix { /* protocol-specific definitions */ +BuildResult ServeProto::Serialise::read(const Store & store, ServeProto::ReadConn conn) +{ + BuildResult status; + status.status = (BuildResult::Status) readInt(conn.from); + conn.from >> status.errorMsg; + + if (GET_PROTOCOL_MINOR(conn.version) >= 3) + conn.from + >> status.timesBuilt + >> status.isNonDeterministic + >> status.startTime + >> status.stopTime; + if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + auto builtOutputs = ServeProto::Serialise::read(store, conn); + for (auto && [output, realisation] : builtOutputs) + status.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); + } + return status; +} + +void ServeProto::Serialise::write(const Store & store, ServeProto::WriteConn conn, const BuildResult & status) +{ + conn.to + << status.status + << status.errorMsg; + + if (GET_PROTOCOL_MINOR(conn.version) >= 3) + conn.to + << status.timesBuilt + << status.isNonDeterministic + << status.startTime + << status.stopTime; + if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + DrvOutputs builtOutputs; + for (auto & [output, realisation] : status.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + ServeProto::write(store, conn, builtOutputs); + } +} + } diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index a627c6ad6..ba159f6e9 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -16,6 +16,9 @@ namespace nix { class Store; struct Source; +// items being serialised +struct BuildResult; + /** * The "serve protocol", used by ssh:// stores. @@ -136,6 +139,9 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op) static void write(const Store & store, ServeProto::WriteConn conn, const T & t); \ }; +template<> +DECLARE_SERVE_SERIALISER(BuildResult); + template DECLARE_SERVE_SERIALISER(std::vector); template diff --git a/src/libstore/tests/serve-protocol.cc b/src/libstore/tests/serve-protocol.cc index ba798ab1c..c8ac87a04 100644 --- a/src/libstore/tests/serve-protocol.cc +++ b/src/libstore/tests/serve-protocol.cc @@ -19,7 +19,7 @@ struct ServeProtoTest : VersionedProtoTest * For serializers that don't care about the minimum version, we * used the oldest one: 1.0. */ - ServeProto::Version defaultVersion = 1 << 8 | 0; + ServeProto::Version defaultVersion = 2 << 8 | 0; }; VERSIONED_CHARACTERIZATION_TEST( @@ -114,6 +114,117 @@ VERSIONED_CHARACTERIZATION_TEST( }, })) +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_2, + "build-result-2.2", + 2 << 8 | 2, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::Built, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_3, + "build-result-2.3", + 2 << 8 | 3, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + BuildResult { + .status = BuildResult::Built, + .startTime = 30, + .stopTime = 50, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_6, + "build-result-2.6", + 2 << 8 | 6, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + BuildResult { + .status = BuildResult::Built, + .timesBuilt = 1, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + .startTime = 30, + .stopTime = 50, +#if 0 + // These fields are not yet serialized. + // FIXME Include in next version of protocol or document + // why they are skipped. + .cpuUser = std::chrono::milliseconds(500s), + .cpuSystem = std::chrono::milliseconds(604s), +#endif + }, + }; + t; + })) + VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, vector, diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index b3e857bc4..e0ce340d8 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -167,9 +167,77 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - buildResult, - "build-result", - defaultVersion, + buildResult_1_27, + "build-result-1.27", + 1 << 8 | 27, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::Built, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_28, + "build-result-1.28", + 1 << 8 | 28, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::Built, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_29, + "build-result-1.29", + 1 << 8 | 29, ({ using namespace std::literals::chrono_literals; std::tuple t { @@ -226,9 +294,9 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - keyedBuildResult, - "keyed-build-result", - defaultVersion, + keyedBuildResult_1_29, + "keyed-build-result-1.29", + 1 << 8 | 29, ({ using namespace std::literals::chrono_literals; std::tuple t { diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 0f3f20ca5..f69322a61 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -103,17 +103,21 @@ BuildResult WorkerProto::Serialise::read(const Store & store, Worke { BuildResult res; res.status = (BuildResult::Status) readInt(conn.from); - conn.from - >> res.errorMsg - >> res.timesBuilt - >> res.isNonDeterministic - >> res.startTime - >> res.stopTime; - auto builtOutputs = WorkerProto::Serialise::read(store, conn); - for (auto && [output, realisation] : builtOutputs) - res.builtOutputs.insert_or_assign( - std::move(output.outputName), - std::move(realisation)); + conn.from >> res.errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 29) { + conn.from + >> res.timesBuilt + >> res.isNonDeterministic + >> res.startTime + >> res.stopTime; + } + if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + auto builtOutputs = WorkerProto::Serialise::read(store, conn); + for (auto && [output, realisation] : builtOutputs) + res.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); + } return res; } @@ -121,15 +125,20 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto { conn.to << res.status - << res.errorMsg - << res.timesBuilt - << res.isNonDeterministic - << res.startTime - << res.stopTime; - DrvOutputs builtOutputs; - for (auto & [output, realisation] : res.builtOutputs) - builtOutputs.insert_or_assign(realisation.id, realisation); - WorkerProto::write(store, conn, builtOutputs); + << res.errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 29) { + conn.to + << res.timesBuilt + << res.isNonDeterministic + << res.startTime + << res.stopTime; + } + if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + DrvOutputs builtOutputs; + for (auto & [output, realisation] : res.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + WorkerProto::write(store, conn, builtOutputs); + } } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index e78720f92..e4dd94585 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -959,17 +959,7 @@ static void opServe(Strings opFlags, Strings opArgs) MonitorFdHup monitor(in.fd); auto status = store->buildDerivation(drvPath, drv); - out << status.status << status.errorMsg; - - if (GET_PROTOCOL_MINOR(clientVersion) >= 3) - out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; - if (GET_PROTOCOL_MINOR(clientVersion) >= 6) { - DrvOutputs builtOutputs; - for (auto & [output, realisation] : status.builtOutputs) - builtOutputs.insert_or_assign(realisation.id, realisation); - ServeProto::write(*store, wconn, builtOutputs); - } - + ServeProto::write(*store, wconn, status); break; } diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.2.bin b/unit-test-data/libstore/serve-protocol/build-result-2.2.bin new file mode 100644 index 0000000000000000000000000000000000000000..ae684778bc26addba4bf1b3e49cc30edf5f038fa GIT binary patch literal 80 gcmZQ&fBf0AiU4H~;_u literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.3.bin b/unit-test-data/libstore/serve-protocol/build-result-2.3.bin new file mode 100644 index 0000000000000000000000000000000000000000..d51e08dfc0d1715210e8a616a5403f03f4b2a46c GIT binary patch literal 176 vcmZQ&fBf0AiU4H~;_u literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.28.bin b/unit-test-data/libstore/worker-protocol/build-result-1.28.bin new file mode 100644 index 0000000000000000000000000000000000000000..74bcd5cf98b828fb63a931ef7810f7fabc805ca5 GIT binary patch literal 648 zcmc&wK?=e!5EQ|aK0%e!+`)6%Si?(*`6_8xaxz$KI6Xx-2t!SGa{k;2f8i+RMR2A;`6>RitBjDY7{lnANJD3OVh8WW_c zkTRG+zDX!Yj%68orEsd#Y$KG=*{FoWL-7`MFAQl%7RmZ0!PYe3jk66aF4r+L$O`tu z&uq-x(J#Q)LAOdzsy>VTJDdcofzX)BfXe~O6CwQ^%woR?}>#sX4il$bfs;n zmv%`YOQ~vvTo;t-%xH@l(n4vSK8bRZQHc`kI^B)Ih0TkNGRhXy8rqZN5BnYj(ieFo zAJ+t*u7l`;??iPt&V)lzi91dfGZFf@g4iVAZN4+jUVRU7o>ol_o!fedeM@Pl_mAVf T^ROZOQyyvZZF!s Date: Tue, 8 Mar 2022 23:09:26 +0000 Subject: [PATCH 448/909] Move `ValidPathInfo` serialization code to `worker-protocol.{cc.hh}` It does not belong with the data type itself. This also materializes the fact that `copyPath` does not do any version negotiation just just hard-codes "16". The non-standard interface of these serializers makes it harder to test, but this is fixed in the next commit which then adds those tests. --- src/libstore/daemon.cc | 4 +-- src/libstore/path-info.cc | 54 --------------------------------- src/libstore/path-info.hh | 5 --- src/libstore/remote-store.cc | 11 +++++-- src/libstore/store-api.cc | 11 ++++++- src/libstore/worker-protocol.cc | 45 ++++++++++++++++++++++++++- src/libstore/worker-protocol.hh | 12 ++++++++ 7 files changed, 76 insertions(+), 66 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 19afe4388..0afaaccba 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -422,7 +422,7 @@ static void performOp(TunnelLogger * logger, ref store, }(); logger->stopWork(); - pathInfo->write(to, *store, GET_PROTOCOL_MINOR(clientVersion)); + WorkerProto::Serialise::write(*store, wconn, *pathInfo); } else { HashType hashAlgo; std::string baseName; @@ -819,7 +819,7 @@ static void performOp(TunnelLogger * logger, ref store, if (info) { if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; - info->write(to, *store, GET_PROTOCOL_MINOR(clientVersion), false); + WorkerProto::Serialise::write(*store, wconn, *info, false); } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); to << 0; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index fb507a847..51fb5f02a 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -1,6 +1,4 @@ #include "path-info.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" #include "store-api.hh" namespace nix { @@ -99,7 +97,6 @@ Strings ValidPathInfo::shortRefs() const return refs; } - ValidPathInfo::ValidPathInfo( const Store & store, std::string_view name, @@ -128,55 +125,4 @@ ValidPathInfo::ValidPathInfo( }, std::move(ca).raw); } - -ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) -{ - return read(source, store, format, store.parseStorePath(readString(source))); -} - -ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format, StorePath && path) -{ - auto deriver = readString(source); - auto narHash = Hash::parseAny(readString(source), htSHA256); - ValidPathInfo info(path, narHash); - if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(store, - WorkerProto::ReadConn { - .from = source, - .version = format, - }); - source >> info.registrationTime >> info.narSize; - if (format >= 16) { - source >> info.ultimate; - info.sigs = readStrings(source); - info.ca = ContentAddress::parseOpt(readString(source)); - } - return info; -} - - -void ValidPathInfo::write( - Sink & sink, - const Store & store, - unsigned int format, - bool includePath) const -{ - if (includePath) - sink << store.printStorePath(path); - sink << (deriver ? store.printStorePath(*deriver) : "") - << narHash.to_string(HashFormat::Base16, false); - WorkerProto::write(store, - WorkerProto::WriteConn { - .to = sink, - .version = format, - }, - references); - sink << registrationTime << narSize; - if (format >= 16) { - sink << ultimate - << sigs - << renderContentAddress(ca); - } -} - } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index fea6d0e5f..919fe32f1 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -121,11 +121,6 @@ struct ValidPathInfo std::string_view name, ContentAddressWithReferences && ca, Hash narHash); virtual ~ValidPathInfo() { } - - static ValidPathInfo read(Source & source, const Store & store, unsigned int format); - static ValidPathInfo read(Source & source, const Store & store, unsigned int format, StorePath && path); - - void write(Sink & sink, const Store & store, unsigned int format, bool includePath = true) const; }; typedef std::map ValidPathInfos; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ef648e01b..aaea5c4a2 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -332,7 +332,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } info = std::make_shared( - ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion), StorePath{path})); + WorkerProto::Serialise::read(*this, *conn, StorePath{path})); } callback(std::move(info)); } catch (...) { callback.rethrow(); } @@ -445,7 +445,7 @@ ref RemoteStore::addCAToStore( } return make_ref( - ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion))); + WorkerProto::Serialise::read(*this, *conn)); } else { if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); @@ -570,7 +570,12 @@ void RemoteStore::addMultipleToStore( auto source = sinkToSource([&](Sink & sink) { sink << pathsToCopy.size(); for (auto & [pathInfo, pathSource] : pathsToCopy) { - pathInfo.write(sink, *this, 16); + WorkerProto::Serialise::write(*this, + WorkerProto::WriteConn { + .to = sink, + .version = 16, + }, + pathInfo); pathSource->drainInto(sink); } }); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 0c58cca0c..785b26062 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -11,6 +11,9 @@ #include "archive.hh" #include "callback.hh" #include "remote-store.hh" +// FIXME this should not be here, see TODO below on +// `addMultipleToStore`. +#include "worker-protocol.hh" #include #include @@ -357,7 +360,13 @@ void Store::addMultipleToStore( { auto expected = readNum(source); for (uint64_t i = 0; i < expected; ++i) { - auto info = ValidPathInfo::read(source, *this, 16); + // FIXME we should not be using the worker protocol here, let + // alone the worker protocol with a hard-coded version! + auto info = WorkerProto::Serialise::read(*this, + WorkerProto::ReadConn { + .from = source, + .version = 16, + }); info.ultimate = false; addToStore(info, source, repair, checkSigs); } diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index f69322a61..94f8c8811 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -6,7 +6,7 @@ #include "worker-protocol.hh" #include "worker-protocol-impl.hh" #include "archive.hh" -#include "derivations.hh" +#include "path-info.hh" #include @@ -142,4 +142,47 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto } +ValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) +{ + auto path = WorkerProto::Serialise::read(store, conn); + return WorkerProto::Serialise::read(store, conn, std::move(path)); +} + +ValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn, StorePath && path) +{ + auto deriver = readString(conn.from); + auto narHash = Hash::parseAny(readString(conn.from), htSHA256); + ValidPathInfo info(path, narHash); + if (deriver != "") info.deriver = store.parseStorePath(deriver); + info.references = WorkerProto::Serialise::read(store, conn); + conn.from >> info.registrationTime >> info.narSize; + if (GET_PROTOCOL_MINOR(conn.version) >= 16) { + conn.from >> info.ultimate; + info.sigs = readStrings(conn.from); + info.ca = ContentAddress::parseOpt(readString(conn.from)); + } + return info; +} + +void WorkerProto::Serialise::write( + const Store & store, + WriteConn conn, + const ValidPathInfo & pathInfo, + bool includePath) +{ + if (includePath) + conn.to << store.printStorePath(pathInfo.path); + conn.to + << (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "") + << pathInfo.narHash.to_string(HashFormat::Base16, false); + WorkerProto::write(store, conn, pathInfo.references); + conn.to << pathInfo.registrationTime << pathInfo.narSize; + if (GET_PROTOCOL_MINOR(conn.version) >= 16) { + conn.to + << pathInfo.ultimate + << pathInfo.sigs + << renderContentAddress(pathInfo.ca); + } +} + } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 2d55d926a..d35e3c678 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -31,6 +31,7 @@ struct Source; struct DerivedPath; struct BuildResult; struct KeyedBuildResult; +struct ValidPathInfo; enum TrustedFlag : bool; @@ -220,4 +221,15 @@ template DECLARE_WORKER_SERIALISER(std::map); #undef COMMA_ +/* These are a non-standard form for historical reasons. */ + +template<> +struct WorkerProto::Serialise +{ + static ValidPathInfo read(const Store & store, WorkerProto::ReadConn conn); + static ValidPathInfo read(const Store & store, WorkerProto::ReadConn conn, StorePath && path); + + static void write(const Store & store, WriteConn conn, const ValidPathInfo & pathInfo, bool includePath = true); +}; + } From 70f8b96c11af275b5766dca0a49737803d1e0339 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Apr 2023 17:13:44 -0400 Subject: [PATCH 449/909] Factor out `UnkeyedValidPathInfo` and test This makes the path info serialisers ideomatic again, which allows me to test them. --- src/libstore/daemon.cc | 2 +- src/libstore/path-info.cc | 23 ++- src/libstore/path-info.hh | 31 ++-- src/libstore/remote-store.cc | 3 +- src/libstore/tests/worker-protocol.cc | 153 ++++++++++++++++++ src/libstore/worker-protocol.cc | 24 +-- src/libstore/worker-protocol.hh | 16 +- .../unkeyed-valid-path-info-1.15.bin | Bin 0 -> 328 bytes .../worker-protocol/valid-path-info-1.15.bin | Bin 0 -> 488 bytes .../worker-protocol/valid-path-info-1.16.bin | Bin 0 -> 952 bytes 10 files changed, 218 insertions(+), 34 deletions(-) create mode 100644 unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin create mode 100644 unit-test-data/libstore/worker-protocol/valid-path-info-1.15.bin create mode 100644 unit-test-data/libstore/worker-protocol/valid-path-info-1.16.bin diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 0afaaccba..007ffc05a 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -819,7 +819,7 @@ static void performOp(TunnelLogger * logger, ref store, if (info) { if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; - WorkerProto::Serialise::write(*store, wconn, *info, false); + WorkerProto::write(*store, wconn, static_cast(*info)); } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); to << 0; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 51fb5f02a..ab39e71f4 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -3,6 +3,25 @@ namespace nix { +GENERATE_CMP_EXT( + , + UnkeyedValidPathInfo, + me->deriver, + me->narHash, + me->references, + me->registrationTime, + me->narSize, + //me->id, + me->ultimate, + me->sigs, + me->ca); + +GENERATE_CMP_EXT( + , + ValidPathInfo, + me->path, + static_cast(*me)); + std::string ValidPathInfo::fingerprint(const Store & store) const { if (narSize == 0) @@ -102,8 +121,8 @@ ValidPathInfo::ValidPathInfo( std::string_view name, ContentAddressWithReferences && ca, Hash narHash) - : path(store.makeFixedOutputPathFromCA(name, ca)) - , narHash(narHash) + : UnkeyedValidPathInfo(narHash) + , path(store.makeFixedOutputPathFromCA(name, ca)) { std::visit(overloaded { [this](TextInfo && ti) { diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 919fe32f1..a82e643ae 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -32,9 +32,8 @@ struct SubstitutablePathInfo typedef std::map SubstitutablePathInfos; -struct ValidPathInfo +struct UnkeyedValidPathInfo { - StorePath path; std::optional deriver; /** * \todo document this @@ -72,6 +71,20 @@ struct ValidPathInfo */ std::optional ca; + UnkeyedValidPathInfo(const UnkeyedValidPathInfo & other) = default; + + UnkeyedValidPathInfo(Hash narHash) : narHash(narHash) { }; + + DECLARE_CMP(UnkeyedValidPathInfo); + + virtual ~UnkeyedValidPathInfo() { } +}; + +struct ValidPathInfo : UnkeyedValidPathInfo { + StorePath path; + + DECLARE_CMP(ValidPathInfo); + /** * Return a fingerprint of the store path to be used in binary * cache signatures. It contains the store path, the base-32 @@ -84,11 +97,11 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); - /** - * @return The `ContentAddressWithReferences` that determines the - * store path for a content-addressed store object, `std::nullopt` - * for an input-addressed store object. - */ + /** + * @return The `ContentAddressWithReferences` that determines the + * store path for a content-addressed store object, `std::nullopt` + * for an input-addressed store object. + */ std::optional contentAddressWithReferences() const; /** @@ -114,8 +127,8 @@ struct ValidPathInfo ValidPathInfo(const ValidPathInfo & other) = default; - ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; - ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; + ValidPathInfo(StorePath && path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(std::move(path)) { }; + ValidPathInfo(const StorePath & path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(path) { }; ValidPathInfo(const Store & store, std::string_view name, ContentAddressWithReferences && ca, Hash narHash); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index aaea5c4a2..7bdc25433 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -332,7 +332,8 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } info = std::make_shared( - WorkerProto::Serialise::read(*this, *conn, StorePath{path})); + StorePath{path}, + WorkerProto::Serialise::read(*this, *conn)); } callback(std::move(info)); } catch (...) { callback.rethrow(); } diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index e0ce340d8..ad5943c69 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -329,6 +329,159 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + unkeyedValidPathInfo_1_15, + "unkeyed-valid-path-info-1.15", + 1 << 8 | 15, + (std::tuple { + ({ + UnkeyedValidPathInfo info { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + ({ + UnkeyedValidPathInfo info { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv", + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + validPathInfo_1_15, + "valid-path-info-1.15", + 1 << 8 | 15, + (std::tuple { + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + // other reference + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo", + }, + // self reference + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + validPathInfo_1_16, + "valid-path-info-1.16", + 1 << 8 | 16, + (std::tuple { + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info.ultimate = true; + info; + }), + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + // other reference + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo", + }, + // self reference + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info.sigs = { + "fake-sig-1", + "fake-sig-2", + }, + info; + }), + ({ + ValidPathInfo info { + *LibStoreTest::store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + })) + VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, optionalTrustedFlag, diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 94f8c8811..d618b9bd8 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -145,14 +145,24 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto ValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) { auto path = WorkerProto::Serialise::read(store, conn); - return WorkerProto::Serialise::read(store, conn, std::move(path)); + return ValidPathInfo { + std::move(path), + WorkerProto::Serialise::read(store, conn), + }; } -ValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn, StorePath && path) +void WorkerProto::Serialise::write(const Store & store, WriteConn conn, const ValidPathInfo & pathInfo) +{ + WorkerProto::write(store, conn, pathInfo.path); + WorkerProto::write(store, conn, static_cast(pathInfo)); +} + + +UnkeyedValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) { auto deriver = readString(conn.from); auto narHash = Hash::parseAny(readString(conn.from), htSHA256); - ValidPathInfo info(path, narHash); + UnkeyedValidPathInfo info(narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); info.references = WorkerProto::Serialise::read(store, conn); conn.from >> info.registrationTime >> info.narSize; @@ -164,14 +174,8 @@ ValidPathInfo WorkerProto::Serialise::read(const Store & store, R return info; } -void WorkerProto::Serialise::write( - const Store & store, - WriteConn conn, - const ValidPathInfo & pathInfo, - bool includePath) +void WorkerProto::Serialise::write(const Store & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo) { - if (includePath) - conn.to << store.printStorePath(pathInfo.path); conn.to << (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "") << pathInfo.narHash.to_string(HashFormat::Base16, false); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index d35e3c678..dcd54ad16 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -32,6 +32,7 @@ struct DerivedPath; struct BuildResult; struct KeyedBuildResult; struct ValidPathInfo; +struct UnkeyedValidPathInfo; enum TrustedFlag : bool; @@ -207,6 +208,10 @@ DECLARE_WORKER_SERIALISER(BuildResult); template<> DECLARE_WORKER_SERIALISER(KeyedBuildResult); template<> +DECLARE_WORKER_SERIALISER(ValidPathInfo); +template<> +DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo); +template<> DECLARE_WORKER_SERIALISER(std::optional); template @@ -221,15 +226,4 @@ template DECLARE_WORKER_SERIALISER(std::map); #undef COMMA_ -/* These are a non-standard form for historical reasons. */ - -template<> -struct WorkerProto::Serialise -{ - static ValidPathInfo read(const Store & store, WorkerProto::ReadConn conn); - static ValidPathInfo read(const Store & store, WorkerProto::ReadConn conn, StorePath && path); - - static void write(const Store & store, WriteConn conn, const ValidPathInfo & pathInfo, bool includePath = true); -}; - } diff --git a/unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin b/unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin new file mode 100644 index 0000000000000000000000000000000000000000..e69ccbe83862b29e3f79810c0c2ff4561a42f86f GIT binary patch literal 328 zcmbu4%?`pK5QOo8D!&WM#q>Qq0R{CUX=_wYUVUsEAs(9a)(w2^a=MH48Q8;TGRcmlTf(6rfXPFjSJ46(yf&k8}#9A2z znmI2!XwnU&AmvqJYIhQk7+x#{!gQkU@$Il_5ceP*O@N zxu!r+#$?P>4g#ry<$$?w`2MnGPahlJ`HLM!?tH6m@;lyRZZ9kI;P`*_d+&94ym9_l z{ZG@h-`{zOWuKw$x?n$F_k7^>Jh*~Rnvu&5}L}Qp+gUA>+#+bF!C(UNw&;S4c literal 0 HcmV?d00001 From 201a4af9a4447a151682de6705a4a74783bf0e23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=A2=D1=80=D0=BE?= =?UTF-8?q?=D1=84=D0=B8=D0=BC=D0=BE=D0=B2?= <62882157+trofkm@users.noreply.github.com> Date: Mon, 23 Oct 2023 01:56:46 +0300 Subject: [PATCH 450/909] Clean up `app.cc` (#9201) - Rename `expected` to `expectedType` - Use early `return` and `continue` to reduce nesting --- .gitignore | 2 ++ src/nix/app.cc | 40 ++++++++++++++++++++++------------------ 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 2d3314015..04d96ca2c 100644 --- a/.gitignore +++ b/.gitignore @@ -138,7 +138,9 @@ nix-rust/target result +# IDE .vscode/ +.idea/ # clangd and possibly more .cache/ diff --git a/src/nix/app.cc b/src/nix/app.cc index 34fac9935..935ed18ec 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -20,20 +20,24 @@ StringPairs resolveRewrites( const std::vector & dependencies) { StringPairs res; - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - for (auto & dep : dependencies) { - if (auto drvDep = std::get_if(&dep.path)) { - for (auto & [ outputName, outputPath ] : drvDep->outputs) { - res.emplace( - DownstreamPlaceholder::fromSingleDerivedPathBuilt( - SingleDerivedPath::Built { - .drvPath = make_ref(drvDep->drvPath->discardOutputPath()), - .output = outputName, - }).render(), - store.printStorePath(outputPath) - ); - } - } + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + return res; + } + for (auto &dep: dependencies) { + auto drvDep = std::get_if(&dep.path); + if (!drvDep) { + continue; + } + + for (const auto & [ outputName, outputPath ] : drvDep->outputs) { + res.emplace( + DownstreamPlaceholder::fromSingleDerivedPathBuilt( + SingleDerivedPath::Built { + .drvPath = make_ref(drvDep->drvPath->discardOutputPath()), + .output = outputName, + }).render(), + store.printStorePath(outputPath) + ); } } return res; @@ -58,11 +62,11 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) auto type = cursor->getAttr("type")->getString(); - std::string expected = !attrPath.empty() && + std::string expectedType = !attrPath.empty() && (state.symbols[attrPath[0]] == "apps" || state.symbols[attrPath[0]] == "defaultApp") ? "app" : "derivation"; - if (type != expected) - throw Error("attribute '%s' should have type '%s'", cursor->getAttrPathStr(), expected); + if (type != expectedType) + throw Error("attribute '%s' should have type '%s'", cursor->getAttrPathStr(), expectedType); if (type == "app") { auto [program, context] = cursor->getAttr("program")->getStringWithContext(); @@ -91,7 +95,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) }, c.raw)); } - return UnresolvedApp{App { + return UnresolvedApp { App { .context = std::move(context2), .program = program, }}; From 256dfb98e864e474921ac5a8264e33de6978f47d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 23 Oct 2023 04:05:02 +0200 Subject: [PATCH 451/909] remove Basic Package Management section (#7974) this is the first thing most beginners see, and it misleads them into assuming `nix-env` is appropriate for doing anything but setting and reverting profile generations. this chapter is the root of most evil around the ecosystem, and today we finally close it for good. --- doc/manual/src/SUMMARY.md.in | 1 - .../package-management/basic-package-mgmt.md | 179 ------------------ 2 files changed, 180 deletions(-) delete mode 100644 doc/manual/src/package-management/basic-package-mgmt.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index e49e77cf5..60ebeb138 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -17,7 +17,6 @@ - [Upgrading Nix](installation/upgrading.md) - [Uninstalling Nix](installation/uninstall.md) - [Package Management](package-management/package-management.md) - - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) - [Garbage Collection](package-management/garbage-collection.md) - [Garbage Collector Roots](package-management/garbage-collector-roots.md) diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md deleted file mode 100644 index 07b92fb76..000000000 --- a/doc/manual/src/package-management/basic-package-mgmt.md +++ /dev/null @@ -1,179 +0,0 @@ -# Basic Package Management - -The main command for package management is -[`nix-env`](../command-ref/nix-env.md). You can use it to install, -upgrade, and erase packages, and to query what packages are installed -or are available for installation. - -In Nix, different users can have different “views” on the set of -installed applications. That is, there might be lots of applications -present on the system (possibly in many different versions), but users -can have a specific selection of those active — where “active” just -means that it appears in a directory in the user’s `PATH`. Such a view -on the set of installed applications is called a *user environment*, -which is just a directory tree consisting of symlinks to the files of -the active applications. - -Components are installed from a set of *Nix expressions* that tell Nix -how to build those packages, including, if necessary, their -dependencies. There is a collection of Nix expressions called the -Nixpkgs package collection that contains packages ranging from basic -development stuff such as GCC and Glibc, to end-user applications like -Mozilla Firefox. (Nix is however not tied to the Nixpkgs package -collection; you could write your own Nix expressions based on Nixpkgs, -or completely new ones.) - -You can manually download the latest version of Nixpkgs from -. However, it’s much more -convenient to use the Nixpkgs [*channel*](../command-ref/nix-channel.md), since it makes -it easy to stay up to date with new versions of Nixpkgs. Nixpkgs is -automatically added to your list of “subscribed” channels when you -install Nix. If this is not the case for some reason, you can add it -as follows: - -```console -$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable -$ nix-channel --update -``` - -> **Note** -> -> On NixOS, you’re automatically subscribed to a NixOS channel -> corresponding to your NixOS major release (e.g. -> ). A NixOS channel is identical -> to the Nixpkgs channel, except that it contains only Linux binaries -> and is updated only if a set of regression tests succeed. - -You can view the set of available packages in Nixpkgs: - -```console -$ nix-env --query --available --attr-path -nixpkgs.aterm aterm-2.2 -nixpkgs.bash bash-3.0 -nixpkgs.binutils binutils-2.15 -nixpkgs.bison bison-1.875d -nixpkgs.blackdown blackdown-1.4.2 -nixpkgs.bzip2 bzip2-1.0.2 -… -``` - -The flag `-q` specifies a query operation, `-a` means that you want -to show the “available” (i.e., installable) packages, as opposed to the -installed packages, and `-P` prints the attribute paths that can be used -to unambiguously select a package for installation (listed in the first column). -If you downloaded Nixpkgs yourself, or if you checked it out from GitHub, -then you need to pass the path to your Nixpkgs tree using the `-f` flag: - -```console -$ nix-env --query --available --attr-path --file /path/to/nixpkgs -aterm aterm-2.2 -bash bash-3.0 -… -``` - -where */path/to/nixpkgs* is where you’ve unpacked or checked out -Nixpkgs. - -You can filter the packages by name: - -```console -$ nix-env --query --available --attr-path firefox -nixpkgs.firefox-esr firefox-91.3.0esr -nixpkgs.firefox firefox-94.0.1 -``` - -and using regular expressions: - -```console -$ nix-env --query --available --attr-path 'firefox.*' -``` - -It is also possible to see the *status* of available packages, i.e., -whether they are installed into the user environment and/or present in -the system: - -```console -$ nix-env --query --available --attr-path --status -… --PS nixpkgs.bash bash-3.0 ---S nixpkgs.binutils binutils-2.15 -IPS nixpkgs.bison bison-1.875d -… -``` - -The first character (`I`) indicates whether the package is installed in -your current user environment. The second (`P`) indicates whether it is -present on your system (in which case installing it into your user -environment would be a very quick operation). The last one (`S`) -indicates whether there is a so-called *substitute* for the package, -which is Nix’s mechanism for doing binary deployment. It just means that -Nix knows that it can fetch a pre-built package from somewhere -(typically a network server) instead of building it locally. - -You can install a package using `nix-env --install --attr `. For instance, - -```console -$ nix-env --install --attr nixpkgs.subversion -``` - -will install the package called `subversion` from `nixpkgs` channel (which is, of course, the -[Subversion version management system](http://subversion.tigris.org/)). - -> **Note** -> -> When you ask Nix to install a package, it will first try to get it in -> pre-compiled form from a *binary cache*. By default, Nix will use the -> binary cache ; it contains binaries for most -> packages in Nixpkgs. Only if no binary is available in the binary -> cache, Nix will build the package from source. So if `nix-env -> -iA nixpkgs.subversion` results in Nix building stuff from source, then either -> the package is not built for your platform by the Nixpkgs build -> servers, or your version of Nixpkgs is too old or too new. For -> instance, if you have a very recent checkout of Nixpkgs, then the -> Nixpkgs build servers may not have had a chance to build everything -> and upload the resulting binaries to . The -> Nixpkgs channel is only updated after all binaries have been uploaded -> to the cache, so if you stick to the Nixpkgs channel (rather than -> using a Git checkout of the Nixpkgs tree), you will get binaries for -> most packages. - -Naturally, packages can also be uninstalled. Unlike when installing, you will -need to use the derivation name (though the version part can be omitted), -instead of the attribute path, as `nix-env` does not record which attribute -was used for installing: - -```console -$ nix-env --uninstall subversion -``` - -Upgrading to a new version is just as easy. If you have a new release of -Nix Packages, you can do: - -```console -$ nix-env --upgrade --attr nixpkgs.subversion -``` - -This will *only* upgrade Subversion if there is a “newer” version in the -new set of Nix expressions, as defined by some pretty arbitrary rules -regarding ordering of version numbers (which generally do what you’d -expect of them). To just unconditionally replace Subversion with -whatever version is in the Nix expressions, use `-i` instead of `-u`; -`-i` will remove whatever version is already installed. - -You can also upgrade all packages for which there are newer versions: - -```console -$ nix-env --upgrade -``` - -Sometimes it’s useful to be able to ask what `nix-env` would do, without -actually doing it. For instance, to find out what packages would be -upgraded by `nix-env --upgrade `, you can do - -```console -$ nix-env --upgrade --dry-run -(dry run; not doing anything) -upgrading `libxslt-1.1.0' to `libxslt-1.1.10' -upgrading `graphviz-1.10' to `graphviz-1.12' -upgrading `coreutils-5.0' to `coreutils-5.2.1' -``` From 34a42f0d0ac20a70fdcc3dbf99a212b523c5f2d4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Oct 2023 11:05:50 +0200 Subject: [PATCH 452/909] Move PosixSourceAccessor into its own file --- src/libfetchers/fs-input-accessor.cc | 1 + src/libutil/archive.cc | 2 +- src/libutil/posix-source-accessor.cc | 86 ++++++++++++++++++++++++++++ src/libutil/posix-source-accessor.hh | 34 +++++++++++ src/libutil/source-accessor.cc | 81 -------------------------- src/libutil/source-accessor.hh | 27 --------- 6 files changed, 122 insertions(+), 109 deletions(-) create mode 100644 src/libutil/posix-source-accessor.cc create mode 100644 src/libutil/posix-source-accessor.hh diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index 3444c4643..7638d2d82 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -1,4 +1,5 @@ #include "fs-input-accessor.hh" +#include "posix-source-accessor.hh" #include "store-api.hh" namespace nix { diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 0cd54e5db..3b1a1e0ef 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -14,7 +14,7 @@ #include "archive.hh" #include "util.hh" #include "config.hh" -#include "source-accessor.hh" +#include "posix-source-accessor.hh" namespace nix { diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc new file mode 100644 index 000000000..48b4fe626 --- /dev/null +++ b/src/libutil/posix-source-accessor.cc @@ -0,0 +1,86 @@ +#include "posix-source-accessor.hh" + +namespace nix { + +void PosixSourceAccessor::readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) +{ + // FIXME: add O_NOFOLLOW since symlinks should be resolved by the + // caller? + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%1%'", path); + + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("statting file"); + + sizeCallback(st.st_size); + + off_t left = st.st_size; + + std::vector buf(64 * 1024); + while (left) { + checkInterrupt(); + ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size())); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading from file '%s'", showPath(path)); + } + else if (rd == 0) + throw SysError("unexpected end-of-file reading '%s'", showPath(path)); + else { + assert(rd <= left); + sink({(char *) buf.data(), (size_t) rd}); + left -= rd; + } + } +} + +bool PosixSourceAccessor::pathExists(const CanonPath & path) +{ + return nix::pathExists(path.abs()); +} + +SourceAccessor::Stat PosixSourceAccessor::lstat(const CanonPath & path) +{ + auto st = nix::lstat(path.abs()); + mtime = std::max(mtime, st.st_mtime); + return Stat { + .type = + S_ISREG(st.st_mode) ? tRegular : + S_ISDIR(st.st_mode) ? tDirectory : + S_ISLNK(st.st_mode) ? tSymlink : + tMisc, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR + }; +} + +SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path) +{ + DirEntries res; + for (auto & entry : nix::readDirectory(path.abs())) { + std::optional type; + switch (entry.type) { + case DT_REG: type = Type::tRegular; break; + case DT_LNK: type = Type::tSymlink; break; + case DT_DIR: type = Type::tDirectory; break; + } + res.emplace(entry.name, type); + } + return res; +} + +std::string PosixSourceAccessor::readLink(const CanonPath & path) +{ + return nix::readLink(path.abs()); +} + +std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) +{ + return path; +} + +} diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh new file mode 100644 index 000000000..608f96ee2 --- /dev/null +++ b/src/libutil/posix-source-accessor.hh @@ -0,0 +1,34 @@ +#pragma once + +#include "source-accessor.hh" + +namespace nix { + +/** + * A source accessor that uses the Unix filesystem. + */ +struct PosixSourceAccessor : SourceAccessor +{ + /** + * The most recent mtime seen by lstat(). This is a hack to + * support dumpPathAndGetMtime(). Should remove this eventually. + */ + time_t mtime = 0; + + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override; + + bool pathExists(const CanonPath & path) override; + + Stat lstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::optional getPhysicalPath(const CanonPath & path) override; +}; + +} diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index 2d03d3d7a..d168a9667 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -55,85 +55,4 @@ std::string SourceAccessor::showPath(const CanonPath & path) return path.abs(); } -void PosixSourceAccessor::readFile( - const CanonPath & path, - Sink & sink, - std::function sizeCallback) -{ - // FIXME: add O_NOFOLLOW since symlinks should be resolved by the - // caller? - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) - throw SysError("opening file '%1%'", path); - - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("statting file"); - - sizeCallback(st.st_size); - - off_t left = st.st_size; - - std::vector buf(64 * 1024); - while (left) { - checkInterrupt(); - ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size())); - if (rd == -1) { - if (errno != EINTR) - throw SysError("reading from file '%s'", showPath(path)); - } - else if (rd == 0) - throw SysError("unexpected end-of-file reading '%s'", showPath(path)); - else { - assert(rd <= left); - sink({(char *) buf.data(), (size_t) rd}); - left -= rd; - } - } -} - -bool PosixSourceAccessor::pathExists(const CanonPath & path) -{ - return nix::pathExists(path.abs()); -} - -SourceAccessor::Stat PosixSourceAccessor::lstat(const CanonPath & path) -{ - auto st = nix::lstat(path.abs()); - mtime = std::max(mtime, st.st_mtime); - return Stat { - .type = - S_ISREG(st.st_mode) ? tRegular : - S_ISDIR(st.st_mode) ? tDirectory : - S_ISLNK(st.st_mode) ? tSymlink : - tMisc, - .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR - }; -} - -SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path) -{ - DirEntries res; - for (auto & entry : nix::readDirectory(path.abs())) { - std::optional type; - switch (entry.type) { - case DT_REG: type = Type::tRegular; break; - case DT_LNK: type = Type::tSymlink; break; - case DT_DIR: type = Type::tDirectory; break; - } - res.emplace(entry.name, type); - } - return res; -} - -std::string PosixSourceAccessor::readLink(const CanonPath & path) -{ - return nix::readLink(path.abs()); -} - -std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) -{ - return path; -} - } diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index f3504c9bb..fd823aa39 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -104,31 +104,4 @@ struct SourceAccessor virtual std::string showPath(const CanonPath & path); }; -/** - * A source accessor that uses the Unix filesystem. - */ -struct PosixSourceAccessor : SourceAccessor -{ - /** - * The most recent mtime seen by lstat(). This is a hack to - * support dumpPathAndGetMtime(). Should remove this eventually. - */ - time_t mtime = 0; - - void readFile( - const CanonPath & path, - Sink & sink, - std::function sizeCallback) override; - - bool pathExists(const CanonPath & path) override; - - Stat lstat(const CanonPath & path) override; - - DirEntries readDirectory(const CanonPath & path) override; - - std::string readLink(const CanonPath & path) override; - - std::optional getPhysicalPath(const CanonPath & path) override; -}; - } From b461cac21aa5b9127d230ed3f1a1ca8e8266fb60 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 23 Oct 2023 09:03:11 -0400 Subject: [PATCH 453/909] Overhaul completions, redo #6693 (#8131) As I complained in https://github.com/NixOS/nix/pull/6784#issuecomment-1421777030 (a comment on the wrong PR, sorry again!), #6693 introduced a second completions mechanism to fix a bug. Having two completion mechanisms isn't so nice. As @thufschmitt also pointed out, it was a bummer to go from `FlakeRef` to `std::string` when collecting flake refs. Now it is `FlakeRefs` again. The underlying issue that sought to work around was that completion of arguments not at the end can still benefit from the information from latter arguments. To fix this better, we rip out that change and simply defer all completion processing until after all the (regular, already-complete) arguments have been passed. In addition, I noticed the original completion logic used some global variables. I do not like global variables, because even if they save lines of code, they also obfuscate the architecture of the code. I got rid of them moved them to a new `RootArgs` class, which now has `parseCmdline` instead of `Args`. The idea is that we have many argument parsers from subcommands and what-not, but only one root args that owns the other per actual parsing invocation. The state that was global is now part of the root args instead. This did, admittedly, add a bunch of new code. And I do feel bad about that. So I went and added a lot of API docs to try to at least make the current state of things clear to the next person. -- This is needed for RFC 134 (tracking issue #7868). It was very hard to modularize `Installable` parsing when there were two completion arguments. I wouldn't go as far as to say it is *easy* now, but at least it is less hard (and the completions test finally passed). Co-authored-by: Valentin Gagarin --- src/libcmd/command.hh | 26 ++++---- src/libcmd/common-eval-args.cc | 4 +- src/libcmd/installables.cc | 114 +++++++++++++++++++-------------- src/libmain/common-args.cc | 9 +-- src/libmain/shared.hh | 3 +- src/libutil/args.cc | 111 ++++++++++++++++++++------------ src/libutil/args.hh | 109 ++++++++++++++++++++----------- src/libutil/args/root.hh | 72 +++++++++++++++++++++ src/libutil/local.mk | 5 ++ src/nix/bundle.cc | 4 +- src/nix/flake.cc | 14 ++-- src/nix/main.cc | 27 ++++---- src/nix/registry.cc | 4 +- src/nix/why-depends.cc | 8 +-- 14 files changed, 329 insertions(+), 181 deletions(-) create mode 100644 src/libutil/args/root.hh diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 5c4569001..dafc0db3b 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -97,8 +97,6 @@ struct MixFlakeOptions : virtual Args, EvalCommand { flake::LockFlags lockFlags; - std::optional needsFlakeInputCompletion = {}; - MixFlakeOptions(); /** @@ -109,12 +107,8 @@ struct MixFlakeOptions : virtual Args, EvalCommand * command is operating with (presumably specified via some other * arguments) so that the completions for these flags can use them. */ - virtual std::vector getFlakesForCompletion() + virtual std::vector getFlakeRefsForCompletion() { return {}; } - - void completeFlakeInput(std::string_view prefix); - - void completionHook() override; }; struct SourceExprCommand : virtual Args, MixFlakeOptions @@ -137,7 +131,13 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions /** * Complete an installable from the given prefix. */ - void completeInstallable(std::string_view prefix); + void completeInstallable(AddCompletions & completions, std::string_view prefix); + + /** + * Convenience wrapper around the underlying function to make setting the + * callback easier. + */ + CompleterClosure getCompleteInstallable(); }; /** @@ -170,7 +170,7 @@ struct RawInstallablesCommand : virtual Args, SourceExprCommand bool readFromStdIn = false; - std::vector getFlakesForCompletion() override; + std::vector getFlakeRefsForCompletion() override; private: @@ -199,10 +199,7 @@ struct InstallableCommand : virtual Args, SourceExprCommand void run(ref store) override; - std::vector getFlakesForCompletion() override - { - return {_installable}; - } + std::vector getFlakeRefsForCompletion() override; private: @@ -329,9 +326,10 @@ struct MixEnvironment : virtual Args { void setEnviron(); }; -void completeFlakeRef(ref store, std::string_view prefix); +void completeFlakeRef(AddCompletions & completions, ref store, std::string_view prefix); void completeFlakeRefWithFragment( + AddCompletions & completions, ref evalState, flake::LockFlags lockFlags, Strings attrPathPrefixes, diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index e6df49a7a..e53bc4c01 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -133,8 +133,8 @@ MixEvalArgs::MixEvalArgs() if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); }}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(openStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, openStore(), prefix); }} }); diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 01c01441c..eff18bbf6 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -28,6 +28,20 @@ namespace nix { +static void completeFlakeInputPath( + AddCompletions & completions, + ref evalState, + const std::vector & flakeRefs, + std::string_view prefix) +{ + for (auto & flakeRef : flakeRefs) { + auto flake = flake::getFlake(*evalState, flakeRef, true); + for (auto & input : flake.inputs) + if (hasPrefix(input.first, prefix)) + completions.add(input.first); + } +} + MixFlakeOptions::MixFlakeOptions() { auto category = "Common flake-related options"; @@ -79,8 +93,8 @@ MixFlakeOptions::MixFlakeOptions() .handler = {[&](std::string s) { lockFlags.inputUpdates.insert(flake::parseInputPath(s)); }}, - .completer = {[&](size_t, std::string_view prefix) { - needsFlakeInputCompletion = {std::string(prefix)}; + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); }} }); @@ -95,11 +109,12 @@ MixFlakeOptions::MixFlakeOptions() flake::parseInputPath(inputPath), parseFlakeRef(flakeRef, absPath("."), true)); }}, - .completer = {[&](size_t n, std::string_view prefix) { - if (n == 0) - needsFlakeInputCompletion = {std::string(prefix)}; - else if (n == 1) - completeFlakeRef(getEvalState()->store, prefix); + .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { + if (n == 0) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); + } else if (n == 1) { + completeFlakeRef(completions, getEvalState()->store, prefix); + } }} }); @@ -146,30 +161,12 @@ MixFlakeOptions::MixFlakeOptions() } } }}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getEvalState()->store, prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getEvalState()->store, prefix); }} }); } -void MixFlakeOptions::completeFlakeInput(std::string_view prefix) -{ - auto evalState = getEvalState(); - for (auto & flakeRefS : getFlakesForCompletion()) { - auto flakeRef = parseFlakeRefWithFragment(expandTilde(flakeRefS), absPath(".")).first; - auto flake = flake::getFlake(*evalState, flakeRef, true); - for (auto & input : flake.inputs) - if (hasPrefix(input.first, prefix)) - completions->add(input.first); - } -} - -void MixFlakeOptions::completionHook() -{ - if (auto & prefix = needsFlakeInputCompletion) - completeFlakeInput(*prefix); -} - SourceExprCommand::SourceExprCommand() { addFlag({ @@ -226,11 +223,18 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes() }; } -void SourceExprCommand::completeInstallable(std::string_view prefix) +Args::CompleterClosure SourceExprCommand::getCompleteInstallable() +{ + return [this](AddCompletions & completions, size_t, std::string_view prefix) { + completeInstallable(completions, prefix); + }; +} + +void SourceExprCommand::completeInstallable(AddCompletions & completions, std::string_view prefix) { try { if (file) { - completionType = ctAttrs; + completions.setType(AddCompletions::Type::Attrs); evalSettings.pureEval = false; auto state = getEvalState(); @@ -265,14 +269,15 @@ void SourceExprCommand::completeInstallable(std::string_view prefix) std::string name = state->symbols[i.name]; if (name.find(searchWord) == 0) { if (prefix_ == "") - completions->add(name); + completions.add(name); else - completions->add(prefix_ + "." + name); + completions.add(prefix_ + "." + name); } } } } else { completeFlakeRefWithFragment( + completions, getEvalState(), lockFlags, getDefaultFlakeAttrPathPrefixes(), @@ -285,6 +290,7 @@ void SourceExprCommand::completeInstallable(std::string_view prefix) } void completeFlakeRefWithFragment( + AddCompletions & completions, ref evalState, flake::LockFlags lockFlags, Strings attrPathPrefixes, @@ -296,9 +302,9 @@ void completeFlakeRefWithFragment( try { auto hash = prefix.find('#'); if (hash == std::string::npos) { - completeFlakeRef(evalState->store, prefix); + completeFlakeRef(completions, evalState->store, prefix); } else { - completionType = ctAttrs; + completions.setType(AddCompletions::Type::Attrs); auto fragment = prefix.substr(hash + 1); std::string prefixRoot = ""; @@ -341,7 +347,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); - completions->add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); + completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } } @@ -352,7 +358,7 @@ void completeFlakeRefWithFragment( for (auto & attrPath : defaultFlakeAttrPaths) { auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath)); if (!attr) continue; - completions->add(flakeRefS + "#" + prefixRoot); + completions.add(flakeRefS + "#" + prefixRoot); } } } @@ -361,15 +367,15 @@ void completeFlakeRefWithFragment( } } -void completeFlakeRef(ref store, std::string_view prefix) +void completeFlakeRef(AddCompletions & completions, ref store, std::string_view prefix) { if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) return; if (prefix == "") - completions->add("."); + completions.add("."); - completeDir(0, prefix); + Args::completeDir(completions, 0, prefix); /* Look for registry entries that match the prefix. */ for (auto & registry : fetchers::getRegistries(store)) { @@ -378,10 +384,10 @@ void completeFlakeRef(ref store, std::string_view prefix) if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { std::string from2(from, 6); if (hasPrefix(from2, prefix)) - completions->add(from2); + completions.add(from2); } else { if (hasPrefix(from, prefix)) - completions->add(from); + completions.add(from); } } } @@ -747,9 +753,7 @@ RawInstallablesCommand::RawInstallablesCommand() expectArgs({ .label = "installables", .handler = {&rawInstallables}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); } @@ -762,6 +766,17 @@ void RawInstallablesCommand::applyDefaultInstallables(std::vector & } } +std::vector RawInstallablesCommand::getFlakeRefsForCompletion() +{ + applyDefaultInstallables(rawInstallables); + std::vector res; + for (auto i : rawInstallables) + res.push_back(parseFlakeRefWithFragment( + expandTilde(i), + absPath(".")).first); + return res; +} + void RawInstallablesCommand::run(ref store) { if (readFromStdIn && !isatty(STDIN_FILENO)) { @@ -775,10 +790,13 @@ void RawInstallablesCommand::run(ref store) run(store, std::move(rawInstallables)); } -std::vector RawInstallablesCommand::getFlakesForCompletion() +std::vector InstallableCommand::getFlakeRefsForCompletion() { - applyDefaultInstallables(rawInstallables); - return rawInstallables; + return { + parseFlakeRefWithFragment( + expandTilde(_installable), + absPath(".")).first + }; } void InstallablesCommand::run(ref store, std::vector && rawInstallables) @@ -794,9 +812,7 @@ InstallableCommand::InstallableCommand() .label = "installable", .optional = true, .handler = {&_installable}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); } diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index f92920d18..205b77808 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,4 +1,5 @@ #include "common-args.hh" +#include "args/root.hh" #include "globals.hh" #include "loggers.hh" @@ -34,21 +35,21 @@ MixCommonArgs::MixCommonArgs(const std::string & programName) .description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).", .category = miscCategory, .labels = {"name", "value"}, - .handler = {[](std::string name, std::string value) { + .handler = {[this](std::string name, std::string value) { try { globalConfig.set(name, value); } catch (UsageError & e) { - if (!completions) + if (!getRoot().completions) warn(e.what()); } }}, - .completer = [](size_t index, std::string_view prefix) { + .completer = [](AddCompletions & completions, size_t index, std::string_view prefix) { if (index == 0) { std::map settings; globalConfig.getSettings(settings); for (auto & s : settings) if (hasPrefix(s.first, prefix)) - completions->add(s.first, fmt("Set the `%s` setting.", s.first)); + completions.add(s.first, fmt("Set the `%s` setting.", s.first)); } } }); diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 9415be78a..3159fe479 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -3,6 +3,7 @@ #include "util.hh" #include "args.hh" +#include "args/root.hh" #include "common-args.hh" #include "path.hh" #include "derived-path.hh" @@ -66,7 +67,7 @@ template N getIntArg(const std::string & opt, } -struct LegacyArgs : public MixCommonArgs +struct LegacyArgs : public MixCommonArgs, public RootArgs { std::function parseArg; diff --git a/src/libutil/args.cc b/src/libutil/args.cc index e410c7eec..6bc3cae07 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -1,4 +1,5 @@ #include "args.hh" +#include "args/root.hh" #include "hash.hh" #include "json-utils.hh" @@ -26,6 +27,11 @@ void Args::removeFlag(const std::string & longName) longFlags.erase(flag); } +void Completions::setType(AddCompletions::Type t) +{ + type = t; +} + void Completions::add(std::string completion, std::string description) { description = trim(description); @@ -37,7 +43,7 @@ void Completions::add(std::string completion, std::string description) if (needs_ellipsis) description.append(" [...]"); } - insert(Completion { + completions.insert(Completion { .completion = completion, .description = description }); @@ -46,12 +52,20 @@ void Completions::add(std::string completion, std::string description) bool Completion::operator<(const Completion & other) const { return completion < other.completion || (completion == other.completion && description < other.description); } -CompletionType completionType = ctNormal; -std::shared_ptr completions; - std::string completionMarker = "___COMPLETE___"; -static std::optional needsCompletion(std::string_view s) +RootArgs & Args::getRoot() +{ + Args * p = this; + while (p->parent) + p = p->parent; + + auto * res = dynamic_cast(p); + assert(res); + return *res; +} + +std::optional RootArgs::needsCompletion(std::string_view s) { if (!completions) return {}; auto i = s.find(completionMarker); @@ -60,7 +74,7 @@ static std::optional needsCompletion(std::string_view s) return {}; } -void Args::parseCmdline(const Strings & _cmdline) +void RootArgs::parseCmdline(const Strings & _cmdline) { Strings pendingArgs; bool dashDash = false; @@ -71,7 +85,7 @@ void Args::parseCmdline(const Strings & _cmdline) size_t n = std::stoi(*s); assert(n > 0 && n <= cmdline.size()); *std::next(cmdline.begin(), n - 1) += completionMarker; - completions = std::make_shared(); + completions = std::make_shared(); verbosity = lvlError; } @@ -125,17 +139,23 @@ void Args::parseCmdline(const Strings & _cmdline) for (auto & f : flagExperimentalFeatures) experimentalFeatureSettings.require(f); + /* Now that all the other args are processed, run the deferred completions. + */ + for (auto d : deferredCompletions) + d.completer(*completions, d.n, d.prefix); } bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) { assert(pos != end); + auto & rootArgs = getRoot(); + auto process = [&](const std::string & name, const Flag & flag) -> bool { ++pos; if (auto & f = flag.experimentalFeature) - flagExperimentalFeatures.insert(*f); + rootArgs.flagExperimentalFeatures.insert(*f); std::vector args; bool anyCompleted = false; @@ -146,10 +166,15 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) "flag '%s' requires %d argument(s), but only %d were given", name, flag.handler.arity, n); } - if (auto prefix = needsCompletion(*pos)) { + if (auto prefix = rootArgs.needsCompletion(*pos)) { anyCompleted = true; - if (flag.completer) - flag.completer(n, *prefix); + if (flag.completer) { + rootArgs.deferredCompletions.push_back({ + .completer = flag.completer, + .n = n, + .prefix = *prefix, + }); + } } args.push_back(*pos++); } @@ -159,14 +184,14 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) }; if (std::string(*pos, 0, 2) == "--") { - if (auto prefix = needsCompletion(*pos)) { + if (auto prefix = rootArgs.needsCompletion(*pos)) { for (auto & [name, flag] : longFlags) { if (!hiddenCategories.count(flag->category) && hasPrefix(name, std::string(*prefix, 2))) { if (auto & f = flag->experimentalFeature) - flagExperimentalFeatures.insert(*f); - completions->add("--" + name, flag->description); + rootArgs.flagExperimentalFeatures.insert(*f); + rootArgs.completions->add("--" + name, flag->description); } } return false; @@ -183,12 +208,12 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) return process(std::string("-") + c, *i->second); } - if (auto prefix = needsCompletion(*pos)) { + if (auto prefix = rootArgs.needsCompletion(*pos)) { if (prefix == "-") { - completions->add("--"); + rootArgs.completions->add("--"); for (auto & [flagName, flag] : shortFlags) if (experimentalFeatureSettings.isEnabled(flag->experimentalFeature)) - completions->add(std::string("-") + flagName, flag->description); + rootArgs.completions->add(std::string("-") + flagName, flag->description); } } @@ -203,6 +228,8 @@ bool Args::processArgs(const Strings & args, bool finish) return true; } + auto & rootArgs = getRoot(); + auto & exp = expectedArgs.front(); bool res = false; @@ -211,15 +238,23 @@ bool Args::processArgs(const Strings & args, bool finish) (exp.handler.arity != ArityAny && args.size() == exp.handler.arity)) { std::vector ss; + bool anyCompleted = false; for (const auto &[n, s] : enumerate(args)) { - if (auto prefix = needsCompletion(s)) { + if (auto prefix = rootArgs.needsCompletion(s)) { + anyCompleted = true; ss.push_back(*prefix); - if (exp.completer) - exp.completer(n, *prefix); + if (exp.completer) { + rootArgs.deferredCompletions.push_back({ + .completer = exp.completer, + .n = n, + .prefix = *prefix, + }); + } } else ss.push_back(s); } - exp.handler.fun(ss); + if (!anyCompleted) + exp.handler.fun(ss); expectedArgs.pop_front(); res = true; } @@ -271,11 +306,11 @@ nlohmann::json Args::toJSON() return res; } -static void hashTypeCompleter(size_t index, std::string_view prefix) +static void hashTypeCompleter(AddCompletions & completions, size_t index, std::string_view prefix) { for (auto & type : hashTypes) if (hasPrefix(type, prefix)) - completions->add(type); + completions.add(type); } Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) @@ -287,7 +322,7 @@ Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) .handler = {[ht](std::string s) { *ht = parseHashType(s); }}, - .completer = hashTypeCompleter + .completer = hashTypeCompleter, }; } @@ -300,13 +335,13 @@ Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional< .handler = {[oht](std::string s) { *oht = std::optional { parseHashType(s) }; }}, - .completer = hashTypeCompleter + .completer = hashTypeCompleter, }; } -static void _completePath(std::string_view prefix, bool onlyDirs) +static void _completePath(AddCompletions & completions, std::string_view prefix, bool onlyDirs) { - completionType = ctFilenames; + completions.setType(Completions::Type::Filenames); glob_t globbuf; int flags = GLOB_NOESCAPE; #ifdef GLOB_ONLYDIR @@ -320,20 +355,20 @@ static void _completePath(std::string_view prefix, bool onlyDirs) auto st = stat(globbuf.gl_pathv[i]); if (!S_ISDIR(st.st_mode)) continue; } - completions->add(globbuf.gl_pathv[i]); + completions.add(globbuf.gl_pathv[i]); } } globfree(&globbuf); } -void completePath(size_t, std::string_view prefix) +void Args::completePath(AddCompletions & completions, size_t, std::string_view prefix) { - _completePath(prefix, false); + _completePath(completions, prefix, false); } -void completeDir(size_t, std::string_view prefix) +void Args::completeDir(AddCompletions & completions, size_t, std::string_view prefix) { - _completePath(prefix, true); + _completePath(completions, prefix, true); } Strings argvToStrings(int argc, char * * argv) @@ -368,10 +403,10 @@ MultiCommand::MultiCommand(const Commands & commands_) command = {s, i->second()}; command->second->parent = this; }}, - .completer = {[&](size_t, std::string_view prefix) { + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { for (auto & [name, command] : commands) if (hasPrefix(name, prefix)) - completions->add(name); + completions.add(name); }} }); @@ -393,14 +428,6 @@ bool MultiCommand::processArgs(const Strings & args, bool finish) return Args::processArgs(args, finish); } -void MultiCommand::completionHook() -{ - if (command) - return command->second->completionHook(); - else - return Args::completionHook(); -} - nlohmann::json MultiCommand::toJSON() { auto cmds = nlohmann::json::object(); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 6457cceed..a5d7cbe4a 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -15,16 +15,14 @@ enum HashType : char; class MultiCommand; +class RootArgs; + +class AddCompletions; + class Args { public: - /** - * Parse the command line, throwing a UsageError if something goes - * wrong. - */ - void parseCmdline(const Strings & cmdline); - /** * Return a short one-line description of the command. */ @@ -123,6 +121,25 @@ protected: { } }; + /** + * The basic function type of the completion callback. + * + * Used to define `CompleterClosure` and some common case completers + * that individual flags/arguments can use. + * + * The `AddCompletions` that is passed is an interface to the state + * stored as part of the root command + */ + typedef void CompleterFun(AddCompletions &, size_t, std::string_view); + + /** + * The closure type of the completion callback. + * + * This is what is actually stored as part of each Flag / Expected + * Arg. + */ + typedef std::function CompleterClosure; + /** * Description of flags / options * @@ -140,7 +157,7 @@ protected: std::string category; Strings labels; Handler handler; - std::function completer; + CompleterClosure completer; std::optional experimentalFeature; @@ -177,7 +194,7 @@ protected: std::string label; bool optional = false; Handler handler; - std::function completer; + CompleterClosure completer; }; /** @@ -211,13 +228,6 @@ protected: */ virtual void initialFlagsProcessed() {} - /** - * Called after the command line has been processed if we need to generate - * completions. Useful for commands that need to know the whole command line - * in order to know what completions to generate. - */ - virtual void completionHook() { } - public: void addFlag(Flag && flag); @@ -252,24 +262,30 @@ public: }); } + static CompleterFun completePath; + + static CompleterFun completeDir; + virtual nlohmann::json toJSON(); friend class MultiCommand; /** * The parent command, used if this is a subcommand. + * + * Invariant: An Args with a null parent must also be a RootArgs + * + * \todo this would probably be better in the CommandClass. + * getRoot() could be an abstract method that peels off at most one + * layer before recuring. */ MultiCommand * parent = nullptr; -private: - /** - * Experimental features needed when parsing args. These are checked - * after flag parsing is completed in order to support enabling - * experimental features coming after the flag that needs the - * experimental feature. + * Traverse parent pointers until we find the \ref RootArgs "root + * arguments" object. */ - std::set flagExperimentalFeatures; + RootArgs & getRoot(); }; /** @@ -320,8 +336,6 @@ public: bool processArgs(const Strings & args, bool finish) override; - void completionHook() override; - nlohmann::json toJSON() override; }; @@ -333,21 +347,40 @@ struct Completion { bool operator<(const Completion & other) const; }; -class Completions : public std::set { + +/** + * The abstract interface for completions callbacks + * + * The idea is to restrict the callback so it can only add additional + * completions to the collection, or set the completion type. By making + * it go through this interface, the callback cannot make any other + * changes, or even view the completions / completion type that have + * been set so far. + */ +class AddCompletions +{ public: - void add(std::string completion, std::string description = ""); + + /** + * The type of completion we are collecting. + */ + enum class Type { + Normal, + Filenames, + Attrs, + }; + + /** + * Set the type of the completions being collected + * + * \todo it should not be possible to change the type after it has been set. + */ + virtual void setType(Type type) = 0; + + /** + * Add a single completion to the collection + */ + virtual void add(std::string completion, std::string description = "") = 0; }; -extern std::shared_ptr completions; - -enum CompletionType { - ctNormal, - ctFilenames, - ctAttrs -}; -extern CompletionType completionType; - -void completePath(size_t, std::string_view prefix); - -void completeDir(size_t, std::string_view prefix); } diff --git a/src/libutil/args/root.hh b/src/libutil/args/root.hh new file mode 100644 index 000000000..bb98732a1 --- /dev/null +++ b/src/libutil/args/root.hh @@ -0,0 +1,72 @@ +#pragma once + +#include "args.hh" + +namespace nix { + +/** + * The concrete implementation of a collection of completions. + * + * This is exposed so that the main entry point can print out the + * collected completions. + */ +struct Completions final : AddCompletions +{ + std::set completions; + Type type = Type::Normal; + + void setType(Type type) override; + void add(std::string completion, std::string description = "") override; +}; + +/** + * The outermost Args object. This is the one we will actually parse a command + * line with, whereas the inner ones (if they exists) are subcommands (and this + * is also a MultiCommand or something like it). + * + * This Args contains completions state shared between it and all of its + * descendent Args. + */ +class RootArgs : virtual public Args +{ +public: + /** Parse the command line, throwing a UsageError if something goes + * wrong. + */ + void parseCmdline(const Strings & cmdline); + + std::shared_ptr completions; + +protected: + + friend class Args; + + /** + * A pointer to the completion and its two arguments; a thunk; + */ + struct DeferredCompletion { + const CompleterClosure & completer; + size_t n; + std::string prefix; + }; + + /** + * Completions are run after all args and flags are parsed, so completions + * of earlier arguments can benefit from later arguments. + */ + std::vector deferredCompletions; + + /** + * Experimental features needed when parsing args. These are checked + * after flag parsing is completed in order to support enabling + * experimental features coming after the flag that needs the + * experimental feature. + */ + std::set flagExperimentalFeatures; + +private: + + std::optional needsCompletion(std::string_view s); +}; + +} diff --git a/src/libutil/local.mk b/src/libutil/local.mk index f880c0fc5..81efaafec 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,8 +6,13 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) +libutil_CXXFLAGS += -I src/libutil + libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context +$(foreach i, $(wildcard $(d)/args/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/args, 0644))) + ifeq ($(HAVE_LIBCPUID), 1) libutil_LDFLAGS += -lcpuid endif diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index fbc83b08e..504e35c81 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -21,8 +21,8 @@ struct CmdBundle : InstallableValueCommand .description = fmt("Use a custom bundler instead of the default (`%s`).", bundler), .labels = {"flake-url"}, .handler = {&bundler}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); }} }); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index b4e4156c0..0116eff2e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -36,8 +36,8 @@ public: .label = "flake-url", .optional = true, .handler = {&flakeUrl}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); }} }); } @@ -52,9 +52,12 @@ public: return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags); } - std::vector getFlakesForCompletion() override + std::vector getFlakeRefsForCompletion() override { - return {flakeUrl}; + return { + // Like getFlakeRef but with expandTilde calld first + parseFlakeRef(expandTilde(flakeUrl), absPath(".")) + }; } }; @@ -762,8 +765,9 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand .description = "The template to use.", .labels = {"template"}, .handler = {&templateUrl}, - .completer = {[&](size_t, std::string_view prefix) { + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { completeFlakeRefWithFragment( + completions, getEvalState(), lockFlags, defaultTemplateAttrPathsPrefixes, diff --git a/src/nix/main.cc b/src/nix/main.cc index ee3878cc7..ffba10099 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -1,5 +1,6 @@ #include +#include "args/root.hh" #include "command.hh" #include "common-args.hh" #include "eval.hh" @@ -56,7 +57,7 @@ static bool haveInternet() std::string programPath; -struct NixArgs : virtual MultiCommand, virtual MixCommonArgs +struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs { bool useNet = true; bool refresh = false; @@ -241,10 +242,7 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) static NixArgs & getNixArgs(Command & cmd) { - assert(cmd.parent); - MultiCommand * toplevel = cmd.parent; - while (toplevel->parent) toplevel = toplevel->parent; - return dynamic_cast(*toplevel); + return dynamic_cast(cmd.getRoot()); } struct CmdHelp : Command @@ -412,16 +410,16 @@ void mainWrapped(int argc, char * * argv) Finally printCompletions([&]() { - if (completions) { - switch (completionType) { - case ctNormal: + if (args.completions) { + switch (args.completions->type) { + case Completions::Type::Normal: logger->cout("normal"); break; - case ctFilenames: + case Completions::Type::Filenames: logger->cout("filenames"); break; - case ctAttrs: + case Completions::Type::Attrs: logger->cout("attrs"); break; } - for (auto & s : *completions) + for (auto & s : args.completions->completions) logger->cout(s.completion + "\t" + trim(s.description)); } }); @@ -429,7 +427,7 @@ void mainWrapped(int argc, char * * argv) try { args.parseCmdline(argvToStrings(argc, argv)); } catch (UsageError &) { - if (!args.helpRequested && !completions) throw; + if (!args.helpRequested && !args.completions) throw; } if (args.helpRequested) { @@ -446,10 +444,7 @@ void mainWrapped(int argc, char * * argv) return; } - if (completions) { - args.completionHook(); - return; - } + if (args.completions) return; if (args.showVersion) { printVersion(programName); diff --git a/src/nix/registry.cc b/src/nix/registry.cc index cb94bbd31..f509ccae8 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -175,8 +175,8 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand .label = "locked", .optional = true, .handler = {&locked}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); }} }); } diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 592de773c..055cf6d0d 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -38,17 +38,13 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions expectArgs({ .label = "package", .handler = {&_package}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); expectArgs({ .label = "dependency", .handler = {&_dependency}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); addFlag({ From fa9642ec459859557ce29e5f68413cf3b0846fd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Wed, 7 Jun 2023 18:12:35 +0200 Subject: [PATCH 454/909] nix-shell: support single quotes in shebangs Single quotes are a basic feature of shell syntax that people expect to work. They are also more convenient for writing literal code expressions with less escaping. --- doc/manual/src/command-ref/nix-shell.md | 6 +++--- doc/manual/src/release-notes/rl-next.md | 2 ++ src/nix-build/nix-build.cc | 26 ++++++++++++++++++------- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 195b72be5..1eaf3c36a 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -235,14 +235,14 @@ package like Terraform: ```bash #! /usr/bin/env nix-shell -#! nix-shell -i bash --packages "terraform.withPlugins (plugins: [ plugins.openstack ])" +#! nix-shell -i bash --packages 'terraform.withPlugins (plugins: [ plugins.openstack ])' terraform apply ``` > **Note** > -> You must use double quotes (`"`) when passing a simple Nix expression +> You must use single or double quotes (`'`, `"`) when passing a simple Nix expression > in a nix-shell shebang. Finally, using the merging of multiple nix-shell shebangs the following @@ -251,7 +251,7 @@ branch): ```haskell #! /usr/bin/env nix-shell -#! nix-shell -i runghc --packages "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])" +#! nix-shell -i runghc --packages 'haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])' #! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.03.tar.gz import Network.Curl.Download diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index d0b516dfb..276252c37 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -12,4 +12,6 @@ - Introduce a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash). +- `nix-shell` shebang lines now support single-quoted arguments. + - `builtins.fetchTree` is now marked as stable. diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 2895c5e3c..ae05444b7 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -40,7 +40,8 @@ static std::vector shellwords(const std::string & s) std::string cur; enum state { sBegin, - sQuote + sSingleQuote, + sDoubleQuote }; state st = sBegin; auto it = begin; @@ -56,15 +57,26 @@ static std::vector shellwords(const std::string & s) } } switch (*it) { + case '\'': + if (st != sDoubleQuote) { + cur.append(begin, it); + begin = it + 1; + st = st == sBegin ? sSingleQuote : sBegin; + } + break; case '"': - cur.append(begin, it); - begin = it + 1; - st = st == sBegin ? sQuote : sBegin; + if (st != sSingleQuote) { + cur.append(begin, it); + begin = it + 1; + st = st == sBegin ? sDoubleQuote : sBegin; + } break; case '\\': - /* perl shellwords mostly just treats the next char as part of the string with no special processing */ - cur.append(begin, it); - begin = ++it; + if (st != sSingleQuote) { + /* perl shellwords mostly just treats the next char as part of the string with no special processing */ + cur.append(begin, it); + begin = ++it; + } break; } } From 595010b631b0f92ee6eb4afb8494bc9c73e8513b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Thu, 8 Jun 2023 03:11:04 +0200 Subject: [PATCH 455/909] nix-shell: fix shebang whitespace parsing Leading whitespace after `nix-shell` used to produce an empty argument, while an empty argument at the end of the line was ignored. Fix the first issue by consuming the initial whitespace before calling shellwords; fix the second issue by returning immediately if whitespace is found at the end of the string instead of checking for an empty string. Also throw if quotes aren't terminated. --- src/nix-build/nix-build.cc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index ae05444b7..e62c4f6b1 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -34,7 +34,7 @@ extern char * * environ __attribute__((weak)); */ static std::vector shellwords(const std::string & s) { - std::regex whitespace("^(\\s+).*"); + std::regex whitespace("^\\s+"); auto begin = s.cbegin(); std::vector res; std::string cur; @@ -51,9 +51,10 @@ static std::vector shellwords(const std::string & s) if (regex_search(it, s.cend(), match, whitespace)) { cur.append(begin, it); res.push_back(cur); - cur.clear(); - it = match[1].second; + it = match[0].second; + if (it == s.cend()) return res; begin = it; + cur.clear(); } } switch (*it) { @@ -80,8 +81,9 @@ static std::vector shellwords(const std::string & s) break; } } + if (st != sBegin) throw Error("unterminated quote in shebang line"); cur.append(begin, it); - if (!cur.empty()) res.push_back(cur); + res.push_back(cur); return res; } @@ -140,7 +142,7 @@ static void main_nix_build(int argc, char * * argv) for (auto line : lines) { line = chomp(line); std::smatch match; - if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell (.*)$"))) + if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell\\s+(.*)$"))) for (const auto & word : shellwords(match[1].str())) args.push_back(word); } From c82066cf73c1b6d8f910ea68767035d0ead7c7d9 Mon Sep 17 00:00:00 2001 From: Kirill Trofimov Date: Mon, 23 Oct 2023 16:56:30 +0300 Subject: [PATCH 456/909] fix: Declare constructor as default --- src/libutil/args.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index a5d7cbe4a..ebd6fb67b 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -57,7 +57,7 @@ protected: std::function)> fun; size_t arity; - Handler() {} + Handler() = default; Handler(std::function)> && fun) : fun(std::move(fun)) From b205da16ef94d02e08fbe751237e333d8f53aca8 Mon Sep 17 00:00:00 2001 From: Kirill Trofimov Date: Mon, 23 Oct 2023 18:06:15 +0300 Subject: [PATCH 457/909] fix: Explicitly pass lambda scope variables. Default capture implicitly also capture *this, which would automatically be used if for example you referenced a method from the enclosing scope. --- src/libutil/args.hh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index ebd6fb67b..021c2a937 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -84,29 +84,29 @@ protected: { } Handler(std::vector * dest) - : fun([=](std::vector ss) { *dest = ss; }) + : fun([dest](std::vector ss) { *dest = ss; }) , arity(ArityAny) { } Handler(std::string * dest) - : fun([=](std::vector ss) { *dest = ss[0]; }) + : fun([dest](std::vector ss) { *dest = ss[0]; }) , arity(1) { } Handler(std::optional * dest) - : fun([=](std::vector ss) { *dest = ss[0]; }) + : fun([dest](std::vector ss) { *dest = ss[0]; }) , arity(1) { } template Handler(T * dest, const T & val) - : fun([=](std::vector ss) { *dest = val; }) + : fun([dest, val](std::vector ss) { *dest = val; }) , arity(0) { } template Handler(I * dest) - : fun([=](std::vector ss) { + : fun([dest](std::vector ss) { *dest = string2IntWithUnitPrefix(ss[0]); }) , arity(1) @@ -114,7 +114,7 @@ protected: template Handler(std::optional * dest) - : fun([=](std::vector ss) { + : fun([dest](std::vector ss) { *dest = string2IntWithUnitPrefix(ss[0]); }) , arity(1) From a31fc5cc8643beae13b8f071fbeac50aac1a94e2 Mon Sep 17 00:00:00 2001 From: Kirill Trofimov Date: Mon, 23 Oct 2023 18:07:17 +0300 Subject: [PATCH 458/909] fix: Use `using` instead of `typedef` for type aliasing. Since C++ 11 we shouldn't use c-style `typedefs`. In addition, `using` can be templated. --- src/libutil/args.hh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 021c2a937..cee672d80 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -130,7 +130,7 @@ protected: * The `AddCompletions` that is passed is an interface to the state * stored as part of the root command */ - typedef void CompleterFun(AddCompletions &, size_t, std::string_view); + using CompleterFun = void(AddCompletions &, size_t, std::string_view); /** * The closure type of the completion callback. @@ -138,7 +138,7 @@ protected: * This is what is actually stored as part of each Flag / Expected * Arg. */ - typedef std::function CompleterClosure; + using CompleterClosure = std::function; /** * Description of flags / options @@ -148,7 +148,7 @@ protected: */ struct Flag { - typedef std::shared_ptr ptr; + using ptr = std::shared_ptr; std::string longName; std::set aliases; @@ -303,7 +303,7 @@ struct Command : virtual public Args */ virtual void run() = 0; - typedef int Category; + using Category = int; static constexpr Category catDefault = 0; @@ -312,7 +312,7 @@ struct Command : virtual public Args virtual Category category() { return catDefault; } }; -typedef std::map()>> Commands; +using Commands = std::map()>>; /** * An argument parser that supports multiple subcommands, From 90e3ed06f84d0f184de9ee60b8219ba40607387e Mon Sep 17 00:00:00 2001 From: Kirill Trofimov Date: Mon, 23 Oct 2023 18:07:57 +0300 Subject: [PATCH 459/909] fix: Use default destructor. --- src/libutil/args.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index cee672d80..ff2bf3cab 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -296,7 +296,7 @@ struct Command : virtual public Args { friend class MultiCommand; - virtual ~Command() { } + virtual ~Command() = default; /** * Entry point to the command From e053eeb272d2a9115afe7f42077002a3de6b8633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Sat, 17 Jun 2023 14:54:34 +0200 Subject: [PATCH 460/909] tests: test nix-shell shebang quoting --- tests/functional/nix-shell.sh | 5 +++++ tests/functional/shell.shebang.nix | 10 ++++++++++ 2 files changed, 15 insertions(+) create mode 100755 tests/functional/shell.shebang.nix diff --git a/tests/functional/nix-shell.sh b/tests/functional/nix-shell.sh index edaa1249b..13403fadb 100644 --- a/tests/functional/nix-shell.sh +++ b/tests/functional/nix-shell.sh @@ -84,6 +84,11 @@ chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby) [ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ] +# Test nix-shell shebang quoting +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.nix > $TEST_ROOT/shell.shebang.nix +chmod a+rx $TEST_ROOT/shell.shebang.nix +$TEST_ROOT/shell.shebang.nix + # Test 'nix develop'. nix develop -f "$shellDotNix" shellDrv -c bash -c '[[ -n $stdenv ]]' diff --git a/tests/functional/shell.shebang.nix b/tests/functional/shell.shebang.nix new file mode 100755 index 000000000..08e43d53c --- /dev/null +++ b/tests/functional/shell.shebang.nix @@ -0,0 +1,10 @@ +#! @ENV_PROG@ nix-shell +#! nix-shell -I nixpkgs=shell.nix --no-substitute +#! nix-shell --argstr s1 'foo "bar" \baz'"'"'qux' --argstr s2 "foo 'bar' \"\baz" --argstr s3 \foo\ bar\'baz --argstr s4 '' +#! nix-shell shell.shebang.nix --command true +{ s1, s2, s3, s4 }: +assert s1 == ''foo "bar" \baz'qux''; +assert s2 == "foo 'bar' \"baz"; +assert s3 == "foo bar'baz"; +assert s4 == ""; +(import {}).runCommand "nix-shell" {} "" From 765436e300b855922d5793092cc2a533c81d521d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 23 Oct 2023 11:15:52 -0400 Subject: [PATCH 461/909] Add `builtins.addDrvOutputDependencies` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit End goal: make `(mkDerivation x).drvPath` behave like a non-DrvDeep context. Problem: users won't be able to recover the DrvDeep behavior when nixpkgs makes this change. Solution: add this primop. The new primop is fairly simple, and is supposed to complement other existing ones (`builtins.storePath`, `builtins.outputOf`) so there are simple ways to construct strings with every type of string context element. (It allows nothing we couldn't already do with `builtins.getContext` and `builtins.appendContext`, which is also true of those other two primops.) This was originally in #8595, but then it was proposed to land some doc changes separately. So now the code changes proper is just moved to this, and the doc will be done in that. Co-authored-by: Robert Hensing Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.nore github.com> Co-authored-by: Valentin Gagarin **Example** + > + > Many operations require a string context to be empty because they are intended only to work with "regular" strings, and also to help users avoid unintentionally loosing track of string context elements. + > `builtins.hasContext` can help create better domain-specific errors in those case. + > + > ```nix + > name: meta: + > + > if builtins.hasContext name + > then throw "package name cannot contain string context" + > else { ${name} = meta; } + > ``` )", .fun = prim_hasContext }); -/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a - builder without causing the derivation to be built (for instance, - in the derivation that builds NARs in nix-push, when doing - source-only deployment). This primop marks the string context so - that builtins.derivation adds the path to drv.inputSrcs rather than - drv.inputDrvs. */ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx pos, Value * * args, Value & v) { NixStringContext context; @@ -66,11 +73,83 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p static RegisterPrimOp primop_unsafeDiscardOutputDependency({ .name = "__unsafeDiscardOutputDependency", - .arity = 1, + .args = {"s"}, + .doc = R"( + Create a copy of the given string where every "derivation deep" string context element is turned into a constant string context element. + + This is the opposite of [`builtins.addDrvOutputDependencies`](#builtins-addDrvOutputDependencies). + + This is unsafe because it allows us to "forget" store objects we would have otherwise refered to with the string context, + whereas Nix normally tracks all dependencies consistently. + Safe operations "grow" but never "shrink" string contexts. + [`builtins.addDrvOutputDependencies`] in contrast is safe because "derivation deep" string context element always refers to the underlying derivation (among many more things). + Replacing a constant string context element with a "derivation deep" element is a safe operation that just enlargens the string context without forgetting anything. + + [`builtins.addDrvOutputDependencies`]: #builtins-addDrvOutputDependencies + )", .fun = prim_unsafeDiscardOutputDependency }); +static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + NixStringContext context; + auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.addDrvOutputDependencies"); + + auto contextSize = context.size(); + if (contextSize != 1) { + throw EvalError({ + .msg = hintfmt("context of string '%s' must have exactly one element, but has %d", *s, contextSize), + .errPos = state.positions[pos] + }); + } + NixStringContext context2 { + (NixStringContextElem { std::visit(overloaded { + [&](const NixStringContextElem::Opaque & c) -> NixStringContextElem::DrvDeep { + if (!c.path.isDerivation()) { + throw EvalError({ + .msg = hintfmt("path '%s' is not a derivation", + state.store->printStorePath(c.path)), + .errPos = state.positions[pos], + }); + } + return NixStringContextElem::DrvDeep { + .drvPath = c.path, + }; + }, + [&](const NixStringContextElem::Built & c) -> NixStringContextElem::DrvDeep { + throw EvalError({ + .msg = hintfmt("`addDrvOutputDependencies` can only act on derivations, not on a derivation output such as '%1%'", c.output), + .errPos = state.positions[pos], + }); + }, + [&](const NixStringContextElem::DrvDeep & c) -> NixStringContextElem::DrvDeep { + /* Reuse original item because we want this to be idempotent. */ + return std::move(c); + }, + }, context.begin()->raw) }), + }; + + v.mkString(*s, context2); +} + +static RegisterPrimOp primop_addDrvOutputDependencies({ + .name = "__addDrvOutputDependencies", + .args = {"s"}, + .doc = R"( + Create a copy of the given string where a single consant string context element is turned into a "derivation deep" string context element. + + The store path that is the constant string context element should point to a valid derivation, and end in `.drv`. + + The original string context element must not be empty or have multiple elements, and it must not have any other type of element other than a constant or derivation deep element. + The latter is supported so this function is idempotent. + + This is the opposite of [`builtins.unsafeDiscardOutputDependency`](#builtins-addDrvOutputDependencies). + )", + .fun = prim_addDrvOutputDependencies +}); + + /* Extract the context of a string as a structured Nix value. The context is represented as an attribute set whose keys are the diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp new file mode 100644 index 000000000..ad91a22aa --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'addDrvOutputDependencies' builtin + + at /pwd/lang/eval-fail-addDrvOutputDependencies-empty-context.nix:1:1: + + 1| builtins.addDrvOutputDependencies "" + | ^ + 2| + + error: context of string '' must have exactly one element, but has 0 diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.nix b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.nix new file mode 100644 index 000000000..dc9ee3ba2 --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.nix @@ -0,0 +1 @@ +builtins.addDrvOutputDependencies "" diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp new file mode 100644 index 000000000..bb389db4e --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp @@ -0,0 +1,11 @@ +error: + … while calling the 'addDrvOutputDependencies' builtin + + at /pwd/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix:18:4: + + 17| + 18| in builtins.addDrvOutputDependencies combo-path + | ^ + 19| + + error: context of string '/nix/store/pg9yqs4yd85yhdm3f4i5dyaqp5jahrsz-fail.drv/nix/store/2dxd5frb715z451vbf7s8birlf3argbk-fail-2.drv' must have exactly one element, but has 2 diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix new file mode 100644 index 000000000..dbde264df --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix @@ -0,0 +1,18 @@ +let + drv0 = derivation { + name = "fail"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + + drv1 = derivation { + name = "fail-2"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + + combo-path = "${drv0.drvPath}${drv1.drvPath}"; + +in builtins.addDrvOutputDependencies combo-path diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp new file mode 100644 index 000000000..070381118 --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp @@ -0,0 +1,11 @@ +error: + … while calling the 'addDrvOutputDependencies' builtin + + at /pwd/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix:9:4: + + 8| + 9| in builtins.addDrvOutputDependencies drv.outPath + | ^ + 10| + + error: `addDrvOutputDependencies` can only act on derivations, not on a derivation output such as 'out' diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix new file mode 100644 index 000000000..e379e1d95 --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix @@ -0,0 +1,9 @@ +let + drv = derivation { + name = "fail"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + +in builtins.addDrvOutputDependencies drv.outPath diff --git a/tests/functional/lang/eval-okay-context-introspection.exp b/tests/functional/lang/eval-okay-context-introspection.exp index 03b400cc8..a136b0035 100644 --- a/tests/functional/lang/eval-okay-context-introspection.exp +++ b/tests/functional/lang/eval-okay-context-introspection.exp @@ -1 +1 @@ -[ true true true true true true ] +[ true true true true true true true true true true true true true ] diff --git a/tests/functional/lang/eval-okay-context-introspection.nix b/tests/functional/lang/eval-okay-context-introspection.nix index 50a78d946..8886cf32e 100644 --- a/tests/functional/lang/eval-okay-context-introspection.nix +++ b/tests/functional/lang/eval-okay-context-introspection.nix @@ -31,11 +31,29 @@ let (builtins.unsafeDiscardStringContext str) (builtins.getContext str); + # Only holds true if string context contains both a `DrvDeep` and + # `Opaque` element. + almostEtaRule = str: + str == builtins.addDrvOutputDependencies + (builtins.unsafeDiscardOutputDependency str); + + addDrvOutputDependencies_idempotent = str: + builtins.addDrvOutputDependencies str == + builtins.addDrvOutputDependencies (builtins.addDrvOutputDependencies str); + + rules = str: [ + (etaRule str) + (almostEtaRule str) + (addDrvOutputDependencies_idempotent str) + ]; + in [ (legit-context == desired-context) (reconstructed-path == combo-path) (etaRule "foo") - (etaRule drv.drvPath) (etaRule drv.foo.outPath) - (etaRule (builtins.unsafeDiscardOutputDependency drv.drvPath)) +] ++ builtins.concatMap rules [ + drv.drvPath + (builtins.addDrvOutputDependencies drv.drvPath) + (builtins.unsafeDiscardOutputDependency drv.drvPath) ] From c9528d20810c577fd394a8668bec645799d5eded Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=A2=D1=80=D0=BE?= =?UTF-8?q?=D1=84=D0=B8=D0=BC=D0=BE=D0=B2?= <62882157+trofkm@users.noreply.github.com> Date: Mon, 23 Oct 2023 20:20:23 +0300 Subject: [PATCH 462/909] fix: Remove extra to from README.md (#9213) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 623a9722c..e1cace3b4 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Full reference documentation can be found in the [Nix manual](https://nixos.org/ ## Building And Developing See our [Hacking guide](https://nixos.org/manual/nix/unstable/contributing/hacking.html) in our manual for instruction on how to -to set up a development environment and build Nix from source. + set up a development environment and build Nix from source. ## Contributing From cd680bd53d90971211fa8926940bfe6d987ca6cf Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 23 Oct 2023 19:22:33 +0200 Subject: [PATCH 463/909] Merge how-to section on S3 buckets into S3 store docs (#7972) Rather than having a misc tutorial page in the grab-bag "package management" section, this information should just be part of the S3 store docs. --------- Co-authored-by: John Ericson --- doc/manual/src/SUMMARY.md.in | 1 - .../src/advanced-topics/post-build-hook.md | 5 +- .../src/package-management/s3-substituter.md | 115 ------------------ src/libstore/s3-binary-cache-store.md | 100 ++++++++++++++- 4 files changed, 100 insertions(+), 121 deletions(-) delete mode 100644 doc/manual/src/package-management/s3-substituter.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 60ebeb138..f3233f8c9 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -24,7 +24,6 @@ - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) - [Copying Closures via SSH](package-management/copy-closure.md) - [Serving a Nix store via SSH](package-management/ssh-substituter.md) - - [Serving a Nix store via S3](package-management/s3-substituter.md) - [Nix Language](language/index.md) - [Data Types](language/values.md) - [Language Constructs](language/constructs.md) diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md index e4475bd9b..3c1cc9b36 100644 --- a/doc/manual/src/advanced-topics/post-build-hook.md +++ b/doc/manual/src/advanced-topics/post-build-hook.md @@ -17,9 +17,8 @@ the build loop. # Prerequisites -This tutorial assumes you have [configured an S3-compatible binary -cache](../package-management/s3-substituter.md), and that the `root` -user's default AWS profile can upload to the bucket. +This tutorial assumes you have configured an [S3-compatible binary cache](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store) as a [substituter](../command-ref/conf-file.md#conf-substituters), +and that the `root` user's default AWS profile can upload to the bucket. # Set up a Signing Key diff --git a/doc/manual/src/package-management/s3-substituter.md b/doc/manual/src/package-management/s3-substituter.md deleted file mode 100644 index d8a1d9105..000000000 --- a/doc/manual/src/package-management/s3-substituter.md +++ /dev/null @@ -1,115 +0,0 @@ -# Serving a Nix store via S3 - -Nix has [built-in support](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store) -for storing and fetching store paths from -Amazon S3 and S3-compatible services. This uses the same *binary* -cache mechanism that Nix usually uses to fetch prebuilt binaries from -[cache.nixos.org](https://cache.nixos.org/). - -In this example we will use the bucket named `example-nix-cache`. - -## Anonymous Reads to your S3-compatible binary cache - -If your binary cache is publicly accessible and does not require -authentication, the simplest and easiest way to use Nix with your S3 -compatible binary cache is to use the HTTP URL for that cache. - -For AWS S3 the binary cache URL for example bucket will be exactly - or -. For S3 compatible binary caches, consult that -cache's documentation. - -Your bucket will need the following bucket policy: - -```json -{ - "Id": "DirectReads", - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AllowDirectReads", - "Action": [ - "s3:GetObject", - "s3:GetBucketLocation" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:s3:::example-nix-cache", - "arn:aws:s3:::example-nix-cache/*" - ], - "Principal": "*" - } - ] -} -``` - -## Authenticated Reads to your S3 binary cache - -For AWS S3 the binary cache URL for example bucket will be exactly -. - -Nix will use the [default credential provider -chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html) -for authenticating requests to Amazon S3. - -Nix supports authenticated reads from Amazon S3 and S3 compatible binary -caches. - -Your bucket will need a bucket policy allowing the desired users to -perform the `s3:GetObject` and `s3:GetBucketLocation` action on all -objects in the bucket. The [anonymous policy given -above](#anonymous-reads-to-your-s3-compatible-binary-cache) can be -updated to have a restricted `Principal` to support this. - -## Authenticated Writes to your S3-compatible binary cache - -Nix support fully supports writing to Amazon S3 and S3 compatible -buckets. The binary cache URL for our example bucket will be -. - -Nix will use the [default credential provider -chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html) -for authenticating requests to Amazon S3. - -Your account will need the following IAM policy to upload to the cache: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "UploadToCache", - "Effect": "Allow", - "Action": [ - "s3:AbortMultipartUpload", - "s3:GetBucketLocation", - "s3:GetObject", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:ListMultipartUploadParts", - "s3:PutObject" - ], - "Resource": [ - "arn:aws:s3:::example-nix-cache", - "arn:aws:s3:::example-nix-cache/*" - ] - } - ] -} -``` - -## Examples - -To upload with a specific credential profile for Amazon S3: - -```console -$ nix copy nixpkgs.hello \ - --to 's3://example-nix-cache?profile=cache-upload®ion=eu-west-2' -``` - -To upload to an S3-compatible binary cache: - -```console -$ nix copy nixpkgs.hello --to \ - 's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com' -``` diff --git a/src/libstore/s3-binary-cache-store.md b/src/libstore/s3-binary-cache-store.md index 70fe0eb09..675470261 100644 --- a/src/libstore/s3-binary-cache-store.md +++ b/src/libstore/s3-binary-cache-store.md @@ -2,7 +2,103 @@ R"( **Store URL format**: `s3://`*bucket-name* -This store allows reading and writing a binary cache stored in an AWS -S3 bucket. +This store allows reading and writing a binary cache stored in an AWS S3 (or S3-compatible service) bucket. +This store shares many idioms with the [HTTP Binary Cache Store](#http-binary-cache-store). + +For AWS S3, the binary cache URL for a bucket named `example-nix-cache` will be exactly . +For S3 compatible binary caches, consult that cache's documentation. + +### Anonymous reads to your S3-compatible binary cache + +> If your binary cache is publicly accessible and does not require authentication, +> it is simplest to use the [HTTP Binary Cache Store] rather than S3 Binary Cache Store with +> instead of . + +Your bucket will need a +[bucket policy](https://docs.aws.amazon.com/AmazonS3/v1/userguide/bucket-policies.html) +like the following to be accessible: + +```json +{ + "Id": "DirectReads", + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowDirectReads", + "Action": [ + "s3:GetObject", + "s3:GetBucketLocation" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::example-nix-cache", + "arn:aws:s3:::example-nix-cache/*" + ], + "Principal": "*" + } + ] +} +``` + +### Authentication + +Nix will use the +[default credential provider chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html) +for authenticating requests to Amazon S3. + +Note that this means Nix will read environment variables and files with different idioms than with Nix's own settings, as implemented by the AWS SDK. +Consult the documentation linked above for further details. + +### Authenticated reads to your S3 binary cache + +Your bucket will need a bucket policy allowing the desired users to perform the `s3:GetObject` and `s3:GetBucketLocation` action on all objects in the bucket. +The [anonymous policy given above](#anonymous-reads-to-your-s3-compatible-binary-cache) can be updated to have a restricted `Principal` to support this. + +### Authenticated writes to your S3-compatible binary cache + +Your account will need an IAM policy to support uploading to the bucket: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "UploadToCache", + "Effect": "Allow", + "Action": [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:ListMultipartUploadParts", + "s3:PutObject" + ], + "Resource": [ + "arn:aws:s3:::example-nix-cache", + "arn:aws:s3:::example-nix-cache/*" + ] + } + ] +} +``` + +### Examples + +With bucket policies and authentication set up as described above, uploading works via [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md) (experimental). + +- To upload with a specific credential profile for Amazon S3: + + ```console + $ nix copy nixpkgs.hello \ + --to 's3://example-nix-cache?profile=cache-upload®ion=eu-west-2' + ``` + +- To upload to an S3-compatible binary cache: + + ```console + $ nix copy nixpkgs.hello --to \ + 's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com' + ``` )" From cde3c63617137a795f9d3e6fa38714d1a2c54bfa Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 23 Oct 2023 19:30:00 +0200 Subject: [PATCH 464/909] system-features: Typo There I was, thinking all of Apple's OSes started with lower case. --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index e90f70f5f..12fb48d93 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -716,7 +716,7 @@ public: - `apple-virt` - Included on darwin if virtualization is available. + Included on Darwin if virtualization is available. - `kvm` From abb1c829c8ced0680ffa7ef9e45c1844fb24bcb5 Mon Sep 17 00:00:00 2001 From: Vignesh Date: Tue, 24 Oct 2023 14:48:00 +0530 Subject: [PATCH 465/909] Release notes updated for #9150 reverted (#9227) --- doc/manual/src/release-notes/rl-2.15.md | 2 +- doc/manual/src/release-notes/rl-2.7.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md index 4faf0b143..133121999 100644 --- a/doc/manual/src/release-notes/rl-2.15.md +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -44,7 +44,7 @@ (The store always had to check whether it trusts the client, but now the client is informed of the store's decision.) This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores. - `nix store info` and `nix doctor` now display this information. + `nix store ping` and `nix doctor` now display this information. * The new command `nix derivation add` allows adding derivations to the store without involving the Nix language. It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store. diff --git a/doc/manual/src/release-notes/rl-2.7.md b/doc/manual/src/release-notes/rl-2.7.md index dd649e166..2f3879422 100644 --- a/doc/manual/src/release-notes/rl-2.7.md +++ b/doc/manual/src/release-notes/rl-2.7.md @@ -24,7 +24,7 @@ [repository](https://github.com/NixOS/bundlers) has various bundlers implemented. -* `nix store info` now reports the version of the remote Nix daemon. +* `nix store ping` now reports the version of the remote Nix daemon. * `nix flake {init,new}` now display information about which files have been created. From f269911641fcc6cef836a07393feae9d067096a8 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Tue, 24 Oct 2023 11:22:02 +0200 Subject: [PATCH 466/909] Document builtins.substring negative length behavior (#9226) --- src/libexpr/primops.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5033d4e2d..704e7007b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3720,10 +3720,11 @@ static RegisterPrimOp primop_substring({ .doc = R"( Return the substring of *s* from character position *start* (zero-based) up to but not including *start + len*. If *start* is - greater than the length of the string, an empty string is returned, - and if *start + len* lies beyond the end of the string, only the - substring up to the end of the string is returned. *start* must be - non-negative. For example, + greater than the length of the string, an empty string is returned. + If *start + len* lies beyond the end of the string or *len* is `-1`, + only the substring up to the end of the string is returned. + *start* must be non-negative. + For example, ```nix builtins.substring 0 3 "nixos" From eaced12c94e13f0fedd8324f4f9bc8684ea351ae Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 24 Oct 2023 19:30:01 +0200 Subject: [PATCH 467/909] Fix signed vs. unsigned comparison warning and improve code --- src/libstore/builtins/buildenv.cc | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 7bba33fb9..c8911d153 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -174,15 +174,19 @@ void builtinBuildenv(const BasicDerivation & drv) /* Convert the stuff we get from the environment back into a * coherent data type. */ Packages pkgs; - auto derivations = tokenizeString(getAttr("derivations")); - while (!derivations.empty()) { - /* !!! We're trusting the caller to structure derivations env var correctly */ - auto active = derivations.front(); derivations.pop_front(); - auto priority = stoi(derivations.front()); derivations.pop_front(); - auto outputs = stoi(derivations.front()); derivations.pop_front(); - for (auto n = 0; n < outputs; n++) { - auto path = derivations.front(); derivations.pop_front(); - pkgs.emplace_back(path, active != "false", priority); + { + auto derivations = tokenizeString(getAttr("derivations")); + + auto itemIt = derivations.begin(); + while (itemIt != derivations.end()) { + /* !!! We're trusting the caller to structure derivations env var correctly */ + const bool active = "false" != *itemIt++; + const int priority = stoi(*itemIt++); + const size_t outputs = stoul(*itemIt++); + + for (size_t n {0}; n < outputs; n++) { + pkgs.emplace_back(std::move(*itemIt++), active, priority); + } } } From b113d925de50c5f40c4fd694167adde3eeb2b060 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 24 Oct 2023 19:30:23 +0200 Subject: [PATCH 468/909] Fix warning --- src/libstore/content-address.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 52c60154c..a5f7cdf81 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -29,12 +29,13 @@ std::string ContentAddressMethod::renderPrefix() const ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) { - ContentAddressMethod method = FileIngestionMethod::Flat; - if (splitPrefix(m, "r:")) - method = FileIngestionMethod::Recursive; - else if (splitPrefix(m, "text:")) - method = TextIngestionMethod {}; - return method; + if (splitPrefix(m, "r:")) { + return FileIngestionMethod::Recursive; + } + else if (splitPrefix(m, "text:")) { + return TextIngestionMethod {}; + } + return FileIngestionMethod::Flat; } std::string ContentAddressMethod::render(HashType ht) const From 7f71fc7540502295202b075f82e89fcf993e7e3a Mon Sep 17 00:00:00 2001 From: Pol Dellaiera Date: Mon, 23 Oct 2023 23:46:06 +0200 Subject: [PATCH 469/909] fix: make sure `tar` reproducibility flags are set --- tests/functional/tarball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index 6e621a28c..e59ee400e 100644 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -18,7 +18,7 @@ test_tarball() { local compressor="$2" tarball=$TEST_ROOT/tarball.tar$ext - (cd $TEST_ROOT && tar cf - tarball) | $compressor > $tarball + (cd $TEST_ROOT && GNUTAR_REPRODUCIBLE= tar --mtime=$tarroot/default.nix --owner=0 --group=0 --numeric-owner --sort=name -c -f - tarball) | $compressor > $tarball nix-env -f file://$tarball -qa --out-path | grepQuiet dependencies From 8d9e0b7aed2e3bd218c5344d1dcf4a8632e0d63a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 25 Oct 2023 04:28:35 +0200 Subject: [PATCH 470/909] document the store concept (#9206) * document the store concept and its purpose reword the glossary to link to more existing information instead of repeating it. move the store documentation to the top of the table of contents, in front of the Nix language. this will provide a natural place to document other aspects of the store as well as the various store types. move the package management section after the Nix language and before Advanced Topics to follow the pattern to layer more complex concepts on top of each other. this structure of the manual will also nudge beginners to learn Nix bottom-up and hopefully make more likely that they understand underlying concepts first before delving into complex use cases that may or may not be easy to implement with what's currently there. [John adds this note] The sort of beginner who likes to dive straight into reference documentation should prefer this approach. Conversely, the sort of beginner who would prefer the opposite top-down approach of trying to solve problems before they understand everything that is going on is better off reading other tutorial/guide material anyways, and will just "random-access" the reference manual as a last resort. For such random-access the order doesn't matter, so this restructure doesn't make them any worse off. Co-authored-by: John Ericson --- doc/manual/src/SUMMARY.md.in | 20 +++++++------ doc/manual/src/architecture/architecture.md | 1 + doc/manual/src/glossary.md | 30 +++++++++---------- .../file-system-object.md | 0 doc/manual/src/store/index.md | 4 +++ 5 files changed, 30 insertions(+), 25 deletions(-) rename doc/manual/src/{architecture => store}/file-system-object.md (100%) create mode 100644 doc/manual/src/store/index.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index f3233f8c9..2fe77d2c6 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -16,14 +16,8 @@ - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) - [Uninstalling Nix](installation/uninstall.md) -- [Package Management](package-management/package-management.md) - - [Profiles](package-management/profiles.md) - - [Garbage Collection](package-management/garbage-collection.md) - - [Garbage Collector Roots](package-management/garbage-collector-roots.md) - - [Sharing Packages Between Machines](package-management/sharing-packages.md) - - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) - - [Copying Closures via SSH](package-management/copy-closure.md) - - [Serving a Nix store via SSH](package-management/ssh-substituter.md) +- [Nix Store](store/index.md) + - [File System Object](store/file-system-object.md) - [Nix Language](language/index.md) - [Data Types](language/values.md) - [Language Constructs](language/constructs.md) @@ -35,7 +29,16 @@ - [Import From Derivation](language/import-from-derivation.md) - [Built-in Constants](language/builtin-constants.md) - [Built-in Functions](language/builtins.md) +- [Package Management](package-management/package-management.md) + - [Profiles](package-management/profiles.md) + - [Garbage Collection](package-management/garbage-collection.md) + - [Garbage Collector Roots](package-management/garbage-collector-roots.md) - [Advanced Topics](advanced-topics/advanced-topics.md) + - [Sharing Packages Between Machines](package-management/sharing-packages.md) + - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) + - [Copying Closures via SSH](package-management/copy-closure.md) + - [Serving a Nix store via SSH](package-management/ssh-substituter.md) + - [Serving a Nix store via S3](package-management/s3-substituter.md) - [Remote Builds](advanced-topics/distributed-builds.md) - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md) - [Verifying Build Reproducibility](advanced-topics/diff-hook.md) @@ -97,7 +100,6 @@ - [Channels](command-ref/files/channels.md) - [Default Nix expression](command-ref/files/default-nix-expression.md) - [Architecture and Design](architecture/architecture.md) - - [File System Object](architecture/file-system-object.md) - [Protocols](protocols/protocols.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 9e969972e..6e832e1f9 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -59,6 +59,7 @@ The [Nix language](../language/index.md) evaluator transforms Nix expressions in The command line interface and Nix expressions are what users deal with most. > **Note** +> > The Nix language itself does not have a notion of *packages* or *configurations*. > As far as we are concerned here, the inputs and results of a build plan are just data. diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index d49d5e52e..ad3cc147b 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -58,22 +58,16 @@ - [store]{#gloss-store} - The location in the file system where store objects live. Typically - `/nix/store`. + A collection of store objects, with operations to manipulate that collection. + See [Nix Store] for details. - From the perspective of the location where Nix is - invoked, the Nix store can be referred to - as a "_local_" or a "_remote_" one: + There are many types of stores. + See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) for a complete list. - + A [local store]{#gloss-local-store} exists on the filesystem of - the machine where Nix is invoked. You can use other - local stores by passing the `--store` flag to the - `nix` command. Local stores can be used for building derivations. - - + A *remote store* exists anywhere other than the - local filesystem. One example is the `/nix/store` - directory on another machine, accessed via `ssh` or - served by the `nix-serve` Perl script. + From the perspective of the location where Nix is invoked, the Nix store can be referred to _local_ or _remote_. + Only a [local store]{#gloss-local-store} exposes a location in the file system of the machine where Nix is invoked that allows access to store objects, typically `/nix/store`. + Local stores can be used for building [derivations](#derivation). + See [Local Store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) for details. [store]: #gloss-store [local store]: #gloss-local-store @@ -103,15 +97,19 @@ The Nix data model for representing simplified file system data. - See [File System Object](@docroot@/architecture/file-system-object.md) for details. + See [File System Object](@docroot@/store/file-system-object.md) for details. [file system object]: #gloss-file-system-object - [store object]{#gloss-store-object} - A store object consists of a [file system object], [reference]s to other store objects, and other metadata. + Part of the contents of a [store]. + + A store object consists of a [file system object], [references][reference] to other store objects, and other metadata. It can be referred to by a [store path]. + See [Store Object](@docroot@/store/index.md#store-object) for details. + [store object]: #gloss-store-object - [IFD]{#gloss-ifd} diff --git a/doc/manual/src/architecture/file-system-object.md b/doc/manual/src/store/file-system-object.md similarity index 100% rename from doc/manual/src/architecture/file-system-object.md rename to doc/manual/src/store/file-system-object.md diff --git a/doc/manual/src/store/index.md b/doc/manual/src/store/index.md new file mode 100644 index 000000000..316e04179 --- /dev/null +++ b/doc/manual/src/store/index.md @@ -0,0 +1,4 @@ +# Nix Store + +The *Nix store* is an abstraction used by Nix to store immutable filesystem artifacts (such as software packages) that can have dependencies (*references*) between them. +There are multiple implementations of the Nix store, such as the actual filesystem (`/nix/store`) and binary caches. From 1d28d613b1d0447b60de2d2044aeac3e1d543aa6 Mon Sep 17 00:00:00 2001 From: Bouke van der Bijl Date: Wed, 25 Oct 2023 11:39:18 +0200 Subject: [PATCH 471/909] config: add included files into parsedContents before applying Fixes #8719 --- src/libutil/config.cc | 16 ++++++++++++---- tests/functional/init.sh | 3 ++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8e06273ee..17380b6d8 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -88,10 +88,9 @@ void Config::getSettings(std::map & res, bool overridd res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } -void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { - unsigned int pos = 0; - std::vector> parsedContents; +static void applyConfigInner(const std::string & contents, const std::string & path, std::vector> & parsedContents) { + unsigned int pos = 0; while (pos < contents.size()) { std::string line; @@ -123,7 +122,10 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); auto p = absPath(tokens[1], dirOf(path)); if (pathExists(p)) { - applyConfigFile(p); + try { + std::string includedContents = readFile(path); + applyConfigInner(includedContents, p, parsedContents); + } catch (SysError &) { } } else if (!ignoreMissing) { throw Error("file '%1%' included from '%2%' not found", p, path); } @@ -143,6 +145,12 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string concatStringsSep(" ", Strings(i, tokens.end())), }); }; +} + +void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { + std::vector> parsedContents; + + applyConfigInner(contents, path, parsedContents); // First apply experimental-feature related settings for (auto & [name, value] : parsedContents) diff --git a/tests/functional/init.sh b/tests/functional/init.sh index c420e8c9f..d697b1a30 100755 --- a/tests/functional/init.sh +++ b/tests/functional/init.sh @@ -20,7 +20,7 @@ cat > "$NIX_CONF_DIR"/nix.conf < "$NIX_CONF_DIR"/nix.conf.extra < Date: Wed, 25 Oct 2023 12:00:56 +0200 Subject: [PATCH 472/909] add notes on comments in code samples --- doc/manual/src/contributing/documentation.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/contributing/documentation.md index f73ab2149..190d367db 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/contributing/documentation.md @@ -73,6 +73,17 @@ It should therefore aim to be correct, consistent, complete, and easy to navigat Non-trivial examples may need additional explanation, especially if they use concepts from outside the given context. +- Always explain code examples in the text. + + Use comments in code samples very sparingly, for instance to highlight a particular aspect. + Readers tend to glance over large amounts of code when scanning for information. + + Especially beginners will likely find reading more complex-looking code strenuous and may therefore avoid it altogether. + + If a code sample appears to require a lot of inline explanation, consider replacing it with a simpler one. + If that's not possible, break the example down into multiple parts, explain them separately, and then show the combined result at the end. + This should be a last resort, as that would amount to writing a [tutorial](https://diataxis.fr/tutorials/) on the given subject. + - Use British English. This is a somewhat arbitrary choice to force consistency, and accounts for the fact that a majority of Nix users and developers are from Europe. From 00c90eae95c5987a8352dd786d9687f3d213f54a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 25 Oct 2023 12:01:17 +0200 Subject: [PATCH 473/909] add note on highlighting examples and syntax definitions --- doc/manual/src/contributing/documentation.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/contributing/documentation.md index f73ab2149..e35a29d93 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/contributing/documentation.md @@ -151,6 +151,24 @@ Please observe these guidelines to ease reviews: > This is a note. ``` + Highlight examples as such: + + ```` + > **Example** + > + > ```console + > $ nix --version + > ``` + ```` + + Highlight syntax definiions as such, using [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) notation: + + ```` + > **Syntax** + > + > *attribute-set* = `{` [ *attribute-name* `=` *expression* `;` ... ] `}` + ```` + ### The `@docroot@` variable `@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own. From 7bc45c6136af4bb922285ee786f58e54753a2b0d Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Wed, 25 Oct 2023 14:49:30 +0200 Subject: [PATCH 474/909] docs: clarify flake types and implied defaults --- src/nix/flake.md | 77 +++++++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index f08648417..d8b5bf435 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -98,7 +98,15 @@ Here are some examples of flake references in their URL-like representation: ## Path-like syntax -Flakes corresponding to a local path can also be referred to by a direct path reference, either `/absolute/path/to/the/flake` or `./relative/path/to/the/flake` (note that the leading `./` is mandatory for relative paths to avoid any ambiguity). +Flakes corresponding to a local path can also be referred to by a direct +path reference, either `/absolute/path/to/the/flake` or`./relative/path/to/the/flake`. +Note that the leading `./` is mandatory for relative paths. If it is +omitted, the path will be interpreted as [URL-like syntax](#url-like-syntax), +which will cause error messages like this: + +```console +error: cannot find flake 'flake:relative/path/to/the/flake' in the flake registries +``` The semantic of such a path is as follows: @@ -153,18 +161,39 @@ can occur in *locked* flake references and are available to Nix code: Currently the `type` attribute can be one of the following: -* `path`: arbitrary local directories, or local Git trees. The - required attribute `path` specifies the path of the flake. The URL - form is +* `indirect`: *The default*. Indirection through the flake registry. + These have the form ``` - [path:](\?(/(/rev)?)? ``` - where *path* is an absolute path. + These perform a lookup of `` in the flake registry. For + example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake + references. The specified `rev` and/or `ref` are merged with the + entry in the registry; see [nix registry](./nix3-registry.md) for + details. - *path* must be a directory in the file system containing a file - named `flake.nix`. + For example, these are valid indirect flake references: + + * `nixpkgs` + * `nixpkgs/nixos-unstable` + * `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293` + * `nixpkgs/nixos-unstable/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293` + * `sub/dir` (if a flake named `sub` is in the registry) + +* `path`: arbitrary local directories. The required attribute `path` + specifies the path of the flake. The URL form is + + ``` + path:(\?)? + ``` + + where *path* is an absolute path to a directory in the file system + containing a file named `flake.nix`. + + If the flake at *path* is not inside a git repository, the `path:` + prefix is implied and can be omitted. *path* generally must be an absolute path. However, on the command line, it can be a relative path (e.g. `.` or `./foo`) which is @@ -173,20 +202,24 @@ Currently the `type` attribute can be one of the following: (e.g. `nixpkgs` is a registry lookup; `./nixpkgs` is a relative path). + For example, these are valid path flake references: + + * `path:/home/user/sub/dir` + * `/home/user/sub/dir` (if `dir/flake.nix` is *not* in a git repository) + * `./sub/dir` (when used on the command line and `dir/flake.nix` is *not* in a git repository) + * `git`: Git repositories. The location of the repository is specified by the attribute `url`. They have the URL form ``` - git(+http|+https|+ssh|+git|+file|):(//)?(\?)? + git(+http|+https|+ssh|+git|+file):(//)?(\?)? ``` - or - - ``` - @: - ``` + If *path* starts with `/` (or `./` when used as an argument on the + command line) and is a local path to a git repository, the leading + `git:` or `+file` prefixes are implied and can be omitted. The `ref` attribute defaults to resolving the `HEAD` reference. @@ -203,6 +236,9 @@ Currently the `type` attribute can be one of the following: For example, the following are valid Git flake references: + * `git:/home/user/sub/dir` + * `/home/user/sub/dir` (if `dir/flake.nix` is in a git repository) + * `./sub/dir` (when used on the command line and `dir/flake.nix` is in a git repository) * `git+https://example.org/my/repo` * `git+https://example.org/my/repo?dir=flake1` * `git+ssh://git@github.com/NixOS/nix?ref=v1.2.3` @@ -324,19 +360,6 @@ Currently the `type` attribute can be one of the following: * `sourcehut:~misterio/nix-colors/182b4b8709b8ffe4e9774a4c5d6877bf6bb9a21c` * `sourcehut:~misterio/nix-colors/21c1a380a6915d890d408e9f22203436a35bb2de?host=hg.sr.ht` -* `indirect`: Indirections through the flake registry. These have the - form - - ``` - [flake:](/(/rev)?)? - ``` - - These perform a lookup of `` in the flake registry. For - example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake - references. The specified `rev` and/or `ref` are merged with the - entry in the registry; see [nix registry](./nix3-registry.md) for - details. - # Flake format As an example, here is a simple `flake.nix` that depends on the From f555c98a343fbed98b0c01b1b19e6796feaea936 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 24 Oct 2023 19:30:37 +0200 Subject: [PATCH 475/909] Improve loop over gid container --- src/libstore/lock.cc | 60 +++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 7202a64b3..165e4969f 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -7,6 +7,31 @@ namespace nix { +#if __linux__ + +static std::vector get_group_list(const char *username, gid_t group_id) +{ + std::vector gids; + gids.resize(32); // Initial guess + + auto getgroupl_failed {[&] { + int ngroups = gids.size(); + int err = getgrouplist(username, group_id, gids.data(), &ngroups); + gids.resize(ngroups); + return err == -1; + }}; + + // The first error means that the vector was not big enough. + // If it happens again, there is some different problem. + if (getgroupl_failed() && getgroupl_failed()) { + throw SysError("failed to get list of supplementary groups for '%s'", username); + } + + return gids; +} +#endif + + struct SimpleUserLock : UserLock { AutoCloseFD fdUserLock; @@ -67,37 +92,14 @@ struct SimpleUserLock : UserLock throw Error("the Nix user should not be a member of '%s'", settings.buildUsersGroup); #if __linux__ - /* Get the list of supplementary groups of this build - user. This is usually either empty or contains a - group such as "kvm". */ - int ngroups = 32; // arbitrary initial guess - std::vector gids; - gids.resize(ngroups); - - int err = getgrouplist( - pw->pw_name, pw->pw_gid, - gids.data(), - &ngroups); - - /* Our initial size of 32 wasn't sufficient, the - correct size has been stored in ngroups, so we try - again. */ - if (err == -1) { - gids.resize(ngroups); - err = getgrouplist( - pw->pw_name, pw->pw_gid, - gids.data(), - &ngroups); - } - - // If it failed once more, then something must be broken. - if (err == -1) - throw Error("failed to get list of supplementary groups for '%s'", pw->pw_name); + /* Get the list of supplementary groups of this user. This is + * usually either empty or contains a group such as "kvm". */ // Finally, trim back the GID list to its real size. - for (auto i = 0; i < ngroups; i++) - if (gids[i] != lock->gid) - lock->supplementaryGIDs.push_back(gids[i]); + for (auto gid : get_group_list(pw->pw_name, pw->pw_gid)) { + if (gid != lock->gid) + lock->supplementaryGIDs.push_back(gid); + } #endif return lock; From 95d657c8b3ae4282e24628ba7426edb90c8f3942 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Oct 2023 18:18:15 +0200 Subject: [PATCH 476/909] Input: Replace markFileChanged() by putFile() Committing a lock file using markFileChanged() required the input to be writable by the caller in the local filesystem (using the path returned by getSourcePath()). putFile() abstracts over this. --- src/libexpr/flake/flake.cc | 67 +++++++++++++++++++----------------- src/libfetchers/cache.hh | 1 + src/libfetchers/fetchers.cc | 17 +++++---- src/libfetchers/fetchers.hh | 21 +++++++---- src/libfetchers/git.cc | 20 +++++++---- src/libfetchers/indirect.cc | 1 + src/libfetchers/mercurial.cc | 21 +++++++---- src/libfetchers/path.cc | 20 +++++++++-- 8 files changed, 109 insertions(+), 59 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index be2cf014c..5c2a1623a 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -623,12 +623,7 @@ LockedFlake lockFlake( debug("new lock file: %s", newLockFile); - auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; auto sourcePath = topRef.input.getSourcePath(); - auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt; - if (lockFlags.outputLockFilePath) { - outputLockFilePath = lockFlags.outputLockFilePath; - } /* Check whether we need to / can write the new lock file. */ if (newLockFile != oldLockFile || lockFlags.outputLockFilePath) { @@ -636,7 +631,7 @@ LockedFlake lockFlake( auto diff = LockFile::diff(oldLockFile, newLockFile); if (lockFlags.writeLockFile) { - if (outputLockFilePath) { + if (sourcePath || lockFlags.outputLockFilePath) { if (auto unlockedInput = newLockFile.isUnlocked()) { if (fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); @@ -644,41 +639,49 @@ LockedFlake lockFlake( if (!lockFlags.updateLockFile) throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); - bool lockFileExists = pathExists(*outputLockFilePath); + auto newLockFileS = fmt("%s\n", newLockFile); - if (lockFileExists) { - auto s = chomp(diff); - if (s.empty()) - warn("updating lock file '%s'", *outputLockFilePath); - else - warn("updating lock file '%s':\n%s", *outputLockFilePath, s); - } else - warn("creating lock file '%s'", *outputLockFilePath); + if (lockFlags.outputLockFilePath) + writeFile(*lockFlags.outputLockFilePath, newLockFileS); + else { + auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; + auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt; - newLockFile.write(*outputLockFilePath); + bool lockFileExists = pathExists(*outputLockFilePath); - std::optional commitMessage = std::nullopt; - if (lockFlags.commitLockFile) { - if (lockFlags.outputLockFilePath) { - throw Error("--commit-lock-file and --output-lock-file are currently incompatible"); - } - std::string cm; + if (lockFileExists) { + auto s = chomp(diff); + if (s.empty()) + warn("updating lock file '%s'", *outputLockFilePath); + else + warn("updating lock file '%s':\n%s", *outputLockFilePath, s); + } else + warn("creating lock file '%s'", *outputLockFilePath); - cm = fetchSettings.commitLockFileSummary.get(); + std::optional commitMessage = std::nullopt; - if (cm == "") { - cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add"); + if (lockFlags.commitLockFile) { + if (lockFlags.outputLockFilePath) { + throw Error("--commit-lock-file and --output-lock-file are currently incompatible"); + } + std::string cm; + + cm = fetchSettings.commitLockFileSummary.get(); + + if (cm == "") { + cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add"); + } + + cm += "\n\nFlake lock file updates:\n\n"; + cm += filterANSIEscapes(diff, true); + commitMessage = cm; } - cm += "\n\nFlake lock file updates:\n\n"; - cm += filterANSIEscapes(diff, true); - commitMessage = cm; + topRef.input.putFile( + CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"), + newLockFileS, commitMessage); } - topRef.input.markChangedFile( - (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", - commitMessage); - /* Rewriting the lockfile changed the top-level repo, so we should re-read it. FIXME: we could also just clear the 'rev' field... */ diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh index ae398d040..af34e66ce 100644 --- a/src/libfetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -2,6 +2,7 @@ ///@file #include "fetchers.hh" +#include "path.hh" namespace nix::fetchers { diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 5688c4dc1..c339c441b 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -196,12 +196,13 @@ std::optional Input::getSourcePath() const return scheme->getSourcePath(*this); } -void Input::markChangedFile( - std::string_view file, +void Input::putFile( + const CanonPath & path, + std::string_view contents, std::optional commitMsg) const { assert(scheme); - return scheme->markChangedFile(*this, file, commitMsg); + return scheme->putFile(*this, path, contents, commitMsg); } std::string Input::getName() const @@ -292,14 +293,18 @@ Input InputScheme::applyOverrides( return input; } -std::optional InputScheme::getSourcePath(const Input & input) +std::optional InputScheme::getSourcePath(const Input & input) const { return {}; } -void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) +void InputScheme::putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const { - assert(false); + throw Error("input '%s' does not support modifying file '%s'", input.to_string(), path); } void InputScheme::clone(const Input & input, const Path & destDir) const diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index ac605ff8e..4212a3e1f 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -3,13 +3,13 @@ #include "types.hh" #include "hash.hh" -#include "path.hh" +#include "canon-path.hh" #include "attrs.hh" #include "url.hh" #include -namespace nix { class Store; } +namespace nix { class Store; class StorePath; } namespace nix::fetchers { @@ -90,8 +90,13 @@ public: std::optional getSourcePath() const; - void markChangedFile( - std::string_view file, + /** + * Write a file to this input, for input types that support + * writing. Optionally commit the change (for e.g. Git inputs). + */ + void putFile( + const CanonPath & path, + std::string_view contents, std::optional commitMsg) const; std::string getName() const; @@ -135,9 +140,13 @@ struct InputScheme virtual void clone(const Input & input, const Path & destDir) const; - virtual std::optional getSourcePath(const Input & input); + virtual std::optional getSourcePath(const Input & input) const; - virtual void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg); + virtual void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const; virtual std::pair fetch(ref store, const Input & input) = 0; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 26b8987d6..4bfd53b0e 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -354,7 +354,7 @@ struct GitInputScheme : InputScheme runProgram("git", true, args, {}, true); } - std::optional getSourcePath(const Input & input) override + std::optional getSourcePath(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); if (url.scheme == "file" && !input.getRef() && !input.getRev()) @@ -362,18 +362,26 @@ struct GitInputScheme : InputScheme return {}; } - void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const override { - auto sourcePath = getSourcePath(input); - assert(sourcePath); + auto root = getSourcePath(input); + if (!root) + throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string()); + + writeFile((CanonPath(*root) + path).abs(), contents); + auto gitDir = ".git"; runProgram("git", true, - { "-C", *sourcePath, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(file) }); + { "-C", *root, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) }); if (commitMsg) runProgram("git", true, - { "-C", *sourcePath, "--git-dir", gitDir, "commit", std::string(file), "-m", *commitMsg }); + { "-C", *root, "--git-dir", gitDir, "commit", std::string(path.rel()), "-m", *commitMsg }); } std::pair getActualUrl(const Input & input) const diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 9a71df3d4..b18411bdc 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -1,5 +1,6 @@ #include "fetchers.hh" #include "url-parts.hh" +#include "path.hh" namespace nix::fetchers { diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index f830a3271..97c48afc9 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -109,7 +109,7 @@ struct MercurialInputScheme : InputScheme return res; } - std::optional getSourcePath(const Input & input) override + std::optional getSourcePath(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); if (url.scheme == "file" && !input.getRef() && !input.getRev()) @@ -117,18 +117,27 @@ struct MercurialInputScheme : InputScheme return {}; } - void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const override { - auto sourcePath = getSourcePath(input); - assert(sourcePath); + auto [isLocal, repoPath] = getActualUrl(input); + if (!isLocal) + throw Error("cannot commit '%s' to Mercurial repository '%s' because it's not a working tree", path, input.to_string()); + + auto absPath = CanonPath(repoPath) + path; + + writeFile(absPath.abs(), contents); // FIXME: shut up if file is already tracked. runHg( - { "add", *sourcePath + "/" + std::string(file) }); + { "add", absPath.abs() }); if (commitMsg) runHg( - { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg }); + { "commit", absPath.abs(), "-m", *commitMsg }); } std::pair getActualUrl(const Input & input) const diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index d829609b5..22be0f1fe 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -66,14 +66,28 @@ struct PathInputScheme : InputScheme }; } - std::optional getSourcePath(const Input & input) override + std::optional getSourcePath(const Input & input) const override { return getStrAttr(input.attrs, "path"); } - void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const override { - // nothing to do + writeFile((CanonPath(getAbsPath(input)) + path).abs(), contents); + } + + CanonPath getAbsPath(const Input & input) const + { + auto path = getStrAttr(input.attrs, "path"); + + if (path[0] == '/') + return CanonPath(path); + + throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); } std::pair fetch(ref store, const Input & _input) override From 15c430f38971c2f852effec22392cbe1da511aec Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Oct 2023 18:44:09 +0200 Subject: [PATCH 477/909] Remove unused LockFile::write() --- src/libexpr/flake/lockfile.cc | 6 ------ src/libexpr/flake/lockfile.hh | 2 -- 2 files changed, 8 deletions(-) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index f3ea9063f..3e99fb2d4 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -214,12 +214,6 @@ std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) return stream; } -void LockFile::write(const Path & path) const -{ - createDirs(dirOf(path)); - writeFile(path, fmt("%s\n", *this)); -} - std::optional LockFile::isUnlocked() const { std::set> nodes; diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index ba4c0c848..5a1493404 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -65,8 +65,6 @@ struct LockFile static LockFile read(const Path & path); - void write(const Path & path) const; - /** * Check whether this lock file has any unlocked inputs. */ From 46028ff76493439921a5a9200d14c015f0a0c025 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 26 Oct 2023 07:05:48 +0200 Subject: [PATCH 478/909] doc: Fix fetchGit default name (#9241) --- src/libexpr/primops/fetchTree.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index a99b0e500..767f559be 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -392,7 +392,7 @@ static RegisterPrimOp primop_fetchGit({ The URL of the repo. - - `name` (default: *basename of the URL*) + - `name` (default: `source`) The name of the directory the repo should be exported to in the store. From b66381e8d8728c040197ec78ed47d5eff88e1d0e Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 24 Oct 2023 20:16:34 +0200 Subject: [PATCH 479/909] Use using instead of typedef --- src/libstore/path-info.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index a82e643ae..580e94189 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -29,7 +29,7 @@ struct SubstitutablePathInfo uint64_t narSize; }; -typedef std::map SubstitutablePathInfos; +using SubstitutablePathInfos = std::map; struct UnkeyedValidPathInfo @@ -136,6 +136,6 @@ struct ValidPathInfo : UnkeyedValidPathInfo { virtual ~ValidPathInfo() { } }; -typedef std::map ValidPathInfos; +using ValidPathInfos = std::map; } From 28c39c370c99148ccc6576d429b41a1f46b08174 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 24 Oct 2023 20:16:46 +0200 Subject: [PATCH 480/909] Provide default value for id to fix warning --- src/libstore/path-info.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 580e94189..c4c4a6366 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -42,7 +42,7 @@ struct UnkeyedValidPathInfo StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown - uint64_t id; // internal use only + uint64_t id = 0; // internal use only /** * Whether the path is ultimately trusted, that is, it's a From e69c764708a78ae2ea38b4c37e5c07119e7097d0 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Thu, 26 Oct 2023 20:53:03 +0100 Subject: [PATCH 481/909] local-derivation-goal.cc: slightly clarify waiting message Before the change builder ID exhaustion printed the following message: [0/1 built] waiting for UID to build '/nix/store/hiy9136x0iyib4ssh3w3r5m8pxjnad50-python3.11-breathe-4.35.0.drv' After the change it should be: [0/1 built] waiting for a free build user ID for '/nix/store/hiy9136x0iyib4ssh3w3r5m8pxjnad50-python3.11-breathe-4.35.0.drv' --- src/libstore/build/local-derivation-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index b5b060d95..738e7051e 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -227,7 +227,7 @@ void LocalDerivationGoal::tryLocalBuild() if (!buildUser) { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); + fmt("waiting for a free build user ID for '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); return; } From a419b6149705bddff60215a0d21ef355857ef2c5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 26 Oct 2023 17:58:45 -0400 Subject: [PATCH 482/909] Turn derivation unit tests into unit characterization tests The brings a number of advantages, including: - Easier to update test data if design changes (and I do think our derivation JSON is not yet complaint with the guidelines). - Easier to reuse test data in other implementations, inching closer to compliance tests for Nix *the concept* rather than any one implementation. --- src/libstore/tests/characterization.hh | 5 + src/libstore/tests/common-protocol.cc | 8 +- src/libstore/tests/derivation.cc | 273 ++++++++---------- src/libstore/tests/protocol.hh | 4 +- .../derivation/bad-old-version-dyn-deps.drv | 1 + .../libstore/derivation/bad-version.drv | 1 + .../libstore/derivation/dynDerivationDeps.drv | 1 + .../derivation/dynDerivationDeps.json | 38 +++ .../derivation/output-caFixedFlat.json | 5 + .../derivation/output-caFixedNAR.json | 5 + .../derivation/output-caFixedText.json | 5 + .../derivation/output-caFloating.json | 3 + .../libstore/derivation/output-deferred.json | 1 + .../libstore/derivation/output-impure.json | 4 + .../derivation/output-inputAddressed.json | 3 + unit-test-data/libstore/derivation/simple.drv | 1 + .../libstore/derivation/simple.json | 25 ++ 17 files changed, 228 insertions(+), 155 deletions(-) create mode 100644 unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv create mode 100644 unit-test-data/libstore/derivation/bad-version.drv create mode 100644 unit-test-data/libstore/derivation/dynDerivationDeps.drv create mode 100644 unit-test-data/libstore/derivation/dynDerivationDeps.json create mode 100644 unit-test-data/libstore/derivation/output-caFixedFlat.json create mode 100644 unit-test-data/libstore/derivation/output-caFixedNAR.json create mode 100644 unit-test-data/libstore/derivation/output-caFixedText.json create mode 100644 unit-test-data/libstore/derivation/output-caFloating.json create mode 100644 unit-test-data/libstore/derivation/output-deferred.json create mode 100644 unit-test-data/libstore/derivation/output-impure.json create mode 100644 unit-test-data/libstore/derivation/output-inputAddressed.json create mode 100644 unit-test-data/libstore/derivation/simple.drv create mode 100644 unit-test-data/libstore/derivation/simple.json diff --git a/src/libstore/tests/characterization.hh b/src/libstore/tests/characterization.hh index 5f366cb42..46bf4b2e5 100644 --- a/src/libstore/tests/characterization.hh +++ b/src/libstore/tests/characterization.hh @@ -20,4 +20,9 @@ static bool testAccept() { return getEnv("_NIX_TEST_ACCEPT") == "1"; } +constexpr std::string_view cannotReadGoldenMaster = + "Cannot read golden master because another test is also updating it"; + +constexpr std::string_view updatingGoldenMaster = + "Updating golden master"; } diff --git a/src/libstore/tests/common-protocol.cc b/src/libstore/tests/common-protocol.cc index 61c2cb70c..b3f4977d2 100644 --- a/src/libstore/tests/common-protocol.cc +++ b/src/libstore/tests/common-protocol.cc @@ -24,14 +24,14 @@ public: { if (testAccept()) { - GTEST_SKIP() << "Cannot read golden master because another test is also updating it"; + GTEST_SKIP() << cannotReadGoldenMaster; } else { - auto expected = readFile(goldenMaster(testStem)); + auto encoded = readFile(goldenMaster(testStem)); T got = ({ - StringSource from { expected }; + StringSource from { encoded }; CommonProto::Serialise::read( *store, CommonProto::ReadConn { .from = from }); @@ -59,7 +59,7 @@ public: { createDirs(dirOf(file)); writeFile(file, to.s); - GTEST_SKIP() << "Updating golden master"; + GTEST_SKIP() << updatingGoldenMaster; } else { diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index c360c9707..ca0cdff71 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -5,9 +5,12 @@ #include "derivations.hh" #include "tests/libstore.hh" +#include "tests/characterization.hh" namespace nix { +using nlohmann::json; + class DerivationTest : public LibStoreTest { public: @@ -16,6 +19,12 @@ public: * to worry about race conditions if the tests run concurrently. */ ExperimentalFeatureSettings mockXpSettings; + + Path unitTestData = getUnitTestData() + "/libstore/derivation"; + + Path goldenMaster(std::string_view testStem) { + return unitTestData + "/" + testStem; + } }; class CaDerivationTest : public DerivationTest @@ -46,7 +55,7 @@ TEST_F(DerivationTest, BadATerm_version) { ASSERT_THROW( parseDerivation( *store, - R"(DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", + readFile(goldenMaster("bad-version.drv")), "whatever", mockXpSettings), FormatError); @@ -56,50 +65,61 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { ASSERT_THROW( parseDerivation( *store, - R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", + readFile(goldenMaster("bad-old-version-dyn-deps.drv")), "dyn-dep-derivation", mockXpSettings), FormatError); } -#define TEST_JSON(FIXTURE, NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ - TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - STR ## _json, \ - (DerivationOutput { VAL }).toJSON( \ - *store, \ - DRV_NAME, \ - OUTPUT_NAME)); \ - } \ - \ - TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - DerivationOutput { VAL }, \ - DerivationOutput::fromJSON( \ - *store, \ - DRV_NAME, \ - OUTPUT_NAME, \ - STR ## _json, \ - mockXpSettings)); \ +#define TEST_JSON(FIXTURE, NAME, VAL, DRV_NAME, OUTPUT_NAME) \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \ + if (testAccept()) \ + { \ + GTEST_SKIP() << cannotReadGoldenMaster; \ + } \ + else \ + { \ + auto encoded = json::parse( \ + readFile(goldenMaster("output-" #NAME ".json"))); \ + DerivationOutput got = DerivationOutput::fromJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME, \ + encoded, \ + mockXpSettings); \ + DerivationOutput expected { VAL }; \ + ASSERT_EQ(got, expected); \ + } \ + } \ + \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ + auto file = goldenMaster("output-" #NAME ".json"); \ + \ + json got = DerivationOutput { VAL }.toJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME); \ + \ + if (testAccept()) \ + { \ + createDirs(dirOf(file)); \ + writeFile(file, got.dump(2) + "\n"); \ + GTEST_SKIP() << updatingGoldenMaster; \ + } \ + else \ + { \ + auto expected = json::parse(readFile(file)); \ + ASSERT_EQ(got, expected); \ + } \ } TEST_JSON(DerivationTest, inputAddressed, - R"({ - "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" - })", (DerivationOutput::InputAddressed { .path = store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"), }), "drv-name", "output-name") TEST_JSON(DerivationTest, caFixedFlat, - R"({ - "hashAlgo": "sha256", - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" - })", (DerivationOutput::CAFixed { .ca = { .method = FileIngestionMethod::Flat, @@ -109,11 +129,6 @@ TEST_JSON(DerivationTest, caFixedFlat, "drv-name", "output-name") TEST_JSON(DerivationTest, caFixedNAR, - R"({ - "hashAlgo": "r:sha256", - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" - })", (DerivationOutput::CAFixed { .ca = { .method = FileIngestionMethod::Recursive, @@ -123,11 +138,6 @@ TEST_JSON(DerivationTest, caFixedNAR, "drv-name", "output-name") TEST_JSON(DynDerivationTest, caFixedText, - R"({ - "hashAlgo": "text:sha256", - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" - })", (DerivationOutput::CAFixed { .ca = { .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), @@ -136,9 +146,6 @@ TEST_JSON(DynDerivationTest, caFixedText, "drv-name", "output-name") TEST_JSON(CaDerivationTest, caFloating, - R"({ - "hashAlgo": "r:sha256" - })", (DerivationOutput::CAFloating { .method = FileIngestionMethod::Recursive, .hashType = htSHA256, @@ -146,15 +153,10 @@ TEST_JSON(CaDerivationTest, caFloating, "drv-name", "output-name") TEST_JSON(DerivationTest, deferred, - R"({ })", DerivationOutput::Deferred { }, "drv-name", "output-name") TEST_JSON(ImpureDerivationTest, impure, - R"({ - "hashAlgo": "r:sha256", - "impure": true - })", (DerivationOutput::Impure { .method = FileIngestionMethod::Recursive, .hashType = htSHA256, @@ -163,43 +165,79 @@ TEST_JSON(ImpureDerivationTest, impure, #undef TEST_JSON -#define TEST_JSON(FIXTURE, NAME, STR, VAL) \ - TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - STR ## _json, \ - (VAL).toJSON(*store)); \ - } \ - \ - TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - (VAL), \ - Derivation::fromJSON( \ - *store, \ - STR ## _json, \ - mockXpSettings)); \ +#define TEST_JSON(FIXTURE, NAME, VAL) \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \ + if (testAccept()) \ + { \ + GTEST_SKIP() << cannotReadGoldenMaster; \ + } \ + else \ + { \ + auto encoded = json::parse( \ + readFile(goldenMaster( #NAME ".json"))); \ + Derivation expected { VAL }; \ + Derivation got = Derivation::fromJSON( \ + *store, \ + encoded, \ + mockXpSettings); \ + ASSERT_EQ(got, expected); \ + } \ + } \ + \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \ + auto file = goldenMaster( #NAME ".json"); \ + \ + json got = Derivation { VAL }.toJSON(*store); \ + \ + if (testAccept()) \ + { \ + createDirs(dirOf(file)); \ + writeFile(file, got.dump(2) + "\n"); \ + GTEST_SKIP() << updatingGoldenMaster; \ + } \ + else \ + { \ + auto expected = json::parse(readFile(file)); \ + ASSERT_EQ(got, expected); \ + } \ } -#define TEST_ATERM(FIXTURE, NAME, STR, VAL, DRV_NAME) \ - TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \ - ASSERT_EQ( \ - STR, \ - (VAL).unparse(*store, false)); \ - } \ - \ - TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \ - auto parsed = parseDerivation( \ - *store, \ - STR, \ - DRV_NAME, \ - mockXpSettings); \ - ASSERT_EQ( \ - (VAL).toJSON(*store), \ - parsed.toJSON(*store)); \ - ASSERT_EQ( \ - (VAL), \ - parsed); \ +#define TEST_ATERM(FIXTURE, NAME, VAL, DRV_NAME) \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \ + if (testAccept()) \ + { \ + GTEST_SKIP() << cannotReadGoldenMaster; \ + } \ + else \ + { \ + auto encoded = readFile(goldenMaster( #NAME ".drv")); \ + Derivation expected { VAL }; \ + auto got = parseDerivation( \ + *store, \ + std::move(encoded), \ + DRV_NAME, \ + mockXpSettings); \ + ASSERT_EQ(got.toJSON(*store), expected.toJSON(*store)) ; \ + ASSERT_EQ(got, expected); \ + } \ + } \ + \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \ + auto file = goldenMaster( #NAME ".drv"); \ + \ + auto got = (VAL).unparse(*store, false); \ + \ + if (testAccept()) \ + { \ + createDirs(dirOf(file)); \ + writeFile(file, got); \ + GTEST_SKIP() << updatingGoldenMaster; \ + } \ + else \ + { \ + auto expected = readFile(file); \ + ASSERT_EQ(got, expected); \ + } \ } Derivation makeSimpleDrv(const Store & store) { @@ -236,36 +274,9 @@ Derivation makeSimpleDrv(const Store & store) { return drv; } -TEST_JSON(DerivationTest, simple, - R"({ - "name": "simple-derivation", - "inputSrcs": [ - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" - ], - "inputDrvs": { - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { - "dynamicOutputs": {}, - "outputs": [ - "cat", - "dog" - ] - } - }, - "system": "wasm-sel4", - "builder": "foo", - "args": [ - "bar", - "baz" - ], - "env": { - "BIG_BAD": "WOLF" - }, - "outputs": {} - })", - makeSimpleDrv(*store)) +TEST_JSON(DerivationTest, simple, makeSimpleDrv(*store)) TEST_ATERM(DerivationTest, simple, - R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", makeSimpleDrv(*store), "simple-derivation") @@ -321,45 +332,9 @@ Derivation makeDynDepDerivation(const Store & store) { return drv; } -TEST_JSON(DynDerivationTest, dynDerivationDeps, - R"({ - "name": "dyn-dep-derivation", - "inputSrcs": [ - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" - ], - "inputDrvs": { - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { - "dynamicOutputs": { - "cat": { - "dynamicOutputs": {}, - "outputs": ["kitten"] - }, - "goose": { - "dynamicOutputs": {}, - "outputs": ["gosling"] - } - }, - "outputs": [ - "cat", - "dog" - ] - } - }, - "system": "wasm-sel4", - "builder": "foo", - "args": [ - "bar", - "baz" - ], - "env": { - "BIG_BAD": "WOLF" - }, - "outputs": {} - })", - makeDynDepDerivation(*store)) +TEST_JSON(DynDerivationTest, dynDerivationDeps, makeDynDepDerivation(*store)) TEST_ATERM(DynDerivationTest, dynDerivationDeps, - R"(DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", makeDynDepDerivation(*store), "dyn-dep-derivation") diff --git a/src/libstore/tests/protocol.hh b/src/libstore/tests/protocol.hh index 496915745..7fdd3e11c 100644 --- a/src/libstore/tests/protocol.hh +++ b/src/libstore/tests/protocol.hh @@ -29,7 +29,7 @@ public: { if (testAccept()) { - GTEST_SKIP() << "Cannot read golden master because another test is also updating it"; + GTEST_SKIP() << cannotReadGoldenMaster; } else { @@ -70,7 +70,7 @@ public: { createDirs(dirOf(file)); writeFile(file, to.s); - GTEST_SKIP() << "Updating golden master"; + GTEST_SKIP() << updatingGoldenMaster; } else { diff --git a/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv b/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv new file mode 100644 index 000000000..3cd1ded02 --- /dev/null +++ b/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv @@ -0,0 +1 @@ +Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/unit-test-data/libstore/derivation/bad-version.drv b/unit-test-data/libstore/derivation/bad-version.drv new file mode 100644 index 000000000..bbf75c114 --- /dev/null +++ b/unit-test-data/libstore/derivation/bad-version.drv @@ -0,0 +1 @@ +DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.drv b/unit-test-data/libstore/derivation/dynDerivationDeps.drv new file mode 100644 index 000000000..cfffe48ec --- /dev/null +++ b/unit-test-data/libstore/derivation/dynDerivationDeps.drv @@ -0,0 +1 @@ +DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.json b/unit-test-data/libstore/derivation/dynDerivationDeps.json new file mode 100644 index 000000000..9dbeb1f15 --- /dev/null +++ b/unit-test-data/libstore/derivation/dynDerivationDeps.json @@ -0,0 +1,38 @@ +{ + "args": [ + "bar", + "baz" + ], + "builder": "foo", + "env": { + "BIG_BAD": "WOLF" + }, + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": { + "cat": { + "dynamicOutputs": {}, + "outputs": [ + "kitten" + ] + }, + "goose": { + "dynamicOutputs": {}, + "outputs": [ + "gosling" + ] + } + }, + "outputs": [ + "cat", + "dog" + ] + } + }, + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "name": "dyn-dep-derivation", + "outputs": {}, + "system": "wasm-sel4" +} diff --git a/unit-test-data/libstore/derivation/output-caFixedFlat.json b/unit-test-data/libstore/derivation/output-caFixedFlat.json new file mode 100644 index 000000000..fe000ea36 --- /dev/null +++ b/unit-test-data/libstore/derivation/output-caFixedFlat.json @@ -0,0 +1,5 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "sha256", + "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" +} diff --git a/unit-test-data/libstore/derivation/output-caFixedNAR.json b/unit-test-data/libstore/derivation/output-caFixedNAR.json new file mode 100644 index 000000000..1afd60223 --- /dev/null +++ b/unit-test-data/libstore/derivation/output-caFixedNAR.json @@ -0,0 +1,5 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "r:sha256", + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" +} diff --git a/unit-test-data/libstore/derivation/output-caFixedText.json b/unit-test-data/libstore/derivation/output-caFixedText.json new file mode 100644 index 000000000..0b2cc8bbc --- /dev/null +++ b/unit-test-data/libstore/derivation/output-caFixedText.json @@ -0,0 +1,5 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "text:sha256", + "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" +} diff --git a/unit-test-data/libstore/derivation/output-caFloating.json b/unit-test-data/libstore/derivation/output-caFloating.json new file mode 100644 index 000000000..9115de851 --- /dev/null +++ b/unit-test-data/libstore/derivation/output-caFloating.json @@ -0,0 +1,3 @@ +{ + "hashAlgo": "r:sha256" +} diff --git a/unit-test-data/libstore/derivation/output-deferred.json b/unit-test-data/libstore/derivation/output-deferred.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/unit-test-data/libstore/derivation/output-deferred.json @@ -0,0 +1 @@ +{} diff --git a/unit-test-data/libstore/derivation/output-impure.json b/unit-test-data/libstore/derivation/output-impure.json new file mode 100644 index 000000000..62b61cdca --- /dev/null +++ b/unit-test-data/libstore/derivation/output-impure.json @@ -0,0 +1,4 @@ +{ + "hashAlgo": "r:sha256", + "impure": true +} diff --git a/unit-test-data/libstore/derivation/output-inputAddressed.json b/unit-test-data/libstore/derivation/output-inputAddressed.json new file mode 100644 index 000000000..86c7f3a05 --- /dev/null +++ b/unit-test-data/libstore/derivation/output-inputAddressed.json @@ -0,0 +1,3 @@ +{ + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" +} diff --git a/unit-test-data/libstore/derivation/simple.drv b/unit-test-data/libstore/derivation/simple.drv new file mode 100644 index 000000000..bda74ad25 --- /dev/null +++ b/unit-test-data/libstore/derivation/simple.drv @@ -0,0 +1 @@ +Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/unit-test-data/libstore/derivation/simple.json b/unit-test-data/libstore/derivation/simple.json new file mode 100644 index 000000000..20d0f8933 --- /dev/null +++ b/unit-test-data/libstore/derivation/simple.json @@ -0,0 +1,25 @@ +{ + "args": [ + "bar", + "baz" + ], + "builder": "foo", + "env": { + "BIG_BAD": "WOLF" + }, + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": {}, + "outputs": [ + "cat", + "dog" + ] + } + }, + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "name": "simple-derivation", + "outputs": {}, + "system": "wasm-sel4" +} From 325db01d269ca8580fc05ca4b56f28232266ecb7 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 27 Oct 2023 07:30:16 +0200 Subject: [PATCH 483/909] fix anchor in conf-file I inadvertently switched it to `opt-` when refactoring, but it should have been `conf` to begin with. --- doc/manual/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 8bf16e9dd..db3daf252 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -103,7 +103,7 @@ $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "opt-"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "conf"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ $(d)/nix.json: $(bindir)/nix From 8381eeda6fa858b74bc7b516b9af9eecbbddd594 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Oct 2023 10:14:27 -0400 Subject: [PATCH 484/909] Systematize fetcher input attribute validation We now have `schemeName` and `allowedAttrs` functions for this purpose. We look up the schema with the former; we restrict the set of input attributes with the latter. --- src/libfetchers/fetchers.cc | 66 ++++++++++++++++++++++++++---------- src/libfetchers/fetchers.hh | 20 ++++++++++- src/libfetchers/git.cc | 30 ++++++++++++---- src/libfetchers/github.cc | 35 +++++++++++-------- src/libfetchers/indirect.cc | 23 +++++++++---- src/libfetchers/mercurial.cc | 23 +++++++++---- src/libfetchers/path.cc | 35 +++++++++++-------- src/libfetchers/tarball.cc | 34 +++++++++++-------- 8 files changed, 184 insertions(+), 82 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 5688c4dc1..7a5c97399 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -5,12 +5,18 @@ namespace nix::fetchers { -std::unique_ptr>> inputSchemes = nullptr; +using InputSchemeMap = std::map>; + +std::unique_ptr inputSchemes = nullptr; void registerInputScheme(std::shared_ptr && inputScheme) { - if (!inputSchemes) inputSchemes = std::make_unique>>(); - inputSchemes->push_back(std::move(inputScheme)); + if (!inputSchemes) + inputSchemes = std::make_unique(); + auto schemeName = inputScheme->schemeName(); + if (inputSchemes->count(schemeName) > 0) + throw Error("Input scheme with name %s already registered", schemeName); + inputSchemes->insert_or_assign(schemeName, std::move(inputScheme)); } Input Input::fromURL(const std::string & url, bool requireTree) @@ -33,7 +39,7 @@ static void fixupInput(Input & input) Input Input::fromURL(const ParsedURL & url, bool requireTree) { - for (auto & inputScheme : *inputSchemes) { + for (auto & [_, inputScheme] : *inputSchemes) { auto res = inputScheme->inputFromURL(url, requireTree); if (res) { experimentalFeatureSettings.require(inputScheme->experimentalFeature()); @@ -48,20 +54,44 @@ Input Input::fromURL(const ParsedURL & url, bool requireTree) Input Input::fromAttrs(Attrs && attrs) { - for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromAttrs(attrs); - if (res) { - experimentalFeatureSettings.require(inputScheme->experimentalFeature()); - res->scheme = inputScheme; - fixupInput(*res); - return std::move(*res); - } - } + auto schemeName = ({ + auto schemeNameOpt = maybeGetStrAttr(attrs, "type"); + if (!schemeNameOpt) + throw Error("'type' attribute to specify input scheme is required but not provided"); + *std::move(schemeNameOpt); + }); - Input input; - input.attrs = attrs; - fixupInput(input); - return input; + auto raw = [&]() { + // Return an input without a scheme; most operations will fail, + // but not all of them. Doing this is to support those other + // operations which are supposed to be robust on + // unknown/uninterpretable inputs. + Input input; + input.attrs = attrs; + fixupInput(input); + return input; + }; + + std::shared_ptr inputScheme = ({ + auto i = inputSchemes->find(schemeName); + i == inputSchemes->end() ? nullptr : i->second; + }); + + if (!inputScheme) return raw(); + + experimentalFeatureSettings.require(inputScheme->experimentalFeature()); + + auto allowedAttrs = inputScheme->allowedAttrs(); + + for (auto & [name, _] : attrs) + if (name != "type" && allowedAttrs.count(name) == 0) + throw Error("input attribute '%s' not supported by scheme '%s'", name, schemeName); + + auto res = inputScheme->inputFromAttrs(attrs); + if (!res) return raw(); + res->scheme = inputScheme; + fixupInput(*res); + return std::move(*res); } ParsedURL Input::toURL() const @@ -307,7 +337,7 @@ void InputScheme::clone(const Input & input, const Path & destDir) const throw Error("do not know how to clone input '%s'", input.to_string()); } -std::optional InputScheme::experimentalFeature() +std::optional InputScheme::experimentalFeature() const { return {}; } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index ac605ff8e..b35d87eeb 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -126,6 +126,24 @@ struct InputScheme virtual std::optional inputFromAttrs(const Attrs & attrs) const = 0; + /** + * What is the name of the scheme? + * + * The `type` attribute is used to select which input scheme is + * used, and then the other fields are forwarded to that input + * scheme. + */ + virtual std::string_view schemeName() const = 0; + + /** + * Allowed attributes in an attribute set that is converted to an + * input. + * + * `type` is not included from this set, because the `type` field is + parsed first to choose which scheme; `type` is always required. + */ + virtual StringSet allowedAttrs() const = 0; + virtual ParsedURL toURL(const Input & input) const; virtual Input applyOverrides( @@ -144,7 +162,7 @@ struct InputScheme /** * Is this `InputScheme` part of an experimental feature? */ - virtual std::optional experimentalFeature(); + virtual std::optional experimentalFeature() const; virtual bool isDirect(const Input & input) const { return true; } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 26b8987d6..bf25434c8 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -285,14 +285,32 @@ struct GitInputScheme : InputScheme return inputFromAttrs(attrs); } + + std::string_view schemeName() const override + { + return "git"; + } + + StringSet allowedAttrs() const override + { + return { + "url", + "ref", + "rev", + "shallow", + "submodules", + "lastModified", + "revCount", + "narHash", + "allRefs", + "name", + "dirtyRev", + "dirtyShortRev", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override { - if (maybeGetStrAttr(attrs, "type") != "git") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev") - throw Error("unsupported Git input attribute '%s'", name); - maybeGetBoolAttr(attrs, "shallow"); maybeGetBoolAttr(attrs, "submodules"); maybeGetBoolAttr(attrs, "allRefs"); diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 617fc7468..6c9b29721 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -27,13 +27,11 @@ std::regex hostRegex(hostRegexS, std::regex::ECMAScript); struct GitArchiveInputScheme : InputScheme { - virtual std::string type() const = 0; - virtual std::optional> accessHeaderFromToken(const std::string & token) const = 0; std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { - if (url.scheme != type()) return {}; + if (url.scheme != schemeName()) return {}; auto path = tokenizeString>(url.path, "/"); @@ -91,7 +89,7 @@ struct GitArchiveInputScheme : InputScheme throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev()); Input input; - input.attrs.insert_or_assign("type", type()); + input.attrs.insert_or_assign("type", std::string { schemeName() }); input.attrs.insert_or_assign("owner", path[0]); input.attrs.insert_or_assign("repo", path[1]); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); @@ -101,14 +99,21 @@ struct GitArchiveInputScheme : InputScheme return input; } + StringSet allowedAttrs() const override + { + return { + "owner", + "repo", + "ref", + "rev", + "narHash", + "lastModified", + "host", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override { - if (maybeGetStrAttr(attrs, "type") != type()) return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host") - throw Error("unsupported input attribute '%s'", name); - getStrAttr(attrs, "owner"); getStrAttr(attrs, "repo"); @@ -128,7 +133,7 @@ struct GitArchiveInputScheme : InputScheme if (ref) path += "/" + *ref; if (rev) path += "/" + rev->to_string(HashFormat::Base16, false); return ParsedURL { - .scheme = type(), + .scheme = std::string { schemeName() }, .path = path, }; } @@ -220,7 +225,7 @@ struct GitArchiveInputScheme : InputScheme return {result.storePath, input}; } - std::optional experimentalFeature() override + std::optional experimentalFeature() const override { return Xp::Flakes; } @@ -228,7 +233,7 @@ struct GitArchiveInputScheme : InputScheme struct GitHubInputScheme : GitArchiveInputScheme { - std::string type() const override { return "github"; } + std::string_view schemeName() const override { return "github"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -309,7 +314,7 @@ struct GitHubInputScheme : GitArchiveInputScheme struct GitLabInputScheme : GitArchiveInputScheme { - std::string type() const override { return "gitlab"; } + std::string_view schemeName() const override { return "gitlab"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -377,7 +382,7 @@ struct GitLabInputScheme : GitArchiveInputScheme struct SourceHutInputScheme : GitArchiveInputScheme { - std::string type() const override { return "sourcehut"; } + std::string_view schemeName() const override { return "sourcehut"; } std::optional> accessHeaderFromToken(const std::string & token) const override { diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 9a71df3d4..06f7f908d 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -49,14 +49,23 @@ struct IndirectInputScheme : InputScheme return input; } + std::string_view schemeName() const override + { + return "indirect"; + } + + StringSet allowedAttrs() const override + { + return { + "id", + "ref", + "rev", + "narHash", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override { - if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash") - throw Error("unsupported indirect input attribute '%s'", name); - auto id = getStrAttr(attrs, "id"); if (!std::regex_match(id, flakeRegex)) throw BadURL("'%s' is not a valid flake ID", id); @@ -92,7 +101,7 @@ struct IndirectInputScheme : InputScheme throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); } - std::optional experimentalFeature() override + std::optional experimentalFeature() const override { return Xp::Flakes; } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index f830a3271..99002a94f 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -69,14 +69,25 @@ struct MercurialInputScheme : InputScheme return inputFromAttrs(attrs); } + std::string_view schemeName() const override + { + return "hg"; + } + + StringSet allowedAttrs() const override + { + return { + "url", + "ref", + "rev", + "revCount", + "narHash", + "name", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override { - if (maybeGetStrAttr(attrs, "type") != "hg") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name") - throw Error("unsupported Mercurial input attribute '%s'", name); - parseURL(getStrAttr(attrs, "url")); if (auto ref = maybeGetStrAttr(attrs, "ref")) { diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index d829609b5..699efbc3b 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -32,23 +32,30 @@ struct PathInputScheme : InputScheme return input; } + std::string_view schemeName() const override + { + return "path"; + } + + StringSet allowedAttrs() const override + { + return { + "path", + /* Allow the user to pass in "fake" tree info + attributes. This is useful for making a pinned tree work + the same as the repository from which is exported (e.g. + path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). + */ + "rev", + "revCount", + "lastModified", + "narHash", + }; + } std::optional inputFromAttrs(const Attrs & attrs) const override { - if (maybeGetStrAttr(attrs, "type") != "path") return {}; - getStrAttr(attrs, "path"); - for (auto & [name, value] : attrs) - /* Allow the user to pass in "fake" tree info - attributes. This is useful for making a pinned tree - work the same as the repository from which is exported - (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ - if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path") - // checked in Input::fromAttrs - ; - else - throw Error("unsupported path input attribute '%s'", name); - Input input; input.attrs = attrs; return input; @@ -121,7 +128,7 @@ struct PathInputScheme : InputScheme return {std::move(*storePath), input}; } - std::optional experimentalFeature() override + std::optional experimentalFeature() const override { return Xp::Flakes; } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index e1ea9b58b..0062878a9 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -184,7 +184,6 @@ DownloadTarballResult downloadTarball( // An input scheme corresponding to a curl-downloadable resource. struct CurlInputScheme : InputScheme { - virtual const std::string inputType() const = 0; const std::set transportUrlSchemes = {"file", "http", "https"}; const bool hasTarballExtension(std::string_view path) const @@ -222,22 +221,27 @@ struct CurlInputScheme : InputScheme url.query.erase("rev"); url.query.erase("revCount"); - input.attrs.insert_or_assign("type", inputType()); + input.attrs.insert_or_assign("type", std::string { schemeName() }); input.attrs.insert_or_assign("url", url.to_string()); return input; } + StringSet allowedAttrs() const override + { + return { + "type", + "url", + "narHash", + "name", + "unpack", + "rev", + "revCount", + "lastModified", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override { - auto type = maybeGetStrAttr(attrs, "type"); - if (type != inputType()) return {}; - - // FIXME: some of these only apply to TarballInputScheme. - std::set allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount", "lastModified"}; - for (auto & [name, value] : attrs) - if (!allowedNames.count(name)) - throw Error("unsupported %s input attribute '%s'", *type, name); - Input input; input.attrs = attrs; @@ -258,14 +262,14 @@ struct CurlInputScheme : InputScheme struct FileInputScheme : CurlInputScheme { - const std::string inputType() const override { return "file"; } + std::string_view schemeName() const override { return "file"; } bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application - ? parsedUrlScheme.application.value() == inputType() + ? parsedUrlScheme.application.value() == schemeName() : (!requireTree && !hasTarballExtension(url.path))); } @@ -278,7 +282,7 @@ struct FileInputScheme : CurlInputScheme struct TarballInputScheme : CurlInputScheme { - const std::string inputType() const override { return "tarball"; } + std::string_view schemeName() const override { return "tarball"; } bool isValidURL(const ParsedURL & url, bool requireTree) const override { @@ -286,7 +290,7 @@ struct TarballInputScheme : CurlInputScheme return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application - ? parsedUrlScheme.application.value() == inputType() + ? parsedUrlScheme.application.value() == schemeName() : (requireTree || hasTarballExtension(url.path))); } From 077de2968e8cf2d125818999adf8c149baf6384e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Oct 2023 10:30:59 -0400 Subject: [PATCH 485/909] Include fetcher input scheme info in the CLI dump Leverages the previous commit. --- src/libfetchers/fetchers.cc | 13 +++++++++++++ src/libfetchers/fetchers.hh | 3 +++ src/nix/main.cc | 1 + 3 files changed, 17 insertions(+) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 7a5c97399..44b3fa4a5 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -19,6 +19,19 @@ void registerInputScheme(std::shared_ptr && inputScheme) inputSchemes->insert_or_assign(schemeName, std::move(inputScheme)); } +nlohmann::json dumpRegisterInputSchemeInfo() { + using nlohmann::json; + + auto res = json::object(); + + for (auto & [name, scheme] : *inputSchemes) { + auto & r = res[name] = json::object(); + r["allowedAttrs"] = scheme->allowedAttrs(); + } + + return res; +} + Input Input::fromURL(const std::string & url, bool requireTree) { return fromURL(parseURL(url), requireTree); diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index b35d87eeb..3a02967f4 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -8,6 +8,7 @@ #include "url.hh" #include +#include namespace nix { class Store; } @@ -170,4 +171,6 @@ struct InputScheme void registerInputScheme(std::shared_ptr && fetcher); +nlohmann::json dumpRegisterInputSchemeInfo(); + } diff --git a/src/nix/main.cc b/src/nix/main.cc index ffba10099..d20bc1f8a 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -188,6 +188,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs j["experimentalFeature"] = storeConfig->experimentalFeature(); } res["stores"] = std::move(stores); + res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo(); return res.dump(); } From 05316d401fa509557c71140e17bb19814412fcb8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Oct 2023 17:03:06 +0100 Subject: [PATCH 486/909] Cleanup --- src/libexpr/flake/flake.cc | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 5c2a1623a..45c9ec8f3 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -641,29 +641,28 @@ LockedFlake lockFlake( auto newLockFileS = fmt("%s\n", newLockFile); - if (lockFlags.outputLockFilePath) + if (lockFlags.outputLockFilePath) { + if (lockFlags.commitLockFile) + throw Error("'--commit-lock-file' and '--output-lock-file' are incompatible"); writeFile(*lockFlags.outputLockFilePath, newLockFileS); - else { + } else { auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; - auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt; + auto outputLockFilePath = *sourcePath + "/" + relPath; - bool lockFileExists = pathExists(*outputLockFilePath); + bool lockFileExists = pathExists(outputLockFilePath); if (lockFileExists) { auto s = chomp(diff); if (s.empty()) - warn("updating lock file '%s'", *outputLockFilePath); + warn("updating lock file '%s'", outputLockFilePath); else - warn("updating lock file '%s':\n%s", *outputLockFilePath, s); + warn("updating lock file '%s':\n%s", outputLockFilePath, s); } else - warn("creating lock file '%s'", *outputLockFilePath); + warn("creating lock file '%s'", outputLockFilePath); std::optional commitMessage = std::nullopt; if (lockFlags.commitLockFile) { - if (lockFlags.outputLockFilePath) { - throw Error("--commit-lock-file and --output-lock-file are currently incompatible"); - } std::string cm; cm = fetchSettings.commitLockFileSummary.get(); From 95f3f9eac978466c812814c06716f26e9f668e54 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 22:21:34 +0000 Subject: [PATCH 487/909] build(deps): bump zeebe-io/backport-action from 1.4.0 to 2.0.0 Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 1.4.0 to 2.0.0. - [Release notes](https://github.com/zeebe-io/backport-action/releases) - [Commits](https://github.com/zeebe-io/backport-action/compare/v1.4.0...v2.0.0) --- updated-dependencies: - dependency-name: zeebe-io/backport-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 12c60c649..312c211dd 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v1.4.0 + uses: zeebe-io/backport-action@v2.0.0 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} From e1b8442fa1dbd2e69598dbeb701da4df8e6d2c38 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 24 Oct 2023 08:20:31 +0200 Subject: [PATCH 488/909] Fetcher cache: Add support for caching facts not related to store paths --- src/libfetchers/cache.cc | 57 ++++++++++++++++++++++++++++++++++++++++ src/libfetchers/cache.hh | 38 +++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 0c8ecac9d..8a3e462d3 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -19,6 +19,9 @@ create table if not exists Cache ( ); )sql"; +// FIXME: we should periodically purge/nuke this cache to prevent it +// from growing too big. + struct CacheImpl : Cache { struct State @@ -47,6 +50,60 @@ struct CacheImpl : Cache "select info, path, immutable, timestamp from Cache where input = ?"); } + void upsert( + const Attrs & inAttrs, + const Attrs & infoAttrs) override + { + _state.lock()->add.use() + (attrsToJSON(inAttrs).dump()) + (attrsToJSON(infoAttrs).dump()) + ("") // no path + (false) + (time(0)).exec(); + } + + std::optional lookup(const Attrs & inAttrs) override + { + if (auto res = lookupExpired(inAttrs)) + return std::move(res->infoAttrs); + return {}; + } + + std::optional lookupWithTTL(const Attrs & inAttrs) override + { + if (auto res = lookupExpired(inAttrs)) { + if (!res->expired) + return std::move(res->infoAttrs); + debug("ignoring expired cache entry '%s'", + attrsToJSON(inAttrs).dump()); + } + return {}; + } + + std::optional lookupExpired(const Attrs & inAttrs) override + { + auto state(_state.lock()); + + auto inAttrsJSON = attrsToJSON(inAttrs).dump(); + + auto stmt(state->lookup.use()(inAttrsJSON)); + if (!stmt.next()) { + debug("did not find cache entry for '%s'", inAttrsJSON); + return {}; + } + + auto infoJSON = stmt.getStr(0); + auto locked = stmt.getInt(2) != 0; + auto timestamp = stmt.getInt(3); + + debug("using cache entry '%s' -> '%s'", inAttrsJSON, infoJSON); + + return Result2 { + .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)), + .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)), + }; + } + void add( ref store, const Attrs & inAttrs, diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh index af34e66ce..b517d496e 100644 --- a/src/libfetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -10,6 +10,44 @@ struct Cache { virtual ~Cache() { } + /* A cache for arbitrary Attrs -> Attrs mappings with a timestamp + for expiration. */ + + /* + * Add a value to the cache. The cache is an arbitrary mapping of + * Attrs to Attrs. + */ + virtual void upsert( + const Attrs & inAttrs, + const Attrs & infoAttrs) = 0; + + /* + * Look up a key with infinite TTL. + */ + virtual std::optional lookup( + const Attrs & inAttrs) = 0; + + /* + * Look up a key. Return nothing if its TTL has exceeded + * `settings.tarballTTL`. + */ + virtual std::optional lookupWithTTL( + const Attrs & inAttrs) = 0; + + struct Result2 + { + bool expired = false; + Attrs infoAttrs; + }; + + /* + * Look up a key. Return a bool denoting whether its TTL has + * exceeded `settings.tarballTTL`. + */ + virtual std::optional lookupExpired( + const Attrs & inAttrs) = 0; + + /* Old cache for things that have a store path. */ virtual void add( ref store, const Attrs & inAttrs, From 1d0e3d84b6ed693c140c3b7fd6a72ef8a8a26ec3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 24 Oct 2023 10:23:46 +0200 Subject: [PATCH 489/909] Provide a InputScheme::fetch() built on top of InputScheme::getAccessor() This is for graceful migration to lazy-trees fetchers (which are all accessor-based). Eventually fetch() will be removed. --- src/libfetchers/fetchers.cc | 13 +++++++++++++ src/libfetchers/fetchers.hh | 6 ++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index c339c441b..3e654dd53 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -1,5 +1,6 @@ #include "fetchers.hh" #include "store-api.hh" +#include "input-accessor.hh" #include @@ -312,6 +313,18 @@ void InputScheme::clone(const Input & input, const Path & destDir) const throw Error("do not know how to clone input '%s'", input.to_string()); } +std::pair InputScheme::fetch(ref store, const Input & input) +{ + auto [accessor, input2] = getAccessor(store, input); + auto storePath = accessor->root().fetchToStore(store, input2.getName()); + return {storePath, input2}; +} + +std::pair, Input> InputScheme::getAccessor(ref store, const Input & input) const +{ + throw UnimplementedError("InputScheme must implement fetch() or getAccessor()"); +} + std::optional InputScheme::experimentalFeature() { return {}; diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 4212a3e1f..7b70ab6e2 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -9,7 +9,7 @@ #include -namespace nix { class Store; class StorePath; } +namespace nix { class Store; class StorePath; struct InputAccessor; } namespace nix::fetchers { @@ -148,7 +148,9 @@ struct InputScheme std::string_view contents, std::optional commitMsg) const; - virtual std::pair fetch(ref store, const Input & input) = 0; + virtual std::pair fetch(ref store, const Input & input); + + virtual std::pair, Input> getAccessor(ref store, const Input & input) const; /** * Is this `InputScheme` part of an experimental feature? From ee36a44bf272c8cca62a2ce96a017a8150c4d35b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Oct 2023 18:55:08 +0200 Subject: [PATCH 490/909] GitInputScheme: Use libgit2 This replaces most calls to the "git" binary with libgit2. --- flake.nix | 1 + src/libfetchers/git-utils.cc | 498 ++++++++++++++ src/libfetchers/git-utils.hh | 56 ++ src/libfetchers/git.cc | 609 ++++++++---------- src/libfetchers/local.mk | 2 +- tests/functional/fetchGit.sh | 7 +- tests/functional/flakes/flake-in-submodule.sh | 5 +- 7 files changed, 839 insertions(+), 339 deletions(-) create mode 100644 src/libfetchers/git-utils.cc create mode 100644 src/libfetchers/git-utils.hh diff --git a/flake.nix b/flake.nix index 398ba10a0..3472bf7a8 100644 --- a/flake.nix +++ b/flake.nix @@ -191,6 +191,7 @@ bzip2 xz brotli editline openssl sqlite libarchive + libgit2 boost lowdown-nix libsodium diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc new file mode 100644 index 000000000..68e39580f --- /dev/null +++ b/src/libfetchers/git-utils.cc @@ -0,0 +1,498 @@ +#include "git-utils.hh" +#include "input-accessor.hh" +#include "cache.hh" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace std { + +template<> struct hash +{ + size_t operator()(const git_oid & oid) const + { + return * (size_t *) oid.id; + } +}; + +} + +std::ostream & operator << (std::ostream & str, const git_oid & oid) +{ + str << git_oid_tostr_s(&oid); + return str; +} + +bool operator == (const git_oid & oid1, const git_oid & oid2) +{ + return git_oid_equal(&oid1, &oid2); +} + +namespace nix { + +// Some wrapper types that ensure that the git_*_free functions get called. +template +struct Deleter +{ + template + void operator()(T * p) const { del(p); }; +}; + +typedef std::unique_ptr> Repository; +typedef std::unique_ptr> TreeEntry; +typedef std::unique_ptr> Tree; +typedef std::unique_ptr> TreeBuilder; +typedef std::unique_ptr> Blob; +typedef std::unique_ptr> Object; +typedef std::unique_ptr> Commit; +typedef std::unique_ptr> Reference; +typedef std::unique_ptr> DescribeResult; +typedef std::unique_ptr> StatusList; +typedef std::unique_ptr> Remote; + +// A helper to ensure that we don't leak objects returned by libgit2. +template +struct Setter +{ + T & t; + typename T::pointer p = nullptr; + + Setter(T & t) : t(t) { } + + ~Setter() { if (p) t = T(p); } + + operator typename T::pointer * () { return &p; } +}; + +Hash toHash(const git_oid & oid) +{ + #ifdef GIT_EXPERIMENTAL_SHA256 + assert(oid.type == GIT_OID_SHA1); + #endif + Hash hash(htSHA1); + memcpy(hash.hash, oid.id, hash.hashSize); + return hash; +} + +static void initLibGit2() +{ + if (git_libgit2_init() < 0) + throw Error("initialising libgit2: %s", git_error_last()->message); +} + +git_oid hashToOID(const Hash & hash) +{ + git_oid oid; + if (git_oid_fromstr(&oid, hash.gitRev().c_str())) + throw Error("cannot convert '%s' to a Git OID", hash.gitRev()); + return oid; +} + +Object lookupObject(git_repository * repo, const git_oid & oid) +{ + Object obj; + if (git_object_lookup(Setter(obj), repo, &oid, GIT_OBJECT_ANY)) { + auto err = git_error_last(); + throw Error("getting Git object '%s': %s", oid, err->message); + } + return obj; +} + +template +T peelObject(git_repository * repo, git_object * obj, git_object_t type) +{ + T obj2; + if (git_object_peel((git_object * *) (typename T::pointer *) Setter(obj2), obj, type)) { + auto err = git_error_last(); + throw Error("peeling Git object '%s': %s", git_object_id(obj), err->message); + } + return obj2; +} + +int statusCallbackTrampoline(const char * path, unsigned int statusFlags, void * payload) +{ + return (*((std::function *) payload))(path, statusFlags); +} + +struct GitRepoImpl : GitRepo, std::enable_shared_from_this +{ + CanonPath path; + Repository repo; + + GitRepoImpl(CanonPath _path, bool create, bool bare) + : path(std::move(_path)) + { + initLibGit2(); + + if (pathExists(path.abs())) { + if (git_repository_open(Setter(repo), path.c_str())) + throw Error("opening Git repository '%s': %s", path, git_error_last()->message); + } else { + if (git_repository_init(Setter(repo), path.c_str(), bare)) + throw Error("creating Git repository '%s': %s", path, git_error_last()->message); + } + } + + operator git_repository * () + { + return repo.get(); + } + + uint64_t getRevCount(const Hash & rev) override + { + std::unordered_set done; + std::queue todo; + + todo.push(peelObject(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT)); + + while (auto commit = pop(todo)) { + if (!done.insert(*git_commit_id(commit->get())).second) continue; + + for (size_t n = 0; n < git_commit_parentcount(commit->get()); ++n) { + git_commit * parent; + if (git_commit_parent(&parent, commit->get(), n)) + throw Error("getting parent of Git commit '%s': %s", *git_commit_id(commit->get()), git_error_last()->message); + todo.push(Commit(parent)); + } + } + + return done.size(); + } + + uint64_t getLastModified(const Hash & rev) override + { + auto commit = peelObject(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT); + + return git_commit_time(commit.get()); + } + + bool isShallow() override + { + return git_repository_is_shallow(*this); + } + + Hash resolveRef(std::string ref) override + { + // Handle revisions used as refs. + { + git_oid oid; + if (git_oid_fromstr(&oid, ref.c_str()) == 0) + return toHash(oid); + } + + // Resolve short names like 'master'. + Reference ref2; + if (!git_reference_dwim(Setter(ref2), *this, ref.c_str())) + ref = git_reference_name(ref2.get()); + + // Resolve full references like 'refs/heads/master'. + Reference ref3; + if (git_reference_lookup(Setter(ref3), *this, ref.c_str())) + throw Error("resolving Git reference '%s': %s", ref, git_error_last()->message); + + auto oid = git_reference_target(ref3.get()); + if (!oid) + throw Error("cannot get OID for Git reference '%s'", git_reference_name(ref3.get())); + + return toHash(*oid); + } + + WorkdirInfo getWorkdirInfo() override + { + WorkdirInfo info; + + /* Get the head revision, if any. */ + git_oid headRev; + if (auto err = git_reference_name_to_id(&headRev, *this, "HEAD")) { + if (err != GIT_ENOTFOUND) + throw Error("resolving HEAD: %s", git_error_last()->message); + } else + info.headRev = toHash(headRev); + + /* Get all tracked files and determine whether the working + directory is dirty. */ + std::function statusCallback = [&](const char * path, unsigned int statusFlags) + { + if (!(statusFlags & GIT_STATUS_INDEX_DELETED) && + !(statusFlags & GIT_STATUS_WT_DELETED)) + info.files.insert(CanonPath(path)); + if (statusFlags != GIT_STATUS_CURRENT) + info.isDirty = true; + return 0; + }; + + git_status_options options = GIT_STATUS_OPTIONS_INIT; + options.flags |= GIT_STATUS_OPT_INCLUDE_UNMODIFIED; + options.flags |= GIT_STATUS_OPT_EXCLUDE_SUBMODULES; + if (git_status_foreach_ext(*this, &options, &statusCallbackTrampoline, &statusCallback)) + throw Error("getting working directory status: %s", git_error_last()->message); + + return info; + } + + std::optional getWorkdirRef() override + { + Reference ref; + if (git_reference_lookup(Setter(ref), *this, "HEAD")) + throw Error("looking up HEAD: %s", git_error_last()->message); + + if (auto target = git_reference_symbolic_target(ref.get())) + return target; + + return std::nullopt; + } + + bool hasObject(const Hash & oid_) override + { + auto oid = hashToOID(oid_); + + Object obj; + if (auto errCode = git_object_lookup(Setter(obj), *this, &oid, GIT_OBJECT_ANY)) { + if (errCode == GIT_ENOTFOUND) return false; + auto err = git_error_last(); + throw Error("getting Git object '%s': %s", oid, err->message); + } + + return true; + } + + ref getAccessor(const Hash & rev) override; + + void fetch( + const std::string & url, + const std::string & refspec) override + { + /* FIXME: use libgit2. Unfortunately, it doesn't support + ssh_config at the moment. */ + #if 0 + Remote remote; + + if (git_remote_create_anonymous(Setter(remote), *this, url.c_str())) + throw Error("cannot create Git remote '%s': %s", url, git_error_last()->message); + + char * refspecs[] = {(char *) refspec.c_str()}; + git_strarray refspecs2 { + .strings = refspecs, + .count = 1 + }; + + if (git_remote_fetch(remote.get(), &refspecs2, nullptr, nullptr)) + throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message); + #endif + + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + runProgram("git", true, + { "-C", path.abs(), + "--bare", + "fetch", + "--quiet", + "--force", + "--", + url, + refspec + }, {}, true); + } +}; + +ref GitRepo::openRepo(const CanonPath & path, bool create, bool bare) +{ + return make_ref(path, create, bare); +} + +struct GitInputAccessor : InputAccessor +{ + ref repo; + Tree root; + + GitInputAccessor(ref repo_, const Hash & rev) + : repo(repo_) + , root(peelObject(*repo, lookupObject(*repo, hashToOID(rev)).get(), GIT_OBJECT_TREE)) + { + } + + std::string readBlob(const CanonPath & path, bool symlink) + { + auto blob = getBlob(path, symlink); + + auto data = std::string_view((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get())); + + return std::string(data); + } + + std::string readFile(const CanonPath & path) override + { + return readBlob(path, false); + } + + bool pathExists(const CanonPath & path) override + { + return path.isRoot() ? true : (bool) lookup(path); + } + + Stat lstat(const CanonPath & path) override + { + if (path.isRoot()) + return Stat { .type = tDirectory }; + + auto entry = need(path); + + auto mode = git_tree_entry_filemode(entry); + + if (mode == GIT_FILEMODE_TREE) + return Stat { .type = tDirectory }; + + else if (mode == GIT_FILEMODE_BLOB) + return Stat { .type = tRegular }; + + else if (mode == GIT_FILEMODE_BLOB_EXECUTABLE) + return Stat { .type = tRegular, .isExecutable = true }; + + else if (mode == GIT_FILEMODE_LINK) + return Stat { .type = tSymlink }; + + else if (mode == GIT_FILEMODE_COMMIT) + // Treat submodules as an empty directory. + return Stat { .type = tDirectory }; + + else + throw Error("file '%s' has an unsupported Git file type"); + } + + DirEntries readDirectory(const CanonPath & path) override + { + return std::visit(overloaded { + [&](Tree tree) { + DirEntries res; + + auto count = git_tree_entrycount(tree.get()); + + for (size_t n = 0; n < count; ++n) { + auto entry = git_tree_entry_byindex(tree.get(), n); + // FIXME: add to cache + res.emplace(std::string(git_tree_entry_name(entry)), DirEntry{}); + } + + return res; + }, + [&](Submodule) { + return DirEntries(); + } + }, getTree(path)); + } + + std::string readLink(const CanonPath & path) override + { + return readBlob(path, true); + } + + std::map lookupCache; + + /* Recursively look up 'path' relative to the root. */ + git_tree_entry * lookup(const CanonPath & path) + { + if (path.isRoot()) return nullptr; + + auto i = lookupCache.find(path); + if (i == lookupCache.end()) { + TreeEntry entry; + if (auto err = git_tree_entry_bypath(Setter(entry), root.get(), std::string(path.rel()).c_str())) { + if (err != GIT_ENOTFOUND) + throw Error("looking up '%s': %s", showPath(path), git_error_last()->message); + } + + i = lookupCache.emplace(path, std::move(entry)).first; + } + + return &*i->second; + } + + git_tree_entry * need(const CanonPath & path) + { + auto entry = lookup(path); + if (!entry) + throw Error("'%s' does not exist", showPath(path)); + return entry; + } + + struct Submodule { }; + + std::variant getTree(const CanonPath & path) + { + if (path.isRoot()) { + Tree tree; + if (git_tree_dup(Setter(tree), root.get())) + throw Error("duplicating directory '%s': %s", showPath(path), git_error_last()->message); + return tree; + } + + auto entry = need(path); + + if (git_tree_entry_type(entry) == GIT_OBJECT_COMMIT) + return Submodule(); + + if (git_tree_entry_type(entry) != GIT_OBJECT_TREE) + throw Error("'%s' is not a directory", showPath(path)); + + Tree tree; + if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(tree), *repo, entry)) + throw Error("looking up directory '%s': %s", showPath(path), git_error_last()->message); + + return tree; + } + + Blob getBlob(const CanonPath & path, bool expectSymlink) + { + auto notExpected = [&]() + { + throw Error( + expectSymlink + ? "'%s' is not a symlink" + : "'%s' is not a regular file", + showPath(path)); + }; + + if (path.isRoot()) notExpected(); + + auto entry = need(path); + + if (git_tree_entry_type(entry) != GIT_OBJECT_BLOB) + notExpected(); + + auto mode = git_tree_entry_filemode(entry); + if (expectSymlink) { + if (mode != GIT_FILEMODE_LINK) + notExpected(); + } else { + if (mode != GIT_FILEMODE_BLOB && mode != GIT_FILEMODE_BLOB_EXECUTABLE) + notExpected(); + } + + Blob blob; + if (git_tree_entry_to_object((git_object * *) (git_blob * *) Setter(blob), *repo, entry)) + throw Error("looking up file '%s': %s", showPath(path), git_error_last()->message); + + return blob; + } +}; + +ref GitRepoImpl::getAccessor(const Hash & rev) +{ + return make_ref(ref(shared_from_this()), rev); +} + +} diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh new file mode 100644 index 000000000..dd2c06672 --- /dev/null +++ b/src/libfetchers/git-utils.hh @@ -0,0 +1,56 @@ +#pragma once + +#include "input-accessor.hh" + +namespace nix { + +struct GitRepo +{ + virtual ~GitRepo() + { } + + static ref openRepo(const CanonPath & path, bool create = false, bool bare = false); + + virtual uint64_t getRevCount(const Hash & rev) = 0; + + virtual uint64_t getLastModified(const Hash & rev) = 0; + + virtual bool isShallow() = 0; + + /* Return the commit hash to which a ref points. */ + virtual Hash resolveRef(std::string ref) = 0; + + struct WorkdirInfo + { + bool isDirty = false; + + /* The checked out commit, or nullopt if there are no commits + in the repo yet. */ + std::optional headRev; + + /* All files in the working directory that are unchanged, + modified or added, but excluding deleted files. */ + std::set files; + }; + + virtual WorkdirInfo getWorkdirInfo() = 0; + + /* Get the ref that HEAD points to. */ + virtual std::optional getWorkdirRef() = 0; + + struct TarballInfo + { + Hash treeHash; + time_t lastModified; + }; + + virtual bool hasObject(const Hash & oid) = 0; + + virtual ref getAccessor(const Hash & rev) = 0; + + virtual void fetch( + const std::string & url, + const std::string & refspec) = 0; +}; + +} diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 4bfd53b0e..55d3a8ebe 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -7,6 +7,8 @@ #include "pathlocks.hh" #include "util.hh" #include "git.hh" +#include "fs-input-accessor.hh" +#include "git-utils.hh" #include "fetch-settings.hh" @@ -137,121 +139,6 @@ bool isNotDotGitDirectory(const Path & path) return baseNameOf(path) != ".git"; } -struct WorkdirInfo -{ - bool clean = false; - bool hasHead = false; -}; - -// Returns whether a git workdir is clean and has commits. -WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir) -{ - const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); - std::string gitDir(".git"); - - auto env = getEnv(); - // Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong - // that way unknown errors can lead to a failure instead of continuing through the wrong code path - env["LC_ALL"] = "C"; - - /* Check whether HEAD points to something that looks like a commit, - since that is the refrence we want to use later on. */ - auto result = runProgram(RunOptions { - .program = "git", - .args = { "-C", workdir, "--git-dir", gitDir, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" }, - .environment = env, - .mergeStderrToStdout = true - }); - auto exitCode = WEXITSTATUS(result.first); - auto errorMessage = result.second; - - if (errorMessage.find("fatal: not a git repository") != std::string::npos) { - throw Error("'%s' is not a Git repository", workdir); - } else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) { - // indicates that the repo does not have any commits - // we want to proceed and will consider it dirty later - } else if (exitCode != 0) { - // any other errors should lead to a failure - throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", workdir, exitCode, errorMessage); - } - - bool clean = false; - bool hasHead = exitCode == 0; - - try { - if (hasHead) { - // Using git diff is preferrable over lower-level operations here, - // because its conceptually simpler and we only need the exit code anyways. - auto gitDiffOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "diff", "HEAD", "--quiet"}); - if (!submodules) { - // Changes in submodules should only make the tree dirty - // when those submodules will be copied as well. - gitDiffOpts.emplace_back("--ignore-submodules"); - } - gitDiffOpts.emplace_back("--"); - runProgram("git", true, gitDiffOpts); - - clean = true; - } - } catch (ExecError & e) { - if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; - } - - return WorkdirInfo { .clean = clean, .hasHead = hasHead }; -} - -std::pair fetchFromWorkdir(ref store, Input & input, const Path & workdir, const WorkdirInfo & workdirInfo) -{ - const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); - auto gitDir = ".git"; - - if (!fetchSettings.allowDirty) - throw Error("Git tree '%s' is dirty", workdir); - - if (fetchSettings.warnDirty) - warn("Git tree '%s' is dirty", workdir); - - auto gitOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "ls-files", "-z" }); - if (submodules) - gitOpts.emplace_back("--recurse-submodules"); - - auto files = tokenizeString>( - runProgram("git", true, gitOpts), "\0"s); - - Path actualPath(absPath(workdir)); - - PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, actualPath)); - std::string file(p, actualPath.size() + 1); - - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; - auto i = files.lower_bound(prefix); - return i != files.end() && hasPrefix(*i, prefix); - } - - return files.count(file); - }; - - auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); - - // FIXME: maybe we should use the timestamp of the last - // modified dirty file? - input.attrs.insert_or_assign( - "lastModified", - workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); - - if (workdirInfo.hasHead) { - input.attrs.insert_or_assign("dirtyRev", chomp( - runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "HEAD" })) + "-dirty"); - input.attrs.insert_or_assign("dirtyShortRev", chomp( - runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "--short", "HEAD" })) + "-dirty"); - } - - return {std::move(storePath), input}; -} } // end namespace struct GitInputScheme : InputScheme @@ -336,11 +223,11 @@ struct GitInputScheme : InputScheme void clone(const Input & input, const Path & destDir) const override { - auto [isLocal, actualUrl] = getActualUrl(input); + auto repoInfo = getRepoInfo(input); Strings args = {"clone"}; - args.push_back(actualUrl); + args.push_back(repoInfo.url); if (auto ref = input.getRef()) { args.push_back("--branch"); @@ -356,10 +243,9 @@ struct GitInputScheme : InputScheme std::optional getSourcePath(const Input & input) const override { - auto url = parseURL(getStrAttr(input.attrs, "url")); - if (url.scheme == "file" && !input.getRef() && !input.getRev()) - return url.path; - return {}; + auto repoInfo = getRepoInfo(input); + if (repoInfo.isLocal) return repoInfo.url; + return std::nullopt; } void putFile( @@ -368,24 +254,79 @@ struct GitInputScheme : InputScheme std::string_view contents, std::optional commitMsg) const override { - auto root = getSourcePath(input); - if (!root) + auto repoInfo = getRepoInfo(input); + if (!repoInfo.isLocal) throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string()); - writeFile((CanonPath(*root) + path).abs(), contents); - - auto gitDir = ".git"; + writeFile((CanonPath(repoInfo.url) + path).abs(), contents); runProgram("git", true, - { "-C", *root, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) }); + { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) }); if (commitMsg) runProgram("git", true, - { "-C", *root, "--git-dir", gitDir, "commit", std::string(path.rel()), "-m", *commitMsg }); + { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-m", *commitMsg }); } - std::pair getActualUrl(const Input & input) const + struct RepoInfo { + bool shallow = false; + bool submodules = false; + bool allRefs = false; + + std::string cacheType; + + /* Whether this is a local, non-bare repository. */ + bool isLocal = false; + + /* Working directory info: the complete list of files, and + whether the working directory is dirty compared to HEAD. */ + GitRepo::WorkdirInfo workdirInfo; + + /* URL of the repo, or its path if isLocal. */ + std::string url; + + void warnDirty() const + { + if (workdirInfo.isDirty) { + if (!fetchSettings.allowDirty) + throw Error("Git tree '%s' is dirty", url); + + if (fetchSettings.warnDirty) + warn("Git tree '%s' is dirty", url); + } + } + + std::string gitDir = ".git"; + }; + + bool getSubmodulesAttr(const Input & input) const + { + return maybeGetBoolAttr(input.attrs, "submodules").value_or(false); + } + + RepoInfo getRepoInfo(const Input & input) const + { + auto checkHashType = [&](const std::optional & hash) + { + if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256)) + throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true)); + }; + + if (auto rev = input.getRev()) + checkHashType(rev); + + RepoInfo repoInfo { + .shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false), + .submodules = getSubmodulesAttr(input), + .allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false) + }; + + repoInfo.cacheType = "git"; + if (repoInfo.shallow) repoInfo.cacheType += "-shallow"; + if (repoInfo.submodules) repoInfo.cacheType += "-submodules"; + if (repoInfo.allRefs) repoInfo.cacheType += "-all-refs"; + // file:// URIs are normally not cloned (but otherwise treated the // same as remote URIs, i.e. we don't use the working tree or // HEAD). Exception: If _NIX_FORCE_HTTP is set, or the repo is a bare git @@ -393,153 +334,142 @@ struct GitInputScheme : InputScheme static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing auto url = parseURL(getStrAttr(input.attrs, "url")); bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git"); - bool isLocal = url.scheme == "file" && !forceHttp && !isBareRepository; - return {isLocal, isLocal ? url.path : url.base}; + repoInfo.isLocal = url.scheme == "file" && !forceHttp && !isBareRepository; + repoInfo.url = repoInfo.isLocal ? url.path : url.base; + + // If this is a local directory and no ref or revision is + // given, then allow the use of an unclean working tree. + if (!input.getRef() && !input.getRev() && repoInfo.isLocal) + repoInfo.workdirInfo = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirInfo(); + + return repoInfo; } - std::pair fetch(ref store, const Input & _input) override + uint64_t getLastModified(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const { - Input input(_input); - auto gitDir = ".git"; + Attrs key{{"_what", "gitLastModified"}, {"rev", rev.gitRev()}}; + + auto cache = getCache(); + + if (auto res = cache->lookup(key)) + return getIntAttr(*res, "lastModified"); + + auto lastModified = GitRepo::openRepo(CanonPath(repoDir))->getLastModified(rev); + + cache->upsert(key, Attrs{{"lastModified", lastModified}}); + + return lastModified; + } + + uint64_t getRevCount(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const + { + Attrs key{{"_what", "gitRevCount"}, {"rev", rev.gitRev()}}; + + auto cache = getCache(); + + if (auto revCountAttrs = cache->lookup(key)) + return getIntAttr(*revCountAttrs, "revCount"); + + Activity act(*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.url)); + + auto revCount = GitRepo::openRepo(CanonPath(repoDir))->getRevCount(rev); + + cache->upsert(key, Attrs{{"revCount", revCount}}); + + return revCount; + } + + std::string getDefaultRef(const RepoInfo & repoInfo) const + { + auto head = repoInfo.isLocal + ? GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirRef() + : readHeadCached(repoInfo.url); + if (!head) { + warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.url); + return "master"; + } + return *head; + } + + static MakeNotAllowedError makeNotAllowedError(std::string url) + { + return [url{std::move(url)}](const CanonPath & path) -> RestrictedPathError + { + if (nix::pathExists(path.abs())) + return RestrictedPathError("access to path '%s' is forbidden because it is not under Git control; maybe you should 'git add' it to the repository '%s'?", path, url); + else + return RestrictedPathError("path '%s' does not exist in Git repository '%s'", path, url); + }; + } + + std::pair, Input> getAccessorFromCommit( + ref store, + RepoInfo & repoInfo, + Input && input) const + { + assert(!repoInfo.workdirInfo.isDirty); + + auto origRev = input.getRev(); std::string name = input.getName(); - bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); - bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); - bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); - - std::string cacheType = "git"; - if (shallow) cacheType += "-shallow"; - if (submodules) cacheType += "-submodules"; - if (allRefs) cacheType += "-all-refs"; - - auto checkHashType = [&](const std::optional & hash) - { - if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256)) - throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true)); - }; - - auto getLockedAttrs = [&]() - { - checkHashType(input.getRev()); - - return Attrs({ - {"type", cacheType}, - {"name", name}, - {"rev", input.getRev()->gitRev()}, - }); - }; - - auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair + auto makeResult2 = [&](const Attrs & infoAttrs, ref accessor) -> std::pair, Input> { assert(input.getRev()); - assert(!_input.getRev() || _input.getRev() == input.getRev()); - if (!shallow) + assert(!origRev || origRev == input.getRev()); + if (!repoInfo.shallow) input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); - return {std::move(storePath), input}; + + return {accessor, std::move(input)}; }; - if (input.getRev()) { - if (auto res = getCache()->lookup(store, getLockedAttrs())) - return makeResult(res->first, std::move(res->second)); - } + auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> std::pair, Input> + { + // FIXME: remove? + //input.attrs.erase("narHash"); + auto narHash = store->queryPathInfo(storePath)->narHash; + input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); - auto [isLocal, actualUrl_] = getActualUrl(input); - auto actualUrl = actualUrl_; // work around clang bug + auto accessor = makeStorePathAccessor(store, storePath, makeNotAllowedError(repoInfo.url)); - /* If this is a local directory and no ref or revision is given, - allow fetching directly from a dirty workdir. */ - if (!input.getRef() && !input.getRev() && isLocal) { - auto workdirInfo = getWorkdirInfo(input, actualUrl); - if (!workdirInfo.clean) { - return fetchFromWorkdir(store, input, actualUrl, workdirInfo); - } - } + return makeResult2(infoAttrs, accessor); + }; - Attrs unlockedAttrs({ - {"type", cacheType}, - {"name", name}, - {"url", actualUrl}, - }); + auto originalRef = input.getRef(); + auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo); + input.attrs.insert_or_assign("ref", ref); Path repoDir; - if (isLocal) { - if (!input.getRef()) { - auto head = readHead(actualUrl); - if (!head) { - warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl); - head = "master"; - } - input.attrs.insert_or_assign("ref", *head); - unlockedAttrs.insert_or_assign("ref", *head); - } - + if (repoInfo.isLocal) { + repoDir = repoInfo.url; if (!input.getRev()) - input.attrs.insert_or_assign("rev", - Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", *input.getRef() })), htSHA1).gitRev()); - - repoDir = actualUrl; + input.attrs.insert_or_assign("rev", GitRepo::openRepo(CanonPath(repoDir))->resolveRef(ref).gitRev()); } else { - const bool useHeadRef = !input.getRef(); - if (useHeadRef) { - auto head = readHeadCached(actualUrl); - if (!head) { - warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl); - head = "master"; - } - input.attrs.insert_or_assign("ref", *head); - unlockedAttrs.insert_or_assign("ref", *head); - } else { - if (!input.getRev()) { - unlockedAttrs.insert_or_assign("ref", input.getRef().value()); - } - } - - if (auto res = getCache()->lookup(store, unlockedAttrs)) { - auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); - if (!input.getRev() || input.getRev() == rev2) { - input.attrs.insert_or_assign("rev", rev2.gitRev()); - return makeResult(res->first, std::move(res->second)); - } - } - - Path cacheDir = getCachePath(actualUrl); + Path cacheDir = getCachePath(repoInfo.url); repoDir = cacheDir; - gitDir = "."; + repoInfo.gitDir = "."; createDirs(dirOf(cacheDir)); - PathLocks cacheDirLock({cacheDir + ".lock"}); + PathLocks cacheDirLock({cacheDir}); - if (!pathExists(cacheDir)) { - runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir }); - } + auto repo = GitRepo::openRepo(CanonPath(cacheDir), true, true); Path localRefFile = - input.getRef()->compare(0, 5, "refs/") == 0 - ? cacheDir + "/" + *input.getRef() - : cacheDir + "/refs/heads/" + *input.getRef(); + ref.compare(0, 5, "refs/") == 0 + ? cacheDir + "/" + ref + : cacheDir + "/refs/heads/" + ref; bool doFetch; time_t now = time(0); /* If a rev was specified, we need to fetch if it's not in the repo. */ - if (input.getRev()) { - try { - runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "cat-file", "-e", input.getRev()->gitRev() }); - doFetch = false; - } catch (ExecError & e) { - if (WIFEXITED(e.status)) { - doFetch = true; - } else { - throw; - } - } + if (auto rev = input.getRev()) { + doFetch = !repo->hasObject(*rev); } else { - if (allRefs) { + if (repoInfo.allRefs) { doFetch = true; } else { /* If the local ref is older than ‘tarball-ttl’ seconds, do a @@ -551,75 +481,80 @@ struct GitInputScheme : InputScheme } if (doFetch) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", repoInfo.url)); - // FIXME: git stderr messes up our progress indicator, so - // we're using --quiet for now. Should process its stderr. try { - auto ref = input.getRef(); - auto fetchRef = allRefs + auto fetchRef = repoInfo.allRefs ? "refs/*" - : ref->compare(0, 5, "refs/") == 0 - ? *ref - : ref == "HEAD" - ? *ref - : "refs/heads/" + *ref; - runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }, {}, true); + : ref.compare(0, 5, "refs/") == 0 + ? ref + : ref == "HEAD" + ? ref + : "refs/heads/" + ref; + + repo->fetch(repoInfo.url, fmt("%s:%s", fetchRef, fetchRef)); } catch (Error & e) { if (!pathExists(localRefFile)) throw; - warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); + logError(e.info()); + warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.url); } if (!touchCacheFile(localRefFile, now)) warn("could not update mtime for file '%s': %s", localRefFile, strerror(errno)); - if (useHeadRef && !storeCachedHead(actualUrl, *input.getRef())) - warn("could not update cached head '%s' for '%s'", *input.getRef(), actualUrl); + if (!originalRef && !storeCachedHead(repoInfo.url, ref)) + warn("could not update cached head '%s' for '%s'", ref, repoInfo.url); } - if (!input.getRev()) + if (auto rev = input.getRev()) { + if (!repo->hasObject(*rev)) + throw Error( + "Cannot find Git revision '%s' in ref '%s' of repository '%s'! " + "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " + ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD + "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", + rev->gitRev(), + ref, + repoInfo.url + ); + } else input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev()); // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder } - bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-parse", "--is-shallow-repository" })) == "true"; + auto isShallow = GitRepo::openRepo(CanonPath(repoDir))->isShallow(); - if (isShallow && !shallow) - throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified.", actualUrl); + if (isShallow && !repoInfo.shallow) + throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", repoInfo.url); - // FIXME: check whether rev is an ancestor of ref. + // FIXME: check whether rev is an ancestor of ref? - printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl); + auto rev = *input.getRev(); - /* Now that we know the ref, check again whether we have it in - the store. */ - if (auto res = getCache()->lookup(store, getLockedAttrs())) - return makeResult(res->first, std::move(res->second)); - - Path tmpDir = createTempDir(); - AutoDelete delTmpDir(tmpDir, true); - PathFilter filter = defaultPathFilter; - - auto result = runProgram(RunOptions { - .program = "git", - .args = { "-C", repoDir, "--git-dir", gitDir, "cat-file", "commit", input.getRev()->gitRev() }, - .mergeStderrToStdout = true + Attrs infoAttrs({ + {"rev", rev.gitRev()}, + {"lastModified", getLastModified(repoInfo, repoDir, rev)}, }); - if (WEXITSTATUS(result.first) == 128 - && result.second.find("bad file") != std::string::npos) - { - throw Error( - "Cannot find Git revision '%s' in ref '%s' of repository '%s'! " - "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " - ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD - "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", - input.getRev()->gitRev(), - *input.getRef(), - actualUrl - ); + + if (!repoInfo.shallow) + infoAttrs.insert_or_assign("revCount", + getRevCount(repoInfo, repoDir, rev)); + + printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.url); + + if (!repoInfo.submodules) { + auto accessor = GitRepo::openRepo(CanonPath(repoDir))->getAccessor(rev); + return makeResult2(infoAttrs, accessor); } - if (submodules) { + else { + // FIXME: use libgit2 + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + PathFilter filter = defaultPathFilter; + + Activity act(*logger, lvlChatty, actUnknown, fmt("copying Git tree '%s' to the store", input.to_string())); + Path tmpGitDir = createTempDir(); AutoDelete delTmpGitDir(tmpGitDir, true); @@ -634,77 +569,89 @@ struct GitInputScheme : InputScheme "--update-head-ok", "--", repoDir, "refs/*:refs/*" }, {}, true); } - runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() }); + runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", rev.gitRev() }); /* Ensure that we use the correct origin for fetching submodules. This matters for submodules with relative URLs. */ - if (isLocal) { - writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + gitDir + "/config")); + if (repoInfo.isLocal) { + writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + repoInfo.gitDir + "/config")); /* Restore the config.bare setting we may have just copied erroneously from the user's repo. */ runProgram("git", true, { "-C", tmpDir, "config", "core.bare", "false" }); } else - runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", actualUrl }); + runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", repoInfo.url }); /* As an optimisation, copy the modules directory of the source repo if it exists. */ - auto modulesPath = repoDir + "/" + gitDir + "/modules"; + auto modulesPath = repoDir + "/" + repoInfo.gitDir + "/modules"; if (pathExists(modulesPath)) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", actualUrl)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", repoInfo.url)); runProgram("cp", true, { "-R", "--", modulesPath, tmpGitDir + "/modules" }); } { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", actualUrl)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", repoInfo.url)); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, {}, true); } filter = isNotDotGitDirectory; - } else { - // FIXME: should pipe this, or find some better way to extract a - // revision. - auto source = sinkToSource([&](Sink & sink) { - runProgram2({ - .program = "git", - .args = { "-C", repoDir, "--git-dir", gitDir, "archive", input.getRev()->gitRev() }, - .standardOut = &sink - }); - }); - unpackTarfile(*source, tmpDir); + auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); + + return makeResult(infoAttrs, std::move(storePath)); + } + } + + std::pair, Input> getAccessorFromWorkdir( + RepoInfo & repoInfo, + Input && input) const + { + if (!repoInfo.workdirInfo.isDirty) { + if (auto ref = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirRef()) + input.attrs.insert_or_assign("ref", *ref); + + auto rev = repoInfo.workdirInfo.headRev.value(); + + input.attrs.insert_or_assign("rev", rev.gitRev()); + + input.attrs.insert_or_assign("revCount", getRevCount(repoInfo, repoInfo.url, rev)); + } else { + repoInfo.warnDirty(); + + if (repoInfo.workdirInfo.headRev) { + input.attrs.insert_or_assign("dirtyRev", + repoInfo.workdirInfo.headRev->gitRev() + "-dirty"); + input.attrs.insert_or_assign("dirtyShortRev", + repoInfo.workdirInfo.headRev->gitShortRev() + "-dirty"); + } } - auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); + input.attrs.insert_or_assign( + "lastModified", + repoInfo.workdirInfo.headRev + ? getLastModified(repoInfo, repoInfo.url, *repoInfo.workdirInfo.headRev) + : 0); - auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() })); + input.locked = true; // FIXME - Attrs infoAttrs({ - {"rev", input.getRev()->gitRev()}, - {"lastModified", lastModified}, - }); + return { + makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url)), + std::move(input) + }; + } - if (!shallow) - infoAttrs.insert_or_assign("revCount", - std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-list", "--count", input.getRev()->gitRev() }))); + std::pair, Input> getAccessor(ref store, const Input & _input) const override + { + Input input(_input); - if (!_input.getRev()) - getCache()->add( - store, - unlockedAttrs, - infoAttrs, - storePath, - false); + auto repoInfo = getRepoInfo(input); - getCache()->add( - store, - getLockedAttrs(), - infoAttrs, - storePath, - true); - - return makeResult(infoAttrs, std::move(storePath)); + if (input.getRef() || input.getRev() || !repoInfo.isLocal) + return getAccessorFromCommit(store, repoInfo, std::move(input)); + else + return getAccessorFromWorkdir(repoInfo, std::move(input)); } }; diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk index 2e8869d83..f21651d77 100644 --- a/src/libfetchers/local.mk +++ b/src/libfetchers/local.mk @@ -8,6 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc) libfetchers_CXXFLAGS += -I src/libutil -I src/libstore -libfetchers_LDFLAGS += -pthread +libfetchers_LDFLAGS += -pthread -lgit2 -larchive libfetchers_LIBS = libutil libstore diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index fc89f2040..c38cd27eb 100644 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -185,11 +185,7 @@ path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = # Nuke the cache rm -rf $TEST_HOME/.cache/nix -# Try again, but without 'git' on PATH. This should fail. -NIX=$(command -v nix) -(! PATH= $NIX eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" ) - -# Try again, with 'git' available. This should work. +# Try again. This should work. path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") [[ $path3 = $path5 ]] @@ -241,6 +237,7 @@ rm -rf $repo/.git # should succeed for a repo without commits git init $repo +git -C $repo add hello # need to add at least one file to cause the root of the repo to be visible path10=$(nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath") # should succeed for a path with a space diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh index 21a4b52de..6e24a80c1 100644 --- a/tests/functional/flakes/flake-in-submodule.sh +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -46,7 +46,8 @@ echo '"expression in root repo"' > $rootRepo/root.nix git -C $rootRepo add root.nix git -C $rootRepo commit -m "Add root.nix" +# FIXME # Flake can live inside a submodule and can be accessed via ?dir=submodule -[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#sub ) = '"expression in submodule"' ]] +#[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#sub ) = '"expression in submodule"' ]] # The flake can access content outside of the submodule -[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#root ) = '"expression in root repo"' ]] +#[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#root ) = '"expression in root repo"' ]] From d88106df24869104cc6c29c726ddfbbfda9dae10 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Oct 2023 18:39:00 +0200 Subject: [PATCH 491/909] Git fetcher: Improve submodule handling Instead of making a complete copy of the repo, fetching the submodules, and writing the result to the store (which is all superexpensive), we now fetch the submodules recursively using the Git fetcher, and return a union accessor that "mounts" the accessors for the submodules on top of the root accessor. --- src/libfetchers/git-utils.cc | 78 +++++++++++++++++ src/libfetchers/git-utils.hh | 12 +++ src/libfetchers/git.cc | 109 +++++++----------------- src/libfetchers/union-input-accessor.cc | 80 +++++++++++++++++ src/libfetchers/union-input-accessor.hh | 9 ++ tests/functional/fetchGitSubmodules.sh | 8 -- 6 files changed, 212 insertions(+), 84 deletions(-) create mode 100644 src/libfetchers/union-input-accessor.cc create mode 100644 src/libfetchers/union-input-accessor.hh diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 68e39580f..5e3e6dae4 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1,11 +1,13 @@ #include "git-utils.hh" #include "input-accessor.hh" #include "cache.hh" +#include "finally.hh" #include #include #include +#include #include #include #include @@ -14,6 +16,7 @@ #include #include #include +#include #include #include @@ -63,6 +66,8 @@ typedef std::unique_ptr> Reference; typedef std::unique_ptr> DescribeResult; typedef std::unique_ptr> StatusList; typedef std::unique_ptr> Remote; +typedef std::unique_ptr> GitConfig; +typedef std::unique_ptr> ConfigIterator; // A helper to ensure that we don't leak objects returned by libgit2. template @@ -256,6 +261,17 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return std::nullopt; } + std::vector getSubmodules(const Hash & rev) override; + + std::string resolveSubmoduleUrl(const std::string & url) override + { + git_buf buf = GIT_BUF_INIT; + if (git_submodule_resolve_url(&buf, *this, url.c_str())) + throw Error("resolving Git submodule URL '%s'", url); + Finally cleanup = [&]() { git_buf_dispose(&buf); }; + return buf.ptr; + } + bool hasObject(const Hash & oid_) override { auto oid = hashToOID(oid_); @@ -400,6 +416,16 @@ struct GitInputAccessor : InputAccessor return readBlob(path, true); } + Hash getSubmoduleRev(const CanonPath & path) + { + auto entry = need(path); + + if (git_tree_entry_type(entry) != GIT_OBJECT_COMMIT) + throw Error("'%s' is not a submodule", showPath(path)); + + return toHash(*git_tree_entry_id(entry)); + } + std::map lookupCache; /* Recursively look up 'path' relative to the root. */ @@ -495,4 +521,56 @@ ref GitRepoImpl::getAccessor(const Hash & rev) return make_ref(ref(shared_from_this()), rev); } +std::vector GitRepoImpl::getSubmodules(const Hash & rev) +{ + /* Read the .gitmodules files from this revision. */ + CanonPath modulesFile(".gitmodules"); + + auto accessor = getAccessor(rev); + if (!accessor->pathExists(modulesFile)) return {}; + + /* Parse it. */ + auto configS = accessor->readFile(modulesFile); + + auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules"); + writeFull(fdTemp.get(), configS); + + GitConfig config; + if (git_config_open_ondisk(Setter(config), pathTemp.c_str())) + throw Error("parsing .gitmodules file: %s", git_error_last()->message); + + ConfigIterator it; + if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$")) + throw Error("iterating over .gitmodules: %s", git_error_last()->message); + + std::map entries; + + while (true) { + git_config_entry * entry = nullptr; + if (auto err = git_config_next(&entry, it.get())) { + if (err == GIT_ITEROVER) break; + throw Error("iterating over .gitmodules: %s", git_error_last()->message); + } + entries.emplace(entry->name + 10, entry->value); + } + + std::vector result; + + for (auto & [key, value] : entries) { + if (!hasSuffix(key, ".path")) continue; + std::string key2(key, 0, key.size() - 5); + auto path = CanonPath(value); + auto rev = accessor.dynamic_pointer_cast()->getSubmoduleRev(path); + result.push_back(Submodule { + .path = path, + .url = entries[key2 + ".url"], + .branch = entries[key2 + ".branch"], + .rev = rev, + }); + } + + return result; +} + + } diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index dd2c06672..55e7ef969 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -38,6 +38,18 @@ struct GitRepo /* Get the ref that HEAD points to. */ virtual std::optional getWorkdirRef() = 0; + struct Submodule + { + CanonPath path; + std::string url; + std::string branch; + Hash rev; + }; + + virtual std::vector getSubmodules(const Hash & rev) = 0; + + virtual std::string resolveSubmoduleUrl(const std::string & url) = 0; + struct TarballInfo { Hash treeHash; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 55d3a8ebe..42b4aa23a 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -8,6 +8,7 @@ #include "util.hh" #include "git.hh" #include "fs-input-accessor.hh" +#include "union-input-accessor.hh" #include "git-utils.hh" #include "fetch-settings.hh" @@ -134,11 +135,6 @@ std::optional readHeadCached(const std::string & actualUrl) return std::nullopt; } -bool isNotDotGitDirectory(const Path & path) -{ - return baseNameOf(path) != ".git"; -} - } // end namespace struct GitInputScheme : InputScheme @@ -413,7 +409,7 @@ struct GitInputScheme : InputScheme std::string name = input.getName(); - auto makeResult2 = [&](const Attrs & infoAttrs, ref accessor) -> std::pair, Input> + auto makeResult = [&](const Attrs & infoAttrs, ref accessor) -> std::pair, Input> { assert(input.getRev()); assert(!origRev || origRev == input.getRev()); @@ -424,18 +420,6 @@ struct GitInputScheme : InputScheme return {accessor, std::move(input)}; }; - auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> std::pair, Input> - { - // FIXME: remove? - //input.attrs.erase("narHash"); - auto narHash = store->queryPathInfo(storePath)->narHash; - input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); - - auto accessor = makeStorePathAccessor(store, storePath, makeNotAllowedError(repoInfo.url)); - - return makeResult2(infoAttrs, accessor); - }; - auto originalRef = input.getRef(); auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo); input.attrs.insert_or_assign("ref", ref); @@ -542,66 +526,39 @@ struct GitInputScheme : InputScheme printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.url); - if (!repoInfo.submodules) { - auto accessor = GitRepo::openRepo(CanonPath(repoDir))->getAccessor(rev); - return makeResult2(infoAttrs, accessor); + auto repo = GitRepo::openRepo(CanonPath(repoDir)); + + auto accessor = repo->getAccessor(rev); + + /* If the repo has submodules, fetch them and return a union + input accessor consisting of the accessor for the top-level + repo and the accessors for the submodules. */ + if (repoInfo.submodules) { + std::map> mounts; + + for (auto & submodule : repo->getSubmodules(rev)) { + auto resolved = repo->resolveSubmoduleUrl(submodule.url); + debug("Git submodule %s: %s %s %s -> %s", + submodule.path, submodule.url, submodule.branch, submodule.rev.gitRev(), resolved); + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "git"); + attrs.insert_or_assign("url", resolved); + if (submodule.branch != "") + attrs.insert_or_assign("ref", submodule.branch); + attrs.insert_or_assign("rev", submodule.rev.gitRev()); + auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); + auto [submoduleAccessor, submoduleInput2] = + submoduleInput.scheme->getAccessor(store, submoduleInput); + mounts.insert_or_assign(submodule.path, submoduleAccessor); + } + + if (!mounts.empty()) { + mounts.insert_or_assign(CanonPath::root, accessor); + accessor = makeUnionInputAccessor(std::move(mounts)); + } } - else { - // FIXME: use libgit2 - Path tmpDir = createTempDir(); - AutoDelete delTmpDir(tmpDir, true); - PathFilter filter = defaultPathFilter; - - Activity act(*logger, lvlChatty, actUnknown, fmt("copying Git tree '%s' to the store", input.to_string())); - - Path tmpGitDir = createTempDir(); - AutoDelete delTmpGitDir(tmpGitDir, true); - - runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", tmpDir, "--separate-git-dir", tmpGitDir }); - - { - // TODO: repoDir might lack the ref (it only checks if rev - // exists, see FIXME above) so use a big hammer and fetch - // everything to ensure we get the rev. - Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir)); - runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", - "--update-head-ok", "--", repoDir, "refs/*:refs/*" }, {}, true); - } - - runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", rev.gitRev() }); - - /* Ensure that we use the correct origin for fetching - submodules. This matters for submodules with relative - URLs. */ - if (repoInfo.isLocal) { - writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + repoInfo.gitDir + "/config")); - - /* Restore the config.bare setting we may have just - copied erroneously from the user's repo. */ - runProgram("git", true, { "-C", tmpDir, "config", "core.bare", "false" }); - } else - runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", repoInfo.url }); - - /* As an optimisation, copy the modules directory of the - source repo if it exists. */ - auto modulesPath = repoDir + "/" + repoInfo.gitDir + "/modules"; - if (pathExists(modulesPath)) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", repoInfo.url)); - runProgram("cp", true, { "-R", "--", modulesPath, tmpGitDir + "/modules" }); - } - - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", repoInfo.url)); - runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, {}, true); - } - - filter = isNotDotGitDirectory; - - auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); - - return makeResult(infoAttrs, std::move(storePath)); - } + return makeResult(infoAttrs, accessor); } std::pair, Input> getAccessorFromWorkdir( diff --git a/src/libfetchers/union-input-accessor.cc b/src/libfetchers/union-input-accessor.cc new file mode 100644 index 000000000..940c0e06c --- /dev/null +++ b/src/libfetchers/union-input-accessor.cc @@ -0,0 +1,80 @@ +#include "union-input-accessor.hh" + +namespace nix { + +struct UnionInputAccessor : InputAccessor +{ + std::map> mounts; + + UnionInputAccessor(std::map> _mounts) + : mounts(std::move(_mounts)) + { + // Currently we require a root filesystem. This could be relaxed. + assert(mounts.contains(CanonPath::root)); + + // FIXME: should check that every mount point exists. Or we + // could return dummy parent directories automatically. + } + + std::string readFile(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->readFile(subpath); + } + + bool pathExists(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->pathExists(subpath); + } + + Stat lstat(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->lstat(subpath); + } + + DirEntries readDirectory(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->readDirectory(subpath); + } + + std::string readLink(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->readLink(subpath); + } + + std::string showPath(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->showPath(subpath); + } + + std::pair, CanonPath> resolve(CanonPath path) + { + // Find the nearest parent of `path` that is a mount point. + std::vector ss; + while (true) { + auto i = mounts.find(path); + if (i != mounts.end()) { + auto subpath = CanonPath::root; + for (auto j = ss.rbegin(); j != ss.rend(); ++j) + subpath.push(*j); + return {i->second, std::move(subpath)}; + } + + assert(!path.isRoot()); + ss.push_back(std::string(*path.baseName())); + path.pop(); + } + } +}; + +ref makeUnionInputAccessor(std::map> mounts) +{ + return make_ref(std::move(mounts)); +} + +} diff --git a/src/libfetchers/union-input-accessor.hh b/src/libfetchers/union-input-accessor.hh new file mode 100644 index 000000000..6a1649c1d --- /dev/null +++ b/src/libfetchers/union-input-accessor.hh @@ -0,0 +1,9 @@ +#pragma once + +#include "input-accessor.hh" + +namespace nix { + +ref makeUnionInputAccessor(std::map> mounts); + +} diff --git a/tests/functional/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh index df81232e5..369cdc5db 100644 --- a/tests/functional/fetchGitSubmodules.sh +++ b/tests/functional/fetchGitSubmodules.sh @@ -118,11 +118,3 @@ cloneRepo=$TEST_ROOT/a/b/gitSubmodulesClone # NB /a/b to make the relative path git clone $rootRepo $cloneRepo pathIndirect=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath") [[ $pathIndirect = $pathWithRelative ]] - -# Test that if the clone has the submodule already, we're not fetching -# it again. -git -C $cloneRepo submodule update --init -rm $TEST_HOME/.cache/nix/fetcher-cache* -rm -rf $subRepo -pathSubmoduleGone=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath") -[[ $pathSubmoduleGone = $pathWithRelative ]] From 669b074f51c4fea6b362313f47eebb4a67f0e89d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 28 Oct 2023 16:16:20 +0200 Subject: [PATCH 492/909] Cleanup --- src/libfetchers/git.cc | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 42b4aa23a..a66a51cca 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -409,17 +409,6 @@ struct GitInputScheme : InputScheme std::string name = input.getName(); - auto makeResult = [&](const Attrs & infoAttrs, ref accessor) -> std::pair, Input> - { - assert(input.getRev()); - assert(!origRev || origRev == input.getRev()); - if (!repoInfo.shallow) - input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); - input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); - - return {accessor, std::move(input)}; - }; - auto originalRef = input.getRef(); auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo); input.attrs.insert_or_assign("ref", ref); @@ -558,7 +547,13 @@ struct GitInputScheme : InputScheme } } - return makeResult(infoAttrs, accessor); + assert(input.getRev()); + assert(!origRev || origRev == input.getRev()); + if (!repoInfo.shallow) + input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); + input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); + + return {accessor, std::move(input)}; } std::pair, Input> getAccessorFromWorkdir( From 1fd0867389c2dd3e98d06decd4d35067885550a0 Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Fri, 11 Aug 2023 21:47:16 +0200 Subject: [PATCH 493/909] Fix missing output when creating lockfile --- src/libexpr/flake/flake.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 45c9ec8f3..8cc803ccf 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -651,14 +651,14 @@ LockedFlake lockFlake( bool lockFileExists = pathExists(outputLockFilePath); + auto s = chomp(diff); if (lockFileExists) { - auto s = chomp(diff); if (s.empty()) warn("updating lock file '%s'", outputLockFilePath); else warn("updating lock file '%s':\n%s", outputLockFilePath, s); } else - warn("creating lock file '%s'", outputLockFilePath); + warn("creating lock file '%s': \n%s", outputLockFilePath, s); std::optional commitMessage = std::nullopt; From c762b65dc5314ed631381cf4bf26f5976e825bdc Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Fri, 11 Aug 2023 21:51:03 +0200 Subject: [PATCH 494/909] Fix documentation of flake command output --- src/nix/flake-lock.md | 7 +++++-- src/nix/flake-update.md | 11 ++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/nix/flake-lock.md b/src/nix/flake-lock.md index 2af0ad81e..100987a88 100644 --- a/src/nix/flake-lock.md +++ b/src/nix/flake-lock.md @@ -7,8 +7,11 @@ R""( ```console # nix flake lock --update-input nixpkgs --update-input nix - * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' - * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' + warning: creating lock file '/home/myself/repos/testflake/flake.lock': + • Added input 'nix': + 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28) + • Added input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) ``` # Description diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md index 8c6042d94..b5a5ff0ec 100644 --- a/src/nix/flake-update.md +++ b/src/nix/flake-update.md @@ -6,9 +6,14 @@ R""( lock file: ```console - # nix flake update --commit-lock-file - * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' - * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' + # nix flake update + warning: updating lock file '/home/myself/repos/testflake/flake.lock': + • Updated input 'nix': + 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28) + → 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' (2023-07-11) + • Updated input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) … warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad' ``` From c7dcdb8325be7b8ecc3d480217808be899fc865a Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Sat, 12 Aug 2023 20:51:19 +0200 Subject: [PATCH 495/909] Overhaul nix flake update and lock commands Closes #5110 --- doc/manual/src/release-notes/rl-next.md | 14 +++++++ src/libcmd/command.hh | 6 +++ src/libcmd/installables.cc | 22 +--------- src/libexpr/flake/flake.cc | 11 +++-- src/nix/flake-lock.md | 45 ++++++++++----------- src/nix/flake-update.md | 53 +++++++++++++++++-------- src/nix/flake.cc | 33 +++++++++++++-- tests/functional/completions.sh | 7 ++-- tests/functional/flakes/circular.sh | 3 +- tests/functional/flakes/flakes.sh | 6 +-- tests/functional/flakes/follow-paths.sh | 4 +- 11 files changed, 124 insertions(+), 80 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 276252c37..3cfb53998 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -15,3 +15,17 @@ - `nix-shell` shebang lines now support single-quoted arguments. - `builtins.fetchTree` is now marked as stable. + + +- The interface for creating and updating lock files has been overhauled: + + - [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now. + It will *never* update existing inputs. + + - [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) does the same, but *will* update inputs. + - Passing no arguments will update all inputs of the current flake, just like it already did. + - Passing input names as arguments will ensure only those are updated. This replaces the functionality of `nix flake lock --update-input` + - To operate on a flake outside the current directory, you must now pass `--flake path/to/flake`. + + - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. + They are superceded by `nix flake update`. diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index dafc0db3b..120c832ac 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -326,6 +326,12 @@ struct MixEnvironment : virtual Args { void setEnviron(); }; +void completeFlakeInputPath( + AddCompletions & completions, + ref evalState, + const std::vector & flakeRefs, + std::string_view prefix); + void completeFlakeRef(AddCompletions & completions, ref store, std::string_view prefix); void completeFlakeRefWithFragment( diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index eff18bbf6..3aff601e0 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -28,7 +28,7 @@ namespace nix { -static void completeFlakeInputPath( +void completeFlakeInputPath( AddCompletions & completions, ref evalState, const std::vector & flakeRefs, @@ -46,13 +46,6 @@ MixFlakeOptions::MixFlakeOptions() { auto category = "Common flake-related options"; - addFlag({ - .longName = "recreate-lock-file", - .description = "Recreate the flake's lock file from scratch.", - .category = category, - .handler = {&lockFlags.recreateLockFile, true} - }); - addFlag({ .longName = "no-update-lock-file", .description = "Do not allow any updates to the flake's lock file.", @@ -85,19 +78,6 @@ MixFlakeOptions::MixFlakeOptions() .handler = {&lockFlags.commitLockFile, true} }); - addFlag({ - .longName = "update-input", - .description = "Update a specific flake input (ignoring its previous entry in the lock file).", - .category = category, - .labels = {"input-path"}, - .handler = {[&](std::string s) { - lockFlags.inputUpdates.insert(flake::parseInputPath(s)); - }}, - .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { - completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); - }} - }); - addFlag({ .longName = "override-input", .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.", diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 8cc803ccf..70ae7b584 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -447,8 +447,8 @@ LockedFlake lockFlake( assert(input.ref); - /* Do we have an entry in the existing lock file? And we - don't have a --update-input flag for this input? */ + /* Do we have an entry in the existing lock file? + And the input is not in updateInputs? */ std::shared_ptr oldLock; updatesUsed.insert(inputPath); @@ -472,9 +472,8 @@ LockedFlake lockFlake( node->inputs.insert_or_assign(id, childNode); - /* If we have an --update-input flag for an input - of this input, then we must fetch the flake to - update it. */ + /* If we have this input in updateInputs, then we + must fetch the flake to update it. */ auto lb = lockFlags.inputUpdates.lower_bound(inputPath); auto mustRefetch = @@ -616,7 +615,7 @@ LockedFlake lockFlake( for (auto & i : lockFlags.inputUpdates) if (!updatesUsed.count(i)) - warn("the flag '--update-input %s' does not match any input", printInputPath(i)); + warn("'%s' does not match any input of this flake", printInputPath(i)); /* Check 'follows' inputs. */ newLockFile.check(); diff --git a/src/nix/flake-lock.md b/src/nix/flake-lock.md index 100987a88..6d10258e3 100644 --- a/src/nix/flake-lock.md +++ b/src/nix/flake-lock.md @@ -2,11 +2,10 @@ R""( # Examples -* Update the `nixpkgs` and `nix` inputs of the flake in the current - directory: +* Create the lock file for the flake in the current directory: ```console - # nix flake lock --update-input nixpkgs --update-input nix + # nix flake lock warning: creating lock file '/home/myself/repos/testflake/flake.lock': • Added input 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28) @@ -14,28 +13,28 @@ R""( 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) ``` +* Add missing inputs to the lock file for a flake in a different directory: + + ```console + # nix flake lock ~/repos/another + warning: updating lock file '/home/myself/repos/another/flake.lock': + • Added input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + ``` + + > **Note** + > + > When trying to refer to a flake in a subdirectory, write `./another` + > instead of `another`. + > Otherwise Nix will try to look up the flake in the registry. + # Description -This command updates the lock file of a flake (`flake.lock`) so that -it contains a lock for every flake input specified in -`flake.nix`. Existing lock file entries are not updated unless -required by a flag such as `--update-input`. +This command adds inputs to the lock file of a flake (`flake.lock`) +so that it contains a lock for every flake input specified in +`flake.nix`. Existing lock file entries are not updated. -Note that every command that operates on a flake will also update the -lock file if needed, and supports the same flags. Therefore, - -```console -# nix flake lock --update-input nixpkgs -# nix build -``` - -is equivalent to: - -```console -# nix build --update-input nixpkgs -``` - -Thus, this command is only useful if you want to update the lock file -separately from any other action such as building. +If you want to update existing lock entries, use +[`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) )"" diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md index b5a5ff0ec..63df3b12a 100644 --- a/src/nix/flake-update.md +++ b/src/nix/flake-update.md @@ -2,8 +2,7 @@ R""( # Examples -* Recreate the lock file (i.e. update all inputs) and commit the new - lock file: +* Update all inputs (i.e. recreate the lock file from scratch): ```console # nix flake update @@ -14,26 +13,46 @@ R""( • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) - … - warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad' ``` +* Update only a single input: + + ```console + # nix flake update nixpkgs + warning: updating lock file '/home/myself/repos/testflake/flake.lock': + • Updated input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) + ``` + +* Update only a single input of a flake in a different directory: + + ```console + # nix flake update nixpkgs --flake ~/repos/another + warning: updating lock file '/home/myself/repos/another/flake.lock': + • Updated input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) + ``` + + > **Note** + > + > When trying to refer to a flake in a subdirectory, write `./another` + > instead of `another`. + > Otherwise Nix will try to look up the flake in the registry. + # Description -This command recreates the lock file of a flake (`flake.lock`), thus -updating the lock for every unlocked input (like `nixpkgs`) to its -current version. This is equivalent to passing `--recreate-lock-file` -to any command that operates on a flake. That is, +This command updates the inputs in a lock file (`flake.lock`). +**By default, all inputs are updated**. If the lock file doesn't exist +yet, it will be created. If inputs are not in the lock file yet, they will be added. -```console -# nix flake update -# nix build -``` +Unlike other `nix flake` commands, `nix flake update` takes a list of names of inputs +to update as its positional arguments and operates on the flake in the current directory. +You can pass a different flake-url with `--flake` to override that default. -is equivalent to: - -```console -# nix build --recreate-lock-file -``` +The related command [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) +also creates lock files and adds missing inputs, but is safer as it +will never update inputs already in the lock file. )"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 0116eff2e..e8906a252 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -24,8 +24,10 @@ using namespace nix; using namespace nix::flake; using json = nlohmann::json; +struct CmdFlakeUpdate; class FlakeCommand : virtual Args, public MixFlakeOptions { +protected: std::string flakeUrl = "."; public: @@ -63,6 +65,8 @@ public: struct CmdFlakeUpdate : FlakeCommand { +public: + std::string description() override { return "update flake lock file"; @@ -70,9 +74,31 @@ struct CmdFlakeUpdate : FlakeCommand CmdFlakeUpdate() { + expectedArgs.clear(); + addFlag({ + .longName="flake", + .description="The flake to operate on. Default is the current directory.", + .labels={"flake-url"}, + .handler={&flakeUrl}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); + }} + }); + expectArgs({ + .label="inputs", + .optional=true, + .handler={[&](std::string inputToUpdate){ + auto inputPath = flake::parseInputPath(inputToUpdate); + if (lockFlags.inputUpdates.contains(inputPath)) + warn("Input '%s' was specified multiple times. You may have done this by accident."); + lockFlags.inputUpdates.insert(inputPath); + }}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); + }} + }); + /* Remove flags that don't make sense. */ - removeFlag("recreate-lock-file"); - removeFlag("update-input"); removeFlag("no-update-lock-file"); removeFlag("no-write-lock-file"); } @@ -87,8 +113,9 @@ struct CmdFlakeUpdate : FlakeCommand void run(nix::ref store) override { settings.tarballTtl = 0; + auto updateAll = lockFlags.inputUpdates.empty(); - lockFlags.recreateLockFile = true; + lockFlags.recreateLockFile = updateAll; lockFlags.writeLockFile = true; lockFlags.applyNixConfig = true; diff --git a/tests/functional/completions.sh b/tests/functional/completions.sh index 7c1e4b287..b9886623a 100644 --- a/tests/functional/completions.sh +++ b/tests/functional/completions.sh @@ -48,11 +48,10 @@ EOF [[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]] ## With tilde expansion [[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]] -[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake show '~/foo' --update-input '')" == $'normal\na\t' ]] -[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix run '~/foo' --update-input '')" == $'normal\na\t' ]] +[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake update --flake '~/foo' '')" == $'normal\na\t' ]] ## Out of order -[[ "$(NIX_GET_COMPLETIONS=3 nix build --update-input '' ./foo)" == $'normal\na\t' ]] -[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --update-input '' ./bar)" == $'normal\na\t\nb\t' ]] +[[ "$(NIX_GET_COMPLETIONS=3 nix build --override-input '' '' ./foo)" == $'normal\na\t' ]] +[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '' '' ./bar)" == $'normal\na\t\nb\t' ]] # Cli flag completion NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format" diff --git a/tests/functional/flakes/circular.sh b/tests/functional/flakes/circular.sh index 09cd02edf..d3bb8e8a3 100644 --- a/tests/functional/flakes/circular.sh +++ b/tests/functional/flakes/circular.sh @@ -42,7 +42,8 @@ git -C $flakeB commit -a -m 'Foo' sed -i $flakeB/flake.nix -e 's/456/789/' git -C $flakeB commit -a -m 'Foo' -[[ $(nix eval --update-input b $flakeA#foo) = 1912 ]] +nix flake update b --flake $flakeA +[[ $(nix eval $flakeA#foo) = 1912 ]] # Test list-inputs with circular dependencies nix flake metadata $flakeA diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 70de28628..b0038935c 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -300,7 +300,7 @@ nix build -o "$TEST_ROOT/result" flake4#xyzzy nix flake lock "$flake3Dir" [[ -z $(git -C "$flake3Dir" diff master || echo failed) ]] -nix flake update "$flake3Dir" --override-flake flake2 nixpkgs +nix flake update --flake "$flake3Dir" --override-flake flake2 nixpkgs [[ ! -z $(git -C "$flake3Dir" diff master || echo failed) ]] # Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore @@ -437,7 +437,7 @@ cat > "$flake3Dir/flake.nix" < Date: Tue, 24 Oct 2023 22:22:05 +0200 Subject: [PATCH 496/909] fix: segfault in positional arg completion Adding the inputPath as a positional feature uncovered this bug. As positional argument forms were discarded from the `expectedArgs` list, their closures were not. When the `.completer` closure was then called, part of the surrounding object did not exist anymore. This didn't cause an issue before, but with the new call to `getEvalState()` in the "inputs" completer in nix/flake.cc, a segfault was triggered reproducibly on invalid memory access to the `this` pointer, which was always 0. The solution of splicing the argument forms into a new list to extend their lifetime is a bit of a hack, but I was unable to get the "nicer" iterator-based solution to work. --- src/libutil/args.cc | 13 ++++++++++++- src/libutil/args.hh | 14 +++++++++++++- tests/functional/completions.sh | 4 ++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 6bc3cae07..811353c18 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -255,7 +255,18 @@ bool Args::processArgs(const Strings & args, bool finish) } if (!anyCompleted) exp.handler.fun(ss); - expectedArgs.pop_front(); + + /* Move the list element to the processedArgs. This is almost the same as + `processedArgs.push_back(expectedArgs.front()); expectedArgs.pop_front()`, + except that it will only adjust the next and prev pointers of the list + elements, meaning the actual contents don't move in memory. This is + critical to prevent invalidating internal pointers! */ + processedArgs.splice( + processedArgs.end(), + expectedArgs, + expectedArgs.begin(), + ++expectedArgs.begin()); + res = true; } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index ff2bf3cab..e3b41313f 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -200,13 +200,25 @@ protected: /** * Queue of expected positional argument forms. * - * Positional arugment descriptions are inserted on the back. + * Positional argument descriptions are inserted on the back. * * As positional arguments are passed, these are popped from the * front, until there are hopefully none left as all args that were * expected in fact were passed. */ std::list expectedArgs; + /** + * List of processed positional argument forms. + * + * All items removed from `expectedArgs` are added here. After all + * arguments were processed, this list should be exactly the same as + * `expectedArgs` was before. + * + * This list is used to extend the lifetime of the argument forms. + * If this is not done, some closures that reference the command + * itself will segfault. + */ + std::list processedArgs; /** * Process some positional arugments diff --git a/tests/functional/completions.sh b/tests/functional/completions.sh index b9886623a..d3d5bbd48 100644 --- a/tests/functional/completions.sh +++ b/tests/functional/completions.sh @@ -44,6 +44,10 @@ EOF # Input override completion [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]] [[ "$(NIX_GET_COMPLETIONS=5 nix flake show ./foo --override-input '')" == $'normal\na\t' ]] +cd ./foo +[[ "$(NIX_GET_COMPLETIONS=3 nix flake update '')" == $'normal\na\t' ]] +cd .. +[[ "$(NIX_GET_COMPLETIONS=5 nix flake update --flake './foo' '')" == $'normal\na\t' ]] ## With multiple input flakes [[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]] ## With tilde expansion From 0c5eac9c4550a6de2cd829d25e628f779e2a29c7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 31 Oct 2023 15:59:25 +0100 Subject: [PATCH 497/909] Git fetcher: Handle submodules for workdirs --- src/libfetchers/git-utils.cc | 83 +++++++++++-------- src/libfetchers/git-utils.hh | 27 ++++-- src/libfetchers/git.cc | 49 +++++++++-- tests/functional/flakes/flake-in-submodule.sh | 14 +++- 4 files changed, 119 insertions(+), 54 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 5e3e6dae4..5b14cfdb1 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -216,6 +216,43 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return toHash(*oid); } + std::vector parseSubmodules(const CanonPath & configFile) + { + GitConfig config; + if (git_config_open_ondisk(Setter(config), configFile.abs().c_str())) + throw Error("parsing .gitmodules file: %s", git_error_last()->message); + + ConfigIterator it; + if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$")) + throw Error("iterating over .gitmodules: %s", git_error_last()->message); + + std::map entries; + + while (true) { + git_config_entry * entry = nullptr; + if (auto err = git_config_next(&entry, it.get())) { + if (err == GIT_ITEROVER) break; + throw Error("iterating over .gitmodules: %s", git_error_last()->message); + } + entries.emplace(entry->name + 10, entry->value); + } + + std::vector result; + + for (auto & [key, value] : entries) { + if (!hasSuffix(key, ".path")) continue; + std::string key2(key, 0, key.size() - 5); + auto path = CanonPath(value); + result.push_back(Submodule { + .path = path, + .url = entries[key2 + ".url"], + .branch = entries[key2 + ".branch"], + }); + } + + return result; + } + WorkdirInfo getWorkdirInfo() override { WorkdirInfo info; @@ -246,6 +283,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this if (git_status_foreach_ext(*this, &options, &statusCallbackTrampoline, &statusCallback)) throw Error("getting working directory status: %s", git_error_last()->message); + /* Get submodule info. */ + auto modulesFile = path + ".gitmodules"; + if (pathExists(modulesFile.abs())) + info.submodules = parseSubmodules(modulesFile); + return info; } @@ -261,7 +303,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return std::nullopt; } - std::vector getSubmodules(const Hash & rev) override; + std::vector> getSubmodules(const Hash & rev) override; std::string resolveSubmoduleUrl(const std::string & url) override { @@ -521,7 +563,7 @@ ref GitRepoImpl::getAccessor(const Hash & rev) return make_ref(ref(shared_from_this()), rev); } -std::vector GitRepoImpl::getSubmodules(const Hash & rev) +std::vector> GitRepoImpl::getSubmodules(const Hash & rev) { /* Read the .gitmodules files from this revision. */ CanonPath modulesFile(".gitmodules"); @@ -529,44 +571,17 @@ std::vector GitRepoImpl::getSubmodules(const Hash & rev) auto accessor = getAccessor(rev); if (!accessor->pathExists(modulesFile)) return {}; - /* Parse it. */ + /* Parse it and get the revision of each submodule. */ auto configS = accessor->readFile(modulesFile); auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules"); writeFull(fdTemp.get(), configS); - GitConfig config; - if (git_config_open_ondisk(Setter(config), pathTemp.c_str())) - throw Error("parsing .gitmodules file: %s", git_error_last()->message); + std::vector> result; - ConfigIterator it; - if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$")) - throw Error("iterating over .gitmodules: %s", git_error_last()->message); - - std::map entries; - - while (true) { - git_config_entry * entry = nullptr; - if (auto err = git_config_next(&entry, it.get())) { - if (err == GIT_ITEROVER) break; - throw Error("iterating over .gitmodules: %s", git_error_last()->message); - } - entries.emplace(entry->name + 10, entry->value); - } - - std::vector result; - - for (auto & [key, value] : entries) { - if (!hasSuffix(key, ".path")) continue; - std::string key2(key, 0, key.size() - 5); - auto path = CanonPath(value); - auto rev = accessor.dynamic_pointer_cast()->getSubmoduleRev(path); - result.push_back(Submodule { - .path = path, - .url = entries[key2 + ".url"], - .branch = entries[key2 + ".branch"], - .rev = rev, - }); + for (auto & submodule : parseSubmodules(CanonPath(pathTemp))) { + auto rev = accessor.dynamic_pointer_cast()->getSubmoduleRev(submodule.path); + result.push_back({std::move(submodule), rev}); } return result; diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 55e7ef969..a425e5814 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -20,6 +20,16 @@ struct GitRepo /* Return the commit hash to which a ref points. */ virtual Hash resolveRef(std::string ref) = 0; + /** + * Info about a submodule. + */ + struct Submodule + { + CanonPath path; + std::string url; + std::string branch; + }; + struct WorkdirInfo { bool isDirty = false; @@ -31,6 +41,9 @@ struct GitRepo /* All files in the working directory that are unchanged, modified or added, but excluding deleted files. */ std::set files; + + /* The submodules listed in .gitmodules of this workdir. */ + std::vector submodules; }; virtual WorkdirInfo getWorkdirInfo() = 0; @@ -38,15 +51,11 @@ struct GitRepo /* Get the ref that HEAD points to. */ virtual std::optional getWorkdirRef() = 0; - struct Submodule - { - CanonPath path; - std::string url; - std::string branch; - Hash rev; - }; - - virtual std::vector getSubmodules(const Hash & rev) = 0; + /** + * Return the submodules of this repo at the indicated revision, + * along with the revision of each submodule. + */ + virtual std::vector> getSubmodules(const Hash & rev) = 0; virtual std::string resolveSubmoduleUrl(const std::string & url) = 0; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index a66a51cca..5471eb260 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -525,16 +525,16 @@ struct GitInputScheme : InputScheme if (repoInfo.submodules) { std::map> mounts; - for (auto & submodule : repo->getSubmodules(rev)) { + for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev)) { auto resolved = repo->resolveSubmoduleUrl(submodule.url); debug("Git submodule %s: %s %s %s -> %s", - submodule.path, submodule.url, submodule.branch, submodule.rev.gitRev(), resolved); + submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved); fetchers::Attrs attrs; attrs.insert_or_assign("type", "git"); attrs.insert_or_assign("url", resolved); if (submodule.branch != "") attrs.insert_or_assign("ref", submodule.branch); - attrs.insert_or_assign("rev", submodule.rev.gitRev()); + attrs.insert_or_assign("rev", submoduleRev.gitRev()); auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = submoduleInput.scheme->getAccessor(store, submoduleInput); @@ -557,9 +557,45 @@ struct GitInputScheme : InputScheme } std::pair, Input> getAccessorFromWorkdir( + ref store, RepoInfo & repoInfo, Input && input) const { + if (repoInfo.submodules) + /* Create mountpoints for the submodules. */ + for (auto & submodule : repoInfo.workdirInfo.submodules) + repoInfo.workdirInfo.files.insert(submodule.path); + + ref accessor = + makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url)); + + /* If the repo has submodules, return a union input accessor + consisting of the accessor for the top-level repo and the + accessors for the submodule workdirs. */ + if (repoInfo.submodules && !repoInfo.workdirInfo.submodules.empty()) { + std::map> mounts; + + for (auto & submodule : repoInfo.workdirInfo.submodules) { + auto submodulePath = CanonPath(repoInfo.url) + submodule.path; + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "git"); + attrs.insert_or_assign("url", submodulePath.abs()); + auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); + auto [submoduleAccessor, submoduleInput2] = + submoduleInput.scheme->getAccessor(store, submoduleInput); + + /* If the submodule is dirty, mark this repo dirty as + well. */ + if (!submoduleInput2.getRev()) + repoInfo.workdirInfo.isDirty = true; + + mounts.insert_or_assign(submodule.path, submoduleAccessor); + } + + mounts.insert_or_assign(CanonPath::root, accessor); + accessor = makeUnionInputAccessor(std::move(mounts)); + } + if (!repoInfo.workdirInfo.isDirty) { if (auto ref = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirRef()) input.attrs.insert_or_assign("ref", *ref); @@ -588,10 +624,7 @@ struct GitInputScheme : InputScheme input.locked = true; // FIXME - return { - makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url)), - std::move(input) - }; + return {accessor, std::move(input)}; } std::pair, Input> getAccessor(ref store, const Input & _input) const override @@ -603,7 +636,7 @@ struct GitInputScheme : InputScheme if (input.getRef() || input.getRev() || !repoInfo.isLocal) return getAccessorFromCommit(store, repoInfo, std::move(input)); else - return getAccessorFromWorkdir(repoInfo, std::move(input)); + return getAccessorFromWorkdir(store, repoInfo, std::move(input)); } }; diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh index 6e24a80c1..85a4d3389 100644 --- a/tests/functional/flakes/flake-in-submodule.sh +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -46,8 +46,16 @@ echo '"expression in root repo"' > $rootRepo/root.nix git -C $rootRepo add root.nix git -C $rootRepo commit -m "Add root.nix" -# FIXME +flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule + # Flake can live inside a submodule and can be accessed via ?dir=submodule -#[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#sub ) = '"expression in submodule"' ]] +[[ $(nix eval --json $flakeref#sub ) = '"expression in submodule"' ]] + # The flake can access content outside of the submodule -#[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#root ) = '"expression in root repo"' ]] +[[ $(nix eval --json $flakeref#root ) = '"expression in root repo"' ]] + +# Check that dirtying a submodule makes the entire thing dirty. +[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) != null ]] +echo '"foo"' > $rootRepo/submodule/sub.nix +[[ $(nix eval --json $flakeref#sub ) = '"foo"' ]] +[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) = null ]] From 1f4525531e9b5e744830a55a2595880b135d93c0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 31 Oct 2023 12:01:13 -0400 Subject: [PATCH 498/909] Add configure test to ensure GCC bug is fixed https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431 (test is adapted from issue, test does not test for GCC-specific behavior but rather absence of bug, so test is good with other compilers too.) --- configure.ac | 3 +++ m4/gcc_bug_80431.m4 | 64 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 m4/gcc_bug_80431.m4 diff --git a/configure.ac b/configure.ac index 225baf6b5..75ce7d01d 100644 --- a/configure.ac +++ b/configure.ac @@ -68,6 +68,9 @@ case "$host_os" in esac +ENSURE_NO_GCC_BUG_80431 + + # Check for pubsetbuf. AC_MSG_CHECKING([for pubsetbuf]) AC_LANG_PUSH(C++) diff --git a/m4/gcc_bug_80431.m4 b/m4/gcc_bug_80431.m4 new file mode 100644 index 000000000..e42f01956 --- /dev/null +++ b/m4/gcc_bug_80431.m4 @@ -0,0 +1,64 @@ +# Ensure that this bug is not present in the C++ toolchain we are using. +# +# URL for bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431 +# +# The test program is from that issue, with only a slight modification +# to set an exit status instead of printing strings. +AC_DEFUN([ENSURE_NO_GCC_BUG_80431], +[ + AC_MSG_CHECKING([that GCC bug 80431 is fixed]) + AC_LANG_PUSH(C++) + AC_RUN_IFELSE( + [AC_LANG_PROGRAM( + [[ + #include + + static bool a = true; + static bool b = true; + + struct Options { }; + + struct Option + { + Option(Options * options) + { + a = false; + } + + ~Option() + { + b = false; + } + }; + + struct MyOptions : Options { }; + + struct MyOptions2 : virtual MyOptions + { + Option foo{this}; + }; + ]], + [[ + { + MyOptions2 opts; + } + return (a << 1) | b; + ]])], + [status_80431=0], + [status_80431=$?], + [ + # Assume we're bug-free when cross-compiling + ]) + AC_LANG_POP(C++) + AS_CASE([$status_80431], + [0],[ + AC_MSG_RESULT(yes) + ], + [2],[ + AC_MSG_RESULT(no) + AC_MSG_ERROR(Cannot build Nix with C++ compiler with this bug) + ], + [ + AC_MSG_RESULT(unexpected result $status_80431: not expected failure with bug, ignoring) + ]) +]) From b2cae33aef63644bf6e09dea253ed6e1af847fb8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Oct 2023 18:12:37 -0400 Subject: [PATCH 499/909] Remove bug-avoiding `StoreConfig *` casts for settings https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431 has been fixed, and per the previous commit we now check that is the case at build time. --- src/libstore/binary-cache-store.hh | 14 +++++++------- src/libstore/legacy-ssh-store.cc | 6 +++--- src/libstore/local-fs-store.hh | 12 ++++-------- src/libstore/local-store.hh | 4 ++-- src/libstore/remote-store.hh | 4 ++-- src/libstore/s3-binary-cache-store.cc | 18 +++++++++--------- src/libstore/ssh-store-config.hh | 8 ++++---- src/libstore/ssh-store.cc | 2 +- 8 files changed, 32 insertions(+), 36 deletions(-) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 49f271d24..218a888e3 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -17,28 +17,28 @@ struct BinaryCacheStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting compression{(StoreConfig*) this, "xz", "compression", + const Setting compression{this, "xz", "compression", "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."}; - const Setting writeNARListing{(StoreConfig*) this, false, "write-nar-listing", + const Setting writeNARListing{this, false, "write-nar-listing", "Whether to write a JSON file that lists the files in each NAR."}; - const Setting writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", + const Setting writeDebugInfo{this, false, "index-debug-info", R"( Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to fetch debug info on demand )"}; - const Setting secretKeyFile{(StoreConfig*) this, "", "secret-key", + const Setting secretKeyFile{this, "", "secret-key", "Path to the secret key used to sign the binary cache."}; - const Setting localNarCache{(StoreConfig*) this, "", "local-nar-cache", + const Setting localNarCache{this, "", "local-nar-cache", "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."}; - const Setting parallelCompression{(StoreConfig*) this, false, "parallel-compression", + const Setting parallelCompression{this, false, "parallel-compression", "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; - const Setting compressionLevel{(StoreConfig*) this, -1, "compression-level", + const Setting compressionLevel{this, -1, "compression-level", R"( The *preset level* to be used when compressing NARs. The meaning and accepted values depend on the compression method selected. diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index c712f7eb1..38fdf118f 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -17,10 +17,10 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig { using CommonSSHStoreConfig::CommonSSHStoreConfig; - const Setting remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", + const Setting remoteProgram{this, "nix-store", "remote-program", "Path to the `nix-store` executable on the remote machine."}; - const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", + const Setting maxConnections{this, 1, "max-connections", "Maximum number of concurrent SSH connections."}; const std::string name() override { return "SSH Store"; } @@ -38,7 +38,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor // Hack for getting remote build log output. // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in // the documentation - const Setting logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; + const Setting logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; struct Connection { diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 488109501..d6bda05d1 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -11,25 +11,21 @@ struct LocalFSStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes - // it to omit the call to the Setting constructor. Clang works fine - // either way. - - const OptionalPathSetting rootDir{(StoreConfig*) this, std::nullopt, + const OptionalPathSetting rootDir{this, std::nullopt, "root", "Directory prefixed to all other paths."}; - const PathSetting stateDir{(StoreConfig*) this, + const PathSetting stateDir{this, rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir, "state", "Directory where Nix will store state."}; - const PathSetting logDir{(StoreConfig*) this, + const PathSetting logDir{this, rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir, "log", "directory where Nix will store log files."}; - const PathSetting realStoreDir{(StoreConfig*) this, + const PathSetting realStoreDir{this, rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real", "Physical path of the Nix store."}; }; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index e97195f5b..fe26a0f27 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -40,12 +40,12 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; - Setting requireSigs{(StoreConfig*) this, + Setting requireSigs{this, settings.requireSigs, "require-sigs", "Whether store paths copied into this store should have a trusted signature."}; - Setting readOnly{(StoreConfig*) this, + Setting readOnly{this, false, "read-only", R"( diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index a1ae82a0f..f0985fdc1 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -22,10 +22,10 @@ struct RemoteStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", + const Setting maxConnections{this, 1, "max-connections", "Maximum number of concurrent connections to the Nix daemon."}; - const Setting maxConnectionAge{(StoreConfig*) this, + const Setting maxConnectionAge{this, std::numeric_limits::max(), "max-connection-age", "Maximum age of a connection before it is closed."}; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index d2fc6abaf..1a62d92d4 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -193,20 +193,20 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - const Setting profile{(StoreConfig*) this, "", "profile", + const Setting profile{this, "", "profile", R"( The name of the AWS configuration profile to use. By default Nix will use the `default` profile. )"}; - const Setting region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", + const Setting region{this, Aws::Region::US_EAST_1, "region", R"( The region of the S3 bucket. If your bucket is not in `us–east-1`, you should always explicitly specify the region parameter. )"}; - const Setting scheme{(StoreConfig*) this, "", "scheme", + const Setting scheme{this, "", "scheme", R"( The scheme used for S3 requests, `https` (default) or `http`. This option allows you to disable HTTPS for binary caches which don't @@ -218,7 +218,7 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig > information. )"}; - const Setting endpoint{(StoreConfig*) this, "", "endpoint", + const Setting endpoint{this, "", "endpoint", R"( The URL of the endpoint of an S3-compatible service such as MinIO. Do not specify this setting if you're using Amazon S3. @@ -229,13 +229,13 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig > addressing instead of virtual host based addressing. )"}; - const Setting narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", + const Setting narinfoCompression{this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; - const Setting lsCompression{(StoreConfig*) this, "", "ls-compression", + const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; - const Setting logCompression{(StoreConfig*) this, "", "log-compression", + const Setting logCompression{this, "", "log-compression", R"( Compression method for `log/*` files. It is recommended to use a compression method supported by most web browsers @@ -243,11 +243,11 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig )"}; const Setting multipartUpload{ - (StoreConfig*) this, false, "multipart-upload", + this, false, "multipart-upload", "Whether to use multi-part uploads."}; const Setting bufferSize{ - (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", + this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; const std::string name() override { return "S3 Binary Cache Store"; } diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/ssh-store-config.hh index c27a5d00f..bf55d20cf 100644 --- a/src/libstore/ssh-store-config.hh +++ b/src/libstore/ssh-store-config.hh @@ -9,16 +9,16 @@ struct CommonSSHStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting sshKey{(StoreConfig*) this, "", "ssh-key", + const Setting sshKey{this, "", "ssh-key", "Path to the SSH private key used to authenticate to the remote machine."}; - const Setting sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", + const Setting sshPublicHostKey{this, "", "base64-ssh-public-host-key", "The public host key of the remote machine."}; - const Setting compress{(StoreConfig*) this, false, "compress", + const Setting compress{this, false, "compress", "Whether to enable SSH compression."}; - const Setting remoteStore{(StoreConfig*) this, "", "remote-store", + const Setting remoteStore{this, "", "remote-store", R"( [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) to be used on the remote machine. The default is `auto` diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 9c6c42ef4..4a6aad449 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -16,7 +16,7 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig using RemoteStoreConfig::RemoteStoreConfig; using CommonSSHStoreConfig::CommonSSHStoreConfig; - const Setting remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", + const Setting remoteProgram{this, "nix-daemon", "remote-program", "Path to the `nix-daemon` executable on the remote machine."}; const std::string name() override { return "Experimental SSH Store"; } From 1093d6585ff6478e50a5845de64cfcf114e35a95 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 31 Oct 2023 20:39:39 -0400 Subject: [PATCH 500/909] Make `ParseSink` a bit better I wouldn't call it *good* yet, but this will do for now. - `RetrieveRegularNARSink` renamed to `RegularFileSink` and moved accordingly because it actually has nothing to do with NARs in particular. - its `fd` field is also marked private - `copyRecursive` introduced to dump a `SourceAccessor` into a `ParseSink`. - `NullParseSink` made so `ParseSink` no longer has sketchy default methods. This was done while updating #8918 to work with the new `SourceAccessor`. --- src/libstore/daemon.cc | 6 +-- src/libstore/export-import.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/store-api.cc | 8 ++-- src/libutil/archive.cc | 8 +--- src/libutil/archive.hh | 27 ----------- src/libutil/fs-sink.cc | 48 +++++++++++++++++++ src/libutil/fs-sink.hh | 86 ++++++++++++++++++++++++++++++----- 8 files changed, 132 insertions(+), 55 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 007ffc05a..105d92f25 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -454,13 +454,13 @@ static void performOp(TunnelLogger * logger, ref store, eagerly consume the entire stream it's given, past the length of the Nar. */ TeeSource savedNARSource(from, saved); - ParseSink sink; /* null sink; just parse the NAR */ + NullParseSink sink; /* just parse the NAR */ parseDump(sink, savedNARSource); } else { /* Incrementally parse the NAR file, stripping the metadata, and streaming the sole file we expect into `saved`. */ - RetrieveRegularNARSink savedRegular { saved }; + RegularFileSink savedRegular { saved }; parseDump(savedRegular, from); if (!savedRegular.regular) throw Error("regular file expected"); } @@ -899,7 +899,7 @@ static void performOp(TunnelLogger * logger, ref store, source = std::make_unique(from, to); else { TeeSource tee { from, saved }; - ParseSink ether; + NullParseSink ether; parseDump(ether, tee); source = std::make_unique(saved.s); } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 91b7e30db..52130f8f6 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -65,7 +65,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) /* Extract the NAR from the source. */ StringSink saved; TeeSource tee { source, saved }; - ParseSink ether; + NullParseSink ether; parseDump(ether, tee); uint32_t magic = readInt(source); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1c2f6023a..a5e9426f8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1200,7 +1200,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, bool narRead = false; Finally cleanup = [&]() { if (!narRead) { - ParseSink sink; + NullParseSink sink; parseDump(sink, source); } }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 0399120d1..e6a4cf9d9 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -410,7 +410,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, /* Note that fileSink and unusualHashTee must be mutually exclusive, since they both write to caHashSink. Note that that requisite is currently true because the former is only used in the flat case. */ - RetrieveRegularNARSink fileSink { caHashSink }; + RegularFileSink fileSink { caHashSink }; TeeSink unusualHashTee { narHashSink, caHashSink }; auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256 @@ -428,10 +428,10 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, information to narSink. */ TeeSource tapped { *fileSource, narSink }; - ParseSink blank; + NullParseSink blank; auto & parseSink = method == FileIngestionMethod::Flat - ? fileSink - : blank; + ? (ParseSink &) fileSink + : (ParseSink &) blank; /* The information that flows from tapped (besides being replicated in narSink), is now put in parseSink. */ diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 3b1a1e0ef..4ca84d357 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -5,12 +5,6 @@ #include // for strcasecmp -#include -#include -#include -#include -#include - #include "archive.hh" #include "util.hh" #include "config.hh" @@ -299,7 +293,7 @@ void copyNAR(Source & source, Sink & sink) // FIXME: if 'source' is the output of dumpPath() followed by EOF, // we should just forward all data directly without parsing. - ParseSink parseSink; /* null sink; just parse the NAR */ + NullParseSink parseSink; /* just parse the NAR */ TeeSource wrapper { source, sink }; diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 3530783c1..2cf8ee891 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -73,33 +73,6 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink, */ void dumpString(std::string_view s, Sink & sink); -/** - * If the NAR archive contains a single file at top-level, then save - * the contents of the file to `s`. Otherwise barf. - */ -struct RetrieveRegularNARSink : ParseSink -{ - bool regular = true; - Sink & sink; - - RetrieveRegularNARSink(Sink & sink) : sink(sink) { } - - void createDirectory(const Path & path) override - { - regular = false; - } - - void receiveContents(std::string_view data) override - { - sink(data); - } - - void createSymlink(const Path & path, const std::string & target) override - { - regular = false; - } -}; - void parseDump(ParseSink & sink, Source & source); void restorePath(const Path & path, Source & source); diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index a08a723a4..925e6f05d 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -5,6 +5,54 @@ namespace nix { +void copyRecursive( + SourceAccessor & accessor, const CanonPath & from, + ParseSink & sink, const Path & to) +{ + auto stat = accessor.lstat(from); + + switch (stat.type) { + case SourceAccessor::tSymlink: + { + sink.createSymlink(to, accessor.readLink(from)); + } + + case SourceAccessor::tRegular: + { + sink.createRegularFile(to); + if (stat.isExecutable) + sink.isExecutable(); + LambdaSink sink2 { + [&](auto d) { + sink.receiveContents(d); + } + }; + accessor.readFile(from, sink2, [&](uint64_t size) { + sink.preallocateContents(size); + }); + break; + } + + case SourceAccessor::tDirectory: + { + sink.createDirectory(to); + for (auto & [name, _] : accessor.readDirectory(from)) { + copyRecursive( + accessor, from + name, + sink, to + "/" + name); + break; + } + } + + case SourceAccessor::tMisc: + throw Error("file '%1%' has an unsupported type", from); + + default: + abort(); + } +} + + struct RestoreSinkSettings : Config { Setting preallocateContents{this, false, "preallocate-contents", diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index 6837e2fc4..c22edd390 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "serialise.hh" +#include "source-accessor.hh" namespace nix { @@ -11,32 +12,93 @@ namespace nix { */ struct ParseSink { - virtual void createDirectory(const Path & path) { }; + virtual void createDirectory(const Path & path) = 0; - virtual void createRegularFile(const Path & path) { }; - virtual void closeRegularFile() { }; - virtual void isExecutable() { }; + virtual void createRegularFile(const Path & path) = 0; + virtual void receiveContents(std::string_view data) = 0; + virtual void isExecutable() = 0; + virtual void closeRegularFile() = 0; + + virtual void createSymlink(const Path & path, const std::string & target) = 0; + + /** + * An optimization. By default, do nothing. + */ virtual void preallocateContents(uint64_t size) { }; - virtual void receiveContents(std::string_view data) { }; - - virtual void createSymlink(const Path & path, const std::string & target) { }; }; +/** + * Recusively copy file system objects from the source into the sink. + */ +void copyRecursive( + SourceAccessor & accessor, const CanonPath & sourcePath, + ParseSink & sink, const Path & destPath); + +/** + * Ignore everything and do nothing + */ +struct NullParseSink : ParseSink +{ + void createDirectory(const Path & path) override { } + void receiveContents(std::string_view data) override { } + void createSymlink(const Path & path, const std::string & target) override { } + void createRegularFile(const Path & path) override { } + void closeRegularFile() override { } + void isExecutable() override { } +}; + +/** + * Write files at the given path + */ struct RestoreSink : ParseSink { Path dstPath; - AutoCloseFD fd; - void createDirectory(const Path & path) override; void createRegularFile(const Path & path) override; - void closeRegularFile() override; - void isExecutable() override; - void preallocateContents(uint64_t size) override; void receiveContents(std::string_view data) override; + void isExecutable() override; + void closeRegularFile() override; void createSymlink(const Path & path, const std::string & target) override; + + void preallocateContents(uint64_t size) override; + +private: + AutoCloseFD fd; +}; + +/** + * Restore a single file at the top level, passing along + * `receiveContents` to the underlying `Sink`. For anything but a single + * file, set `regular = true` so the caller can fail accordingly. + */ +struct RegularFileSink : ParseSink +{ + bool regular = true; + Sink & sink; + + RegularFileSink(Sink & sink) : sink(sink) { } + + void createDirectory(const Path & path) override + { + regular = false; + } + + void receiveContents(std::string_view data) override + { + sink(data); + } + + void createSymlink(const Path & path, const std::string & target) override + { + regular = false; + } + + void createRegularFile(const Path & path) override { } + void closeRegularFile() override { } + void isExecutable() override { } }; } From bc4a1695ac71483831ac9ad591c872105794e88f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Nov 2023 11:44:16 +0100 Subject: [PATCH 501/909] doc/hacking: Fix clangd for tests --- doc/manual/src/contributing/hacking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 38c144fcc..fe08ceb94 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -210,7 +210,7 @@ See [supported compilation environments](#compilation-environments) and instruct To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running: ```console -make clean && bear -- make -j$NIX_BUILD_CORES install +make clean && bear -- make -j$NIX_BUILD_CORES default check install ``` Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). From b2ac6fc040223a58f9b923a89798f72b48e310e5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 14:36:40 +0100 Subject: [PATCH 502/909] Remove FSAccessor::Type::tMissing Instead stat() now returns std::nullopt to denote that the file doesn't exist. --- src/libstore/binary-cache-store.cc | 6 +-- src/libstore/fs-accessor.hh | 19 +++++---- src/libstore/local-fs-store.cc | 8 ++-- src/libstore/nar-accessor.cc | 68 +++++++++++++++--------------- src/libstore/remote-fs-accessor.cc | 2 +- src/libstore/remote-fs-accessor.hh | 2 +- src/nix/cat.cc | 11 +++-- src/nix/ls.cc | 26 ++++++------ src/nix/run.cc | 2 +- src/nix/why-depends.cc | 7 +-- 10 files changed, 77 insertions(+), 74 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 2a91233ec..06d89c478 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -208,7 +208,7 @@ ref BinaryCacheStore::addToStoreCommon( std::string buildIdDir = "/lib/debug/.build-id"; - if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) { + if (auto st = narAccessor->stat(buildIdDir); st && st->type == FSAccessor::tDirectory) { ThreadPool threadPool(25); @@ -234,14 +234,14 @@ ref BinaryCacheStore::addToStoreCommon( for (auto & s1 : narAccessor->readDirectory(buildIdDir)) { auto dir = buildIdDir + "/" + s1; - if (narAccessor->stat(dir).type != FSAccessor::tDirectory + if (auto st = narAccessor->stat(dir); !st || st->type != FSAccessor::tDirectory || !std::regex_match(s1, regex1)) continue; for (auto & s2 : narAccessor->readDirectory(dir)) { auto debugPath = dir + "/" + s2; - if (narAccessor->stat(debugPath).type != FSAccessor::tRegular + if (auto st = narAccessor->stat(debugPath); !st || st->type != FSAccessor::tRegular || !std::regex_match(s2, regex2)) continue; diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index 1df19e647..9bae0be74 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -3,6 +3,8 @@ #include "types.hh" +#include + namespace nix { /** @@ -12,28 +14,29 @@ namespace nix { class FSAccessor { public: - enum Type { tMissing, tRegular, tSymlink, tDirectory }; + enum Type { tRegular, tSymlink, tDirectory }; struct Stat { - Type type = tMissing; + Type type; /** - * regular files only + * For regular files only: the size of the file. */ uint64_t fileSize = 0; /** - * regular files only + * For regular files only: whether this is an executable. */ - bool isExecutable = false; // regular files only + bool isExecutable = false; /** - * regular files only + * For regular files only: the position of the contents of this + * file in the NAR. */ - uint64_t narOffset = 0; // regular files only + uint64_t narOffset = 0; }; virtual ~FSAccessor() { } - virtual Stat stat(const Path & path) = 0; + virtual std::optional stat(const Path & path) = 0; virtual StringSet readDirectory(const Path & path) = 0; diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index b224fc3e9..bb83a9cd4 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -27,25 +27,25 @@ struct LocalStoreAccessor : public FSAccessor return store->getRealStoreDir() + std::string(path, store->storeDir.size()); } - FSAccessor::Stat stat(const Path & path) override + std::optional stat(const Path & path) override { auto realPath = toRealPath(path); struct stat st; if (lstat(realPath.c_str(), &st)) { - if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; + if (errno == ENOENT || errno == ENOTDIR) return std::nullopt; throw SysError("getting status of '%1%'", path); } if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) throw Error("file '%1%' has unsupported type", path); - return { + return {{ S_ISREG(st.st_mode) ? Type::tRegular : S_ISLNK(st.st_mode) ? Type::tSymlink : Type::tDirectory, S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0, - S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}; + S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}}; } StringSet readDirectory(const Path & path) override diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index f0dfcb19b..9123bd59d 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -11,13 +11,7 @@ namespace nix { struct NarMember { - FSAccessor::Type type = FSAccessor::Type::tMissing; - - bool isExecutable = false; - - /* If this is a regular file, position of the contents of this - file in the NAR. */ - uint64_t start = 0, size = 0; + FSAccessor::Stat stat; std::string target; @@ -57,7 +51,7 @@ struct NarAccessor : public FSAccessor acc.root = std::move(member); parents.push(&acc.root); } else { - if (parents.top()->type != FSAccessor::Type::tDirectory) + if (parents.top()->stat.type != FSAccessor::Type::tDirectory) throw Error("NAR file missing parent directory of path '%s'", path); auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); parents.push(&result.first->second); @@ -79,14 +73,15 @@ struct NarAccessor : public FSAccessor void isExecutable() override { - parents.top()->isExecutable = true; + parents.top()->stat.isExecutable = true; } void preallocateContents(uint64_t size) override { assert(size <= std::numeric_limits::max()); - parents.top()->size = (uint64_t) size; - parents.top()->start = pos; + auto & st = parents.top()->stat; + st.fileSize = (uint64_t) size; + st.narOffset = pos; } void receiveContents(std::string_view data) override @@ -95,7 +90,9 @@ struct NarAccessor : public FSAccessor void createSymlink(const Path & path, const std::string & target) override { createMember(path, - NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); + NarMember{ + .stat = {.type = FSAccessor::Type::tSymlink}, + .target = target}); } size_t read(char * data, size_t len) override @@ -130,18 +127,20 @@ struct NarAccessor : public FSAccessor std::string type = v["type"]; if (type == "directory") { - member.type = FSAccessor::Type::tDirectory; + member.stat = {.type = FSAccessor::Type::tDirectory}; for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) { std::string name = i.key(); recurse(member.children[name], i.value()); } } else if (type == "regular") { - member.type = FSAccessor::Type::tRegular; - member.size = v["size"]; - member.isExecutable = v.value("executable", false); - member.start = v["narOffset"]; + member.stat = { + .type = FSAccessor::Type::tRegular, + .fileSize = v["size"], + .isExecutable = v.value("executable", false), + .narOffset = v["narOffset"] + }; } else if (type == "symlink") { - member.type = FSAccessor::Type::tSymlink; + member.stat = {.type = FSAccessor::Type::tSymlink}; member.target = v.value("target", ""); } else return; }; @@ -158,7 +157,7 @@ struct NarAccessor : public FSAccessor for (auto it = path.begin(); it != end; ) { // because it != end, the remaining component is non-empty so we need // a directory - if (current->type != FSAccessor::Type::tDirectory) return nullptr; + if (current->stat.type != FSAccessor::Type::tDirectory) return nullptr; // skip slash (canonPath above ensures that this is always a slash) assert(*it == '/'); @@ -183,19 +182,19 @@ struct NarAccessor : public FSAccessor return *result; } - Stat stat(const Path & path) override + std::optional stat(const Path & path) override { auto i = find(path); if (i == nullptr) - return {FSAccessor::Type::tMissing, 0, false}; - return {i->type, i->size, i->isExecutable, i->start}; + return std::nullopt; + return i->stat; } StringSet readDirectory(const Path & path) override { auto i = get(path); - if (i.type != FSAccessor::Type::tDirectory) + if (i.stat.type != FSAccessor::Type::tDirectory) throw Error("path '%1%' inside NAR file is not a directory", path); StringSet res; @@ -208,19 +207,19 @@ struct NarAccessor : public FSAccessor std::string readFile(const Path & path, bool requireValidPath = true) override { auto i = get(path); - if (i.type != FSAccessor::Type::tRegular) + if (i.stat.type != FSAccessor::Type::tRegular) throw Error("path '%1%' inside NAR file is not a regular file", path); - if (getNarBytes) return getNarBytes(i.start, i.size); + if (getNarBytes) return getNarBytes(i.stat.narOffset, i.stat.fileSize); assert(nar); - return std::string(*nar, i.start, i.size); + return std::string(*nar, i.stat.narOffset, i.stat.fileSize); } std::string readLink(const Path & path) override { auto i = get(path); - if (i.type != FSAccessor::Type::tSymlink) + if (i.stat.type != FSAccessor::Type::tSymlink) throw Error("path '%1%' inside NAR file is not a symlink", path); return i.target; } @@ -246,17 +245,19 @@ using nlohmann::json; json listNar(ref accessor, const Path & path, bool recurse) { auto st = accessor->stat(path); + if (!st) + throw Error("path '%s' does not exist in NAR", path); json obj = json::object(); - switch (st.type) { + switch (st->type) { case FSAccessor::Type::tRegular: obj["type"] = "regular"; - obj["size"] = st.fileSize; - if (st.isExecutable) + obj["size"] = st->fileSize; + if (st->isExecutable) obj["executable"] = true; - if (st.narOffset) - obj["narOffset"] = st.narOffset; + if (st->narOffset) + obj["narOffset"] = st->narOffset; break; case FSAccessor::Type::tDirectory: obj["type"] = "directory"; @@ -275,9 +276,6 @@ json listNar(ref accessor, const Path & path, bool recurse) obj["type"] = "symlink"; obj["target"] = accessor->readLink(path); break; - case FSAccessor::Type::tMissing: - default: - throw Error("path '%s' does not exist in NAR", path); } return obj; } diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index fcfb527f5..6c87ebeaa 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -101,7 +101,7 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath}; } -FSAccessor::Stat RemoteFSAccessor::stat(const Path & path) +std::optional RemoteFSAccessor::stat(const Path & path) { auto res = fetch(path); return res.first->stat(res.second); diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index e2673b6f6..5cf759aa0 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -28,7 +28,7 @@ public: RemoteFSAccessor(ref store, const /* FIXME: use std::optional */ Path & cacheDir = ""); - Stat stat(const Path & path) override; + std::optional stat(const Path & path) override; StringSet readDirectory(const Path & path) override; diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 60aa66ce0..b5fe2506f 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -11,13 +11,12 @@ struct MixCat : virtual Args void cat(ref accessor) { - auto st = accessor->stat(path); - if (st.type == FSAccessor::Type::tMissing) + if (auto st = accessor->stat(path)) { + if (st->type != FSAccessor::Type::tRegular) + throw Error("path '%1%' is not a regular file", path); + writeFull(STDOUT_FILENO, accessor->readFile(path)); + } else throw Error("path '%1%' does not exist", path); - if (st.type != FSAccessor::Type::tRegular) - throw Error("path '%1%' is not a regular file", path); - - writeFull(STDOUT_FILENO, accessor->readFile(path)); } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index c990a303c..8dc8a47b4 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -46,23 +46,25 @@ struct MixLs : virtual Args, MixJSON auto showFile = [&](const Path & curPath, const std::string & relPath) { if (verbose) { auto st = accessor->stat(curPath); + assert(st); std::string tp = - st.type == FSAccessor::Type::tRegular ? - (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : - st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : + st->type == FSAccessor::Type::tRegular ? + (st->isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : + st->type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : "dr-xr-xr-x"; - auto line = fmt("%s %20d %s", tp, st.fileSize, relPath); - if (st.type == FSAccessor::Type::tSymlink) + auto line = fmt("%s %20d %s", tp, st->fileSize, relPath); + if (st->type == FSAccessor::Type::tSymlink) line += " -> " + accessor->readLink(curPath); logger->cout(line); - if (recursive && st.type == FSAccessor::Type::tDirectory) - doPath(st, curPath, relPath, false); + if (recursive && st->type == FSAccessor::Type::tDirectory) + doPath(*st, curPath, relPath, false); } else { logger->cout(relPath); if (recursive) { auto st = accessor->stat(curPath); - if (st.type == FSAccessor::Type::tDirectory) - doPath(st, curPath, relPath, false); + assert(st); + if (st->type == FSAccessor::Type::tDirectory) + doPath(*st, curPath, relPath, false); } } }; @@ -79,10 +81,10 @@ struct MixLs : virtual Args, MixJSON }; auto st = accessor->stat(path); - if (st.type == FSAccessor::Type::tMissing) + if (!st) throw Error("path '%1%' does not exist", path); - doPath(st, path, - st.type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)), + doPath(*st, path, + st->type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)), showDirectory); } diff --git a/src/nix/run.cc b/src/nix/run.cc index 1baf299ab..f6c229adc 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -120,7 +120,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment unixPath.push_front(store->printStorePath(path) + "/bin"); auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages"; - if (accessor->stat(propPath).type == FSAccessor::tRegular) { + if (auto st = accessor->stat(propPath); st && st->type == FSAccessor::tRegular) { for (auto & p : tokenizeString(readFile(propPath))) todo.push(store->parseStorePath(p)); } diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 055cf6d0d..912ba72fb 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -214,6 +214,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions visitPath = [&](const Path & p) { auto st = accessor->stat(p); + assert(st); auto p2 = p == pathS ? "/" : std::string(p, pathS.size() + 1); @@ -221,13 +222,13 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; }; - if (st.type == FSAccessor::Type::tDirectory) { + if (st->type == FSAccessor::Type::tDirectory) { auto names = accessor->readDirectory(p); for (auto & name : names) visitPath(p + "/" + name); } - else if (st.type == FSAccessor::Type::tRegular) { + else if (st->type == FSAccessor::Type::tRegular) { auto contents = accessor->readFile(p); for (auto & hash : hashes) { @@ -245,7 +246,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions } } - else if (st.type == FSAccessor::Type::tSymlink) { + else if (st->type == FSAccessor::Type::tSymlink) { auto target = accessor->readLink(p); for (auto & hash : hashes) { From 8ffd1695ce31ff81b038fdc995dd8da03b180f03 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 14:43:20 +0100 Subject: [PATCH 503/909] Unify FSAccessor::Type and SourceAccessor::Type --- src/libstore/binary-cache-store.cc | 6 +++--- src/libstore/fs-accessor.hh | 3 ++- src/libstore/nar-accessor.cc | 30 ++++++++++++++++-------------- src/nix/run.cc | 2 +- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 06d89c478..f9abd8cbd 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -208,7 +208,7 @@ ref BinaryCacheStore::addToStoreCommon( std::string buildIdDir = "/lib/debug/.build-id"; - if (auto st = narAccessor->stat(buildIdDir); st && st->type == FSAccessor::tDirectory) { + if (auto st = narAccessor->stat(buildIdDir); st && st->type == SourceAccessor::tDirectory) { ThreadPool threadPool(25); @@ -234,14 +234,14 @@ ref BinaryCacheStore::addToStoreCommon( for (auto & s1 : narAccessor->readDirectory(buildIdDir)) { auto dir = buildIdDir + "/" + s1; - if (auto st = narAccessor->stat(dir); !st || st->type != FSAccessor::tDirectory + if (auto st = narAccessor->stat(dir); !st || st->type != SourceAccessor::tDirectory || !std::regex_match(s1, regex1)) continue; for (auto & s2 : narAccessor->readDirectory(dir)) { auto debugPath = dir + "/" + s2; - if (auto st = narAccessor->stat(debugPath); !st || st->type != FSAccessor::tRegular + if (auto st = narAccessor->stat(debugPath); !st || st->type != SourceAccessor::tRegular || !std::regex_match(s2, regex2)) continue; diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index 9bae0be74..1e951ec57 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -2,6 +2,7 @@ ///@file #include "types.hh" +#include "source-accessor.hh" #include @@ -14,7 +15,7 @@ namespace nix { class FSAccessor { public: - enum Type { tRegular, tSymlink, tDirectory }; + using Type = SourceAccessor::Type; struct Stat { diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 9123bd59d..43a78a362 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -51,7 +51,7 @@ struct NarAccessor : public FSAccessor acc.root = std::move(member); parents.push(&acc.root); } else { - if (parents.top()->stat.type != FSAccessor::Type::tDirectory) + if (parents.top()->stat.type != Type::tDirectory) throw Error("NAR file missing parent directory of path '%s'", path); auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); parents.push(&result.first->second); @@ -60,12 +60,12 @@ struct NarAccessor : public FSAccessor void createDirectory(const Path & path) override { - createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0}); + createMember(path, {Type::tDirectory, false, 0, 0}); } void createRegularFile(const Path & path) override { - createMember(path, {FSAccessor::Type::tRegular, false, 0, 0}); + createMember(path, {Type::tRegular, false, 0, 0}); } void closeRegularFile() override @@ -91,7 +91,7 @@ struct NarAccessor : public FSAccessor { createMember(path, NarMember{ - .stat = {.type = FSAccessor::Type::tSymlink}, + .stat = {.type = Type::tSymlink}, .target = target}); } @@ -127,20 +127,20 @@ struct NarAccessor : public FSAccessor std::string type = v["type"]; if (type == "directory") { - member.stat = {.type = FSAccessor::Type::tDirectory}; + member.stat = {.type = Type::tDirectory}; for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) { std::string name = i.key(); recurse(member.children[name], i.value()); } } else if (type == "regular") { member.stat = { - .type = FSAccessor::Type::tRegular, + .type = Type::tRegular, .fileSize = v["size"], .isExecutable = v.value("executable", false), .narOffset = v["narOffset"] }; } else if (type == "symlink") { - member.stat = {.type = FSAccessor::Type::tSymlink}; + member.stat = {.type = Type::tSymlink}; member.target = v.value("target", ""); } else return; }; @@ -157,7 +157,7 @@ struct NarAccessor : public FSAccessor for (auto it = path.begin(); it != end; ) { // because it != end, the remaining component is non-empty so we need // a directory - if (current->stat.type != FSAccessor::Type::tDirectory) return nullptr; + if (current->stat.type != Type::tDirectory) return nullptr; // skip slash (canonPath above ensures that this is always a slash) assert(*it == '/'); @@ -194,7 +194,7 @@ struct NarAccessor : public FSAccessor { auto i = get(path); - if (i.stat.type != FSAccessor::Type::tDirectory) + if (i.stat.type != Type::tDirectory) throw Error("path '%1%' inside NAR file is not a directory", path); StringSet res; @@ -207,7 +207,7 @@ struct NarAccessor : public FSAccessor std::string readFile(const Path & path, bool requireValidPath = true) override { auto i = get(path); - if (i.stat.type != FSAccessor::Type::tRegular) + if (i.stat.type != Type::tRegular) throw Error("path '%1%' inside NAR file is not a regular file", path); if (getNarBytes) return getNarBytes(i.stat.narOffset, i.stat.fileSize); @@ -219,7 +219,7 @@ struct NarAccessor : public FSAccessor std::string readLink(const Path & path) override { auto i = get(path); - if (i.stat.type != FSAccessor::Type::tSymlink) + if (i.stat.type != Type::tSymlink) throw Error("path '%1%' inside NAR file is not a symlink", path); return i.target; } @@ -251,7 +251,7 @@ json listNar(ref accessor, const Path & path, bool recurse) json obj = json::object(); switch (st->type) { - case FSAccessor::Type::tRegular: + case SourceAccessor::Type::tRegular: obj["type"] = "regular"; obj["size"] = st->fileSize; if (st->isExecutable) @@ -259,7 +259,7 @@ json listNar(ref accessor, const Path & path, bool recurse) if (st->narOffset) obj["narOffset"] = st->narOffset; break; - case FSAccessor::Type::tDirectory: + case SourceAccessor::Type::tDirectory: obj["type"] = "directory"; { obj["entries"] = json::object(); @@ -272,10 +272,12 @@ json listNar(ref accessor, const Path & path, bool recurse) } } break; - case FSAccessor::Type::tSymlink: + case SourceAccessor::Type::tSymlink: obj["type"] = "symlink"; obj["target"] = accessor->readLink(path); break; + case SourceAccessor::Type::tMisc: + assert(false); // cannot happen for NARs } return obj; } diff --git a/src/nix/run.cc b/src/nix/run.cc index f6c229adc..07806283c 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -120,7 +120,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment unixPath.push_front(store->printStorePath(path) + "/bin"); auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages"; - if (auto st = accessor->stat(propPath); st && st->type == FSAccessor::tRegular) { + if (auto st = accessor->stat(propPath); st && st->type == SourceAccessor::tRegular) { for (auto & p : tokenizeString(readFile(propPath))) todo.push(store->parseStorePath(p)); } From cdb27c1519cd802f477e8fa90beabe1bddc4bac7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 15:26:07 +0100 Subject: [PATCH 504/909] SourceAccessor: Change the main interface from lstat() to maybeLstat() --- src/libexpr/primops.cc | 6 ++---- src/libfetchers/fs-input-accessor.cc | 4 ++-- src/libfetchers/memory-input-accessor.cc | 4 ++-- src/libutil/posix-source-accessor.cc | 8 ++++++-- src/libutil/posix-source-accessor.hh | 2 +- src/libutil/source-accessor.cc | 10 +++++----- src/libutil/source-accessor.hh | 4 ++-- .../lang/eval-fail-bad-string-interpolation-2.err.exp | 2 +- tests/functional/lang/eval-fail-nonexist-path.err.exp | 2 +- 9 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 704e7007b..e3c775d90 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1548,10 +1548,8 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, try { auto checked = state.checkSourcePath(path); - auto exists = checked.pathExists(); - if (exists && mustBeDir) { - exists = checked.lstat().type == InputAccessor::tDirectory; - } + auto st = checked.maybeLstat(); + auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory); v.mkBool(exists); } catch (SysError & e) { /* Don't give away info from errors while canonicalising diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index 7638d2d82..81be64482 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -36,11 +36,11 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath); } - Stat lstat(const CanonPath & path) override + std::optional maybeLstat(const CanonPath & path) override { auto absPath = makeAbsPath(path); checkAllowed(absPath); - return PosixSourceAccessor::lstat(absPath); + return PosixSourceAccessor::maybeLstat(absPath); } DirEntries readDirectory(const CanonPath & path) override diff --git a/src/libfetchers/memory-input-accessor.cc b/src/libfetchers/memory-input-accessor.cc index 817d063ba..6468ece41 100644 --- a/src/libfetchers/memory-input-accessor.cc +++ b/src/libfetchers/memory-input-accessor.cc @@ -20,12 +20,12 @@ struct MemoryInputAccessorImpl : MemoryInputAccessor return i != files.end(); } - Stat lstat(const CanonPath & path) override + std::optional maybeLstat(const CanonPath & path) override { auto i = files.find(path); if (i != files.end()) return Stat { .type = tRegular, .isExecutable = false }; - throw Error("file '%s' does not exist", path); + return std::nullopt; } DirEntries readDirectory(const CanonPath & path) override diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 48b4fe626..8a8d64f3f 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -44,9 +44,13 @@ bool PosixSourceAccessor::pathExists(const CanonPath & path) return nix::pathExists(path.abs()); } -SourceAccessor::Stat PosixSourceAccessor::lstat(const CanonPath & path) +std::optional PosixSourceAccessor::maybeLstat(const CanonPath & path) { - auto st = nix::lstat(path.abs()); + struct stat st; + if (::lstat(path.c_str(), &st)) { + if (errno == ENOENT) return std::nullopt; + throw SysError("getting status of '%s'", showPath(path)); + } mtime = std::max(mtime, st.st_mtime); return Stat { .type = diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh index 608f96ee2..cf087d26e 100644 --- a/src/libutil/posix-source-accessor.hh +++ b/src/libutil/posix-source-accessor.hh @@ -22,7 +22,7 @@ struct PosixSourceAccessor : SourceAccessor bool pathExists(const CanonPath & path) override; - Stat lstat(const CanonPath & path) override; + std::optional maybeLstat(const CanonPath & path) override; DirEntries readDirectory(const CanonPath & path) override; diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index d168a9667..5b0c7dd34 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -42,12 +42,12 @@ Hash SourceAccessor::hashPath( return sink.finish().first; } -std::optional SourceAccessor::maybeLstat(const CanonPath & path) +SourceAccessor::Stat SourceAccessor::lstat(const CanonPath & path) { - // FIXME: merge these into one operation. - if (!pathExists(path)) - return {}; - return lstat(path); + if (auto st = maybeLstat(path)) + return *st; + else + throw Error("path '%s' does not exist", showPath(path)); } std::string SourceAccessor::showPath(const CanonPath & path) diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index fd823aa39..80bc02b48 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -61,9 +61,9 @@ struct SourceAccessor bool isExecutable = false; // regular files only }; - virtual Stat lstat(const CanonPath & path) = 0; + Stat lstat(const CanonPath & path); - std::optional maybeLstat(const CanonPath & path); + virtual std::optional maybeLstat(const CanonPath & path) = 0; typedef std::optional DirEntry; diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp index dea119ae8..a287067cd 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp @@ -1 +1 @@ -error: getting status of '/pwd/lang/fnord': No such file or directory +error: path '/pwd/lang/fnord' does not exist diff --git a/tests/functional/lang/eval-fail-nonexist-path.err.exp b/tests/functional/lang/eval-fail-nonexist-path.err.exp index dea119ae8..a287067cd 100644 --- a/tests/functional/lang/eval-fail-nonexist-path.err.exp +++ b/tests/functional/lang/eval-fail-nonexist-path.err.exp @@ -1 +1 @@ -error: getting status of '/pwd/lang/fnord': No such file or directory +error: path '/pwd/lang/fnord' does not exist From 53811238790f4bb5f9df74bb25047fe5b734a61f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 15:33:19 +0100 Subject: [PATCH 505/909] Unify DirEntries types --- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/fs-accessor.hh | 4 +++- src/libstore/local-fs-store.cc | 6 +++--- src/libstore/nar-accessor.cc | 8 ++++---- src/libstore/remote-fs-accessor.cc | 2 +- src/libstore/remote-fs-accessor.hh | 2 +- src/nix/ls.cc | 2 +- src/nix/why-depends.cc | 2 +- 8 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index f9abd8cbd..b61868413 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -231,14 +231,14 @@ ref BinaryCacheStore::addToStoreCommon( std::regex regex1("^[0-9a-f]{2}$"); std::regex regex2("^[0-9a-f]{38}\\.debug$"); - for (auto & s1 : narAccessor->readDirectory(buildIdDir)) { + for (auto & [s1, _type] : narAccessor->readDirectory(buildIdDir)) { auto dir = buildIdDir + "/" + s1; if (auto st = narAccessor->stat(dir); !st || st->type != SourceAccessor::tDirectory || !std::regex_match(s1, regex1)) continue; - for (auto & s2 : narAccessor->readDirectory(dir)) { + for (auto & [s2, _type] : narAccessor->readDirectory(dir)) { auto debugPath = dir + "/" + s2; if (auto st = narAccessor->stat(debugPath); !st || st->type != SourceAccessor::tRegular diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index 1e951ec57..f04a92206 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -39,7 +39,9 @@ public: virtual std::optional stat(const Path & path) = 0; - virtual StringSet readDirectory(const Path & path) = 0; + using DirEntries = SourceAccessor::DirEntries; + + virtual DirEntries readDirectory(const Path & path) = 0; /** * Read a file inside the store. diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index bb83a9cd4..65cbb9e35 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -48,15 +48,15 @@ struct LocalStoreAccessor : public FSAccessor S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}}; } - StringSet readDirectory(const Path & path) override + DirEntries readDirectory(const Path & path) override { auto realPath = toRealPath(path); auto entries = nix::readDirectory(realPath); - StringSet res; + DirEntries res; for (auto & entry : entries) - res.insert(entry.name); + res.insert_or_assign(entry.name, std::nullopt); return res; } diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 43a78a362..fe857a60e 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -190,16 +190,16 @@ struct NarAccessor : public FSAccessor return i->stat; } - StringSet readDirectory(const Path & path) override + DirEntries readDirectory(const Path & path) override { auto i = get(path); if (i.stat.type != Type::tDirectory) throw Error("path '%1%' inside NAR file is not a directory", path); - StringSet res; + DirEntries res; for (auto & child : i.children) - res.insert(child.first); + res.insert_or_assign(child.first, std::nullopt); return res; } @@ -264,7 +264,7 @@ json listNar(ref accessor, const Path & path, bool recurse) { obj["entries"] = json::object(); json &res2 = obj["entries"]; - for (auto & name : accessor->readDirectory(path)) { + for (auto & [name, type] : accessor->readDirectory(path)) { if (recurse) { res2[name] = listNar(accessor, path + "/" + name, true); } else diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 6c87ebeaa..21419700c 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -107,7 +107,7 @@ std::optional RemoteFSAccessor::stat(const Path & path) return res.first->stat(res.second); } -StringSet RemoteFSAccessor::readDirectory(const Path & path) +SourceAccessor::DirEntries RemoteFSAccessor::readDirectory(const Path & path) { auto res = fetch(path); return res.first->readDirectory(res.second); diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 5cf759aa0..8de3b7bcd 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -30,7 +30,7 @@ public: std::optional stat(const Path & path) override; - StringSet readDirectory(const Path & path) override; + DirEntries readDirectory(const Path & path) override; std::string readFile(const Path & path, bool requireValidPath = true) override; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 8dc8a47b4..0ca08cea8 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -74,7 +74,7 @@ struct MixLs : virtual Args, MixJSON { if (st.type == FSAccessor::Type::tDirectory && !showDirectory) { auto names = accessor->readDirectory(curPath); - for (auto & name : names) + for (auto & [name, type] : names) showFile(curPath + "/" + name, relPath + "/" + name); } else showFile(curPath, relPath); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 912ba72fb..04c1a0c1c 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -224,7 +224,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions if (st->type == FSAccessor::Type::tDirectory) { auto names = accessor->readDirectory(p); - for (auto & name : names) + for (auto & [name, type] : names) visitPath(p + "/" + name); } From 50aae0a14c5bbbde5785ead8f46b28333e6248ae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 15:39:40 +0100 Subject: [PATCH 506/909] FSAccessor: Make the fileSize and narOffset fields optional The narOffset field only applies to NAR accessors. The fileSize field may be too expensive to compute for certain accessors (e.g. libgit). --- src/libstore/fs-accessor.hh | 4 ++-- src/libstore/nar-accessor.cc | 11 ++++++----- src/nix/ls.cc | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index f04a92206..f6c002a2d 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -23,7 +23,7 @@ public: /** * For regular files only: the size of the file. */ - uint64_t fileSize = 0; + std::optional fileSize; /** * For regular files only: whether this is an executable. */ @@ -32,7 +32,7 @@ public: * For regular files only: the position of the contents of this * file in the NAR. */ - uint64_t narOffset = 0; + std::optional narOffset; }; virtual ~FSAccessor() { } diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index fe857a60e..f1be5606e 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -210,10 +210,10 @@ struct NarAccessor : public FSAccessor if (i.stat.type != Type::tRegular) throw Error("path '%1%' inside NAR file is not a regular file", path); - if (getNarBytes) return getNarBytes(i.stat.narOffset, i.stat.fileSize); + if (getNarBytes) return getNarBytes(*i.stat.narOffset, *i.stat.fileSize); assert(nar); - return std::string(*nar, i.stat.narOffset, i.stat.fileSize); + return std::string(*nar, *i.stat.narOffset, *i.stat.fileSize); } std::string readLink(const Path & path) override @@ -253,11 +253,12 @@ json listNar(ref accessor, const Path & path, bool recurse) switch (st->type) { case SourceAccessor::Type::tRegular: obj["type"] = "regular"; - obj["size"] = st->fileSize; + if (st->fileSize) + obj["size"] = *st->fileSize; if (st->isExecutable) obj["executable"] = true; - if (st->narOffset) - obj["narOffset"] = st->narOffset; + if (st->narOffset && *st->narOffset) + obj["narOffset"] = *st->narOffset; break; case SourceAccessor::Type::tDirectory: obj["type"] = "directory"; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 0ca08cea8..da978f379 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -52,7 +52,7 @@ struct MixLs : virtual Args, MixJSON (st->isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : st->type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : "dr-xr-xr-x"; - auto line = fmt("%s %20d %s", tp, st->fileSize, relPath); + auto line = fmt("%s %20d %s", tp, st->fileSize.value_or(0), relPath); if (st->type == FSAccessor::Type::tSymlink) line += " -> " + accessor->readLink(curPath); logger->cout(line); From 581693bdea3981eb0b106c904c7a1fed7f7582ae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 16:33:22 +0100 Subject: [PATCH 507/909] fmt(): Handle std::string_view --- src/libutil/fmt.hh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 727255b45..ac72e47fb 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -44,6 +44,11 @@ inline std::string fmt(const std::string & s) return s; } +inline std::string fmt(std::string_view s) +{ + return std::string(s); +} + inline std::string fmt(const char * s) { return s; From 1a902f5fa7d4f268d0fec3e44a48ecc2445b3b6b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 17:09:28 +0100 Subject: [PATCH 508/909] Merge FSAccessor into SourceAccessor --- src/libstore/binary-cache-store.cc | 24 +++++------ src/libstore/binary-cache-store.hh | 2 +- src/libstore/derivations.cc | 1 - src/libstore/dummy-store.cc | 2 +- src/libstore/fs-accessor.hh | 58 --------------------------- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-fs-store.cc | 34 +++++++++------- src/libstore/local-fs-store.hh | 2 +- src/libstore/nar-accessor.cc | 64 ++++++++++++------------------ src/libstore/nar-accessor.hh | 11 ++--- src/libstore/remote-fs-accessor.cc | 28 ++++++------- src/libstore/remote-fs-accessor.hh | 21 +++++----- src/libstore/remote-store.cc | 2 +- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 8 ++-- src/libstore/store-api.hh | 4 +- src/libstore/uds-remote-store.hh | 4 +- src/libutil/source-accessor.cc | 5 +++ src/libutil/source-accessor.hh | 22 ++++++++-- src/nix/bundle.cc | 1 - src/nix/cat.cc | 13 +++--- src/nix/ls.cc | 54 +++++++++++-------------- src/nix/run.cc | 8 ++-- src/nix/why-depends.cc | 22 +++++----- tests/functional/nar-access.sh | 8 +++- 25 files changed, 178 insertions(+), 224 deletions(-) delete mode 100644 src/libstore/fs-accessor.hh diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index b61868413..dd9e2f3af 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -2,7 +2,7 @@ #include "binary-cache-store.hh" #include "compression.hh" #include "derivations.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "globals.hh" #include "nar-info.hh" #include "sync.hh" @@ -143,7 +143,7 @@ ref BinaryCacheStore::addToStoreCommon( write the compressed NAR to disk), into a HashSink (to get the NAR hash), and into a NarAccessor (to get the NAR listing). */ HashSink fileHashSink { htSHA256 }; - std::shared_ptr narAccessor; + std::shared_ptr narAccessor; HashSink narHashSink { htSHA256 }; { FdSink fileSink(fdTemp.get()); @@ -195,7 +195,7 @@ ref BinaryCacheStore::addToStoreCommon( if (writeNARListing) { nlohmann::json j = { {"version", 1}, - {"root", listNar(ref(narAccessor), "", true)}, + {"root", listNar(ref(narAccessor), CanonPath::root, true)}, }; upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json"); @@ -206,9 +206,9 @@ ref BinaryCacheStore::addToStoreCommon( specify the NAR file and member containing the debug info. */ if (writeDebugInfo) { - std::string buildIdDir = "/lib/debug/.build-id"; + CanonPath buildIdDir("lib/debug/.build-id"); - if (auto st = narAccessor->stat(buildIdDir); st && st->type == SourceAccessor::tDirectory) { + if (auto st = narAccessor->maybeLstat(buildIdDir); st && st->type == SourceAccessor::tDirectory) { ThreadPool threadPool(25); @@ -232,16 +232,16 @@ ref BinaryCacheStore::addToStoreCommon( std::regex regex2("^[0-9a-f]{38}\\.debug$"); for (auto & [s1, _type] : narAccessor->readDirectory(buildIdDir)) { - auto dir = buildIdDir + "/" + s1; + auto dir = buildIdDir + s1; - if (auto st = narAccessor->stat(dir); !st || st->type != SourceAccessor::tDirectory + if (narAccessor->lstat(dir).type != SourceAccessor::tDirectory || !std::regex_match(s1, regex1)) continue; for (auto & [s2, _type] : narAccessor->readDirectory(dir)) { - auto debugPath = dir + "/" + s2; + auto debugPath = dir + s2; - if (auto st = narAccessor->stat(debugPath); !st || st->type != SourceAccessor::tRegular + if ( narAccessor->lstat(debugPath).type != SourceAccessor::tRegular || !std::regex_match(s2, regex2)) continue; @@ -250,7 +250,7 @@ ref BinaryCacheStore::addToStoreCommon( std::string key = "debuginfo/" + buildId; std::string target = "../" + narInfo->url; - threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target)); + threadPool.enqueue(std::bind(doFile, std::string(debugPath.rel()), key, target)); } } @@ -503,9 +503,9 @@ void BinaryCacheStore::registerDrvOutput(const Realisation& info) { upsertFile(filePath, info.toJSON().dump(), "application/json"); } -ref BinaryCacheStore::getFSAccessor() +ref BinaryCacheStore::getFSAccessor(bool requireValidPath) { - return make_ref(ref(shared_from_this()), localNarCache); + return make_ref(ref(shared_from_this()), requireValidPath, localNarCache); } void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 218a888e3..cea2a571f 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -148,7 +148,7 @@ public: void narFromPath(const StorePath & path, Sink & sink) override; - ref getFSAccessor() override; + ref getFSAccessor(bool requireValidPath) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index a5ceb29dc..efdad18e1 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -6,7 +6,6 @@ #include "split.hh" #include "common-protocol.hh" #include "common-protocol-impl.hh" -#include "fs-accessor.hh" #include #include diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 74d6ed3b5..821cda399 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -72,7 +72,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store Callback> callback) noexcept override { callback(nullptr); } - virtual ref getFSAccessor() override + virtual ref getFSAccessor(bool requireValidPath) override { unsupported("getFSAccessor"); } }; diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh deleted file mode 100644 index f6c002a2d..000000000 --- a/src/libstore/fs-accessor.hh +++ /dev/null @@ -1,58 +0,0 @@ -#pragma once -///@file - -#include "types.hh" -#include "source-accessor.hh" - -#include - -namespace nix { - -/** - * An abstract class for accessing a filesystem-like structure, such - * as a (possibly remote) Nix store or the contents of a NAR file. - */ -class FSAccessor -{ -public: - using Type = SourceAccessor::Type; - - struct Stat - { - Type type; - /** - * For regular files only: the size of the file. - */ - std::optional fileSize; - /** - * For regular files only: whether this is an executable. - */ - bool isExecutable = false; - /** - * For regular files only: the position of the contents of this - * file in the NAR. - */ - std::optional narOffset; - }; - - virtual ~FSAccessor() { } - - virtual std::optional stat(const Path & path) = 0; - - using DirEntries = SourceAccessor::DirEntries; - - virtual DirEntries readDirectory(const Path & path) = 0; - - /** - * Read a file inside the store. - * - * If `requireValidPath` is set to `true` (the default), the path must be - * inside a valid store path, otherwise it just needs to be physically - * present (but not necessarily properly registered) - */ - virtual std::string readFile(const Path & path, bool requireValidPath = true) = 0; - - virtual std::string readLink(const Path & path) = 0; -}; - -} diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 38fdf118f..731457354 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -363,7 +363,7 @@ public: void ensurePath(const StorePath & path) override { unsupported("ensurePath"); } - virtual ref getFSAccessor() override + virtual ref getFSAccessor(bool requireValidPath) override { unsupported("getFSAccessor"); } /** diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 65cbb9e35..63497acbd 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,5 +1,5 @@ #include "archive.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" @@ -13,26 +13,31 @@ LocalFSStore::LocalFSStore(const Params & params) { } -struct LocalStoreAccessor : public FSAccessor +struct LocalStoreAccessor : public SourceAccessor { ref store; + bool requireValidPath; - LocalStoreAccessor(ref store) : store(store) { } + LocalStoreAccessor(ref store, bool requireValidPath) + : store(store) + , requireValidPath(requireValidPath) + { } - Path toRealPath(const Path & path, bool requireValidPath = true) + Path toRealPath(const CanonPath & path) { - auto storePath = store->toStorePath(path).first; + auto storePath = store->toStorePath(path.abs()).first; if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); - return store->getRealStoreDir() + std::string(path, store->storeDir.size()); + return store->getRealStoreDir() + path.abs().substr(store->storeDir.size()); } - std::optional stat(const Path & path) override + std::optional maybeLstat(const CanonPath & path) override { auto realPath = toRealPath(path); + // FIXME: use PosixSourceAccessor. struct stat st; - if (lstat(realPath.c_str(), &st)) { + if (::lstat(realPath.c_str(), &st)) { if (errno == ENOENT || errno == ENOTDIR) return std::nullopt; throw SysError("getting status of '%1%'", path); } @@ -48,7 +53,7 @@ struct LocalStoreAccessor : public FSAccessor S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}}; } - DirEntries readDirectory(const Path & path) override + DirEntries readDirectory(const CanonPath & path) override { auto realPath = toRealPath(path); @@ -61,21 +66,22 @@ struct LocalStoreAccessor : public FSAccessor return res; } - std::string readFile(const Path & path, bool requireValidPath = true) override + std::string readFile(const CanonPath & path) override { - return nix::readFile(toRealPath(path, requireValidPath)); + return nix::readFile(toRealPath(path)); } - std::string readLink(const Path & path) override + std::string readLink(const CanonPath & path) override { return nix::readLink(toRealPath(path)); } }; -ref LocalFSStore::getFSAccessor() +ref LocalFSStore::getFSAccessor(bool requireValidPath) { return make_ref(ref( - std::dynamic_pointer_cast(shared_from_this()))); + std::dynamic_pointer_cast(shared_from_this())), + requireValidPath); } void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index d6bda05d1..bf855b67e 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -43,7 +43,7 @@ public: LocalFSStore(const Params & params); void narFromPath(const StorePath & path, Sink & sink) override; - ref getFSAccessor() override; + ref getFSAccessor(bool requireValidPath) override; /** * Creates symlink from the `gcRoot` to the `storePath` and diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index f1be5606e..02993680f 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -11,7 +11,7 @@ namespace nix { struct NarMember { - FSAccessor::Stat stat; + SourceAccessor::Stat stat; std::string target; @@ -19,7 +19,7 @@ struct NarMember std::map children; }; -struct NarAccessor : public FSAccessor +struct NarAccessor : public SourceAccessor { std::optional nar; @@ -149,48 +149,36 @@ struct NarAccessor : public FSAccessor recurse(root, v); } - NarMember * find(const Path & path) + NarMember * find(const CanonPath & path) { - Path canon = path == "" ? "" : canonPath(path); NarMember * current = &root; - auto end = path.end(); - for (auto it = path.begin(); it != end; ) { - // because it != end, the remaining component is non-empty so we need - // a directory + + for (auto & i : path) { if (current->stat.type != Type::tDirectory) return nullptr; - - // skip slash (canonPath above ensures that this is always a slash) - assert(*it == '/'); - it += 1; - - // lookup current component - auto next = std::find(it, end, '/'); - auto child = current->children.find(std::string(it, next)); + auto child = current->children.find(std::string(i)); if (child == current->children.end()) return nullptr; current = &child->second; - - it = next; } return current; } - NarMember & get(const Path & path) { + NarMember & get(const CanonPath & path) { auto result = find(path); - if (result == nullptr) + if (!result) throw Error("NAR file does not contain path '%1%'", path); return *result; } - std::optional stat(const Path & path) override + std::optional maybeLstat(const CanonPath & path) override { auto i = find(path); - if (i == nullptr) + if (!i) return std::nullopt; return i->stat; } - DirEntries readDirectory(const Path & path) override + DirEntries readDirectory(const CanonPath & path) override { auto i = get(path); @@ -204,7 +192,7 @@ struct NarAccessor : public FSAccessor return res; } - std::string readFile(const Path & path, bool requireValidPath = true) override + std::string readFile(const CanonPath & path) override { auto i = get(path); if (i.stat.type != Type::tRegular) @@ -216,7 +204,7 @@ struct NarAccessor : public FSAccessor return std::string(*nar, *i.stat.narOffset, *i.stat.fileSize); } - std::string readLink(const Path & path) override + std::string readLink(const CanonPath & path) override { auto i = get(path); if (i.stat.type != Type::tSymlink) @@ -225,40 +213,38 @@ struct NarAccessor : public FSAccessor } }; -ref makeNarAccessor(std::string && nar) +ref makeNarAccessor(std::string && nar) { return make_ref(std::move(nar)); } -ref makeNarAccessor(Source & source) +ref makeNarAccessor(Source & source) { return make_ref(source); } -ref makeLazyNarAccessor(const std::string & listing, +ref makeLazyNarAccessor(const std::string & listing, GetNarBytes getNarBytes) { return make_ref(listing, getNarBytes); } using nlohmann::json; -json listNar(ref accessor, const Path & path, bool recurse) +json listNar(ref accessor, const CanonPath & path, bool recurse) { - auto st = accessor->stat(path); - if (!st) - throw Error("path '%s' does not exist in NAR", path); + auto st = accessor->lstat(path); json obj = json::object(); - switch (st->type) { + switch (st.type) { case SourceAccessor::Type::tRegular: obj["type"] = "regular"; - if (st->fileSize) - obj["size"] = *st->fileSize; - if (st->isExecutable) + if (st.fileSize) + obj["size"] = *st.fileSize; + if (st.isExecutable) obj["executable"] = true; - if (st->narOffset && *st->narOffset) - obj["narOffset"] = *st->narOffset; + if (st.narOffset && *st.narOffset) + obj["narOffset"] = *st.narOffset; break; case SourceAccessor::Type::tDirectory: obj["type"] = "directory"; @@ -267,7 +253,7 @@ json listNar(ref accessor, const Path & path, bool recurse) json &res2 = obj["entries"]; for (auto & [name, type] : accessor->readDirectory(path)) { if (recurse) { - res2[name] = listNar(accessor, path + "/" + name, true); + res2[name] = listNar(accessor, path + name, true); } else res2[name] = json::object(); } diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 5e19bd3c7..433774524 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -1,10 +1,11 @@ #pragma once ///@file +#include "source-accessor.hh" + #include #include -#include "fs-accessor.hh" namespace nix { @@ -14,9 +15,9 @@ struct Source; * Return an object that provides access to the contents of a NAR * file. */ -ref makeNarAccessor(std::string && nar); +ref makeNarAccessor(std::string && nar); -ref makeNarAccessor(Source & source); +ref makeNarAccessor(Source & source); /** * Create a NAR accessor from a NAR listing (in the format produced by @@ -26,7 +27,7 @@ ref makeNarAccessor(Source & source); */ typedef std::function GetNarBytes; -ref makeLazyNarAccessor( +ref makeLazyNarAccessor( const std::string & listing, GetNarBytes getNarBytes); @@ -34,6 +35,6 @@ ref makeLazyNarAccessor( * Write a JSON representation of the contents of a NAR (except file * contents). */ -nlohmann::json listNar(ref accessor, const Path & path, bool recurse); +nlohmann::json listNar(ref accessor, const CanonPath & path, bool recurse); } diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 21419700c..03e57a565 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -8,8 +8,9 @@ namespace nix { -RemoteFSAccessor::RemoteFSAccessor(ref store, const Path & cacheDir) +RemoteFSAccessor::RemoteFSAccessor(ref store, bool requireValidPath, const Path & cacheDir) : store(store) + , requireValidPath(requireValidPath) , cacheDir(cacheDir) { if (cacheDir != "") @@ -22,7 +23,7 @@ Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::strin return fmt("%s/%s.%s", cacheDir, hashPart, ext); } -ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar) +ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar) { if (cacheDir != "") { try { @@ -38,7 +39,7 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::str if (cacheDir != "") { try { - nlohmann::json j = listNar(narAccessor, "", true); + nlohmann::json j = listNar(narAccessor, CanonPath::root, true); writeFile(makeCacheFile(hashPart, "ls"), j.dump()); } catch (...) { ignoreException(); @@ -48,11 +49,10 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::str return narAccessor; } -std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath) +std::pair, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path) { - auto path = canonPath(path_); - - auto [storePath, restPath] = store->toStorePath(path); + auto [storePath, restPath_] = store->toStorePath(path.abs()); + auto restPath = CanonPath(restPath_); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); @@ -63,7 +63,7 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo std::string listing; Path cacheFile; - if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { + if (cacheDir != "" && nix::pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { try { listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls")); @@ -101,25 +101,25 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath}; } -std::optional RemoteFSAccessor::stat(const Path & path) +std::optional RemoteFSAccessor::maybeLstat(const CanonPath & path) { auto res = fetch(path); - return res.first->stat(res.second); + return res.first->maybeLstat(res.second); } -SourceAccessor::DirEntries RemoteFSAccessor::readDirectory(const Path & path) +SourceAccessor::DirEntries RemoteFSAccessor::readDirectory(const CanonPath & path) { auto res = fetch(path); return res.first->readDirectory(res.second); } -std::string RemoteFSAccessor::readFile(const Path & path, bool requireValidPath) +std::string RemoteFSAccessor::readFile(const CanonPath & path) { - auto res = fetch(path, requireValidPath); + auto res = fetch(path); return res.first->readFile(res.second); } -std::string RemoteFSAccessor::readLink(const Path & path) +std::string RemoteFSAccessor::readLink(const CanonPath & path) { auto res = fetch(path); return res.first->readLink(res.second); diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 8de3b7bcd..d09762a53 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -1,40 +1,43 @@ #pragma once ///@file -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "ref.hh" #include "store-api.hh" namespace nix { -class RemoteFSAccessor : public FSAccessor +class RemoteFSAccessor : public SourceAccessor { ref store; - std::map> nars; + std::map> nars; + + bool requireValidPath; Path cacheDir; - std::pair, Path> fetch(const Path & path_, bool requireValidPath = true); + std::pair, CanonPath> fetch(const CanonPath & path); friend class BinaryCacheStore; Path makeCacheFile(std::string_view hashPart, const std::string & ext); - ref addToCache(std::string_view hashPart, std::string && nar); + ref addToCache(std::string_view hashPart, std::string && nar); public: RemoteFSAccessor(ref store, + bool requireValidPath = true, const /* FIXME: use std::optional */ Path & cacheDir = ""); - std::optional stat(const Path & path) override; + std::optional maybeLstat(const CanonPath & path) override; - DirEntries readDirectory(const Path & path) override; + DirEntries readDirectory(const CanonPath & path) override; - std::string readFile(const Path & path, bool requireValidPath = true) override; + std::string readFile(const CanonPath & path) override; - std::string readLink(const Path & path) override; + std::string readLink(const CanonPath & path) override; }; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 7bdc25433..f16949f42 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -970,7 +970,7 @@ void RemoteStore::narFromPath(const StorePath & path, Sink & sink) copyNAR(conn->from, sink); } -ref RemoteStore::getFSAccessor() +ref RemoteStore::getFSAccessor(bool requireValidPath) { return make_ref(ref(shared_from_this())); } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index f0985fdc1..1cc11af86 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -185,7 +185,7 @@ protected: friend struct ConnectionHandle; - virtual ref getFSAccessor() override; + virtual ref getFSAccessor(bool requireValidPath) override; virtual void narFromPath(const StorePath & path, Sink & sink) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 0399120d1..665b5fed7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,5 +1,5 @@ #include "crypto.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "globals.hh" #include "derivations.hh" #include "store-api.hh" @@ -1338,12 +1338,12 @@ Derivation Store::derivationFromPath(const StorePath & drvPath) return readDerivation(drvPath); } -Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool requireValidPath) +static Derivation readDerivationCommon(Store & store, const StorePath & drvPath, bool requireValidPath) { - auto accessor = store.getFSAccessor(); + auto accessor = store.getFSAccessor(requireValidPath); try { return parseDerivation(store, - accessor->readFile(store.printStorePath(drvPath), requireValidPath), + accessor->readFile(CanonPath(store.printStorePath(drvPath))), Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg()); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e123fccc5..6aa317e3d 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -70,7 +70,7 @@ MakeError(InvalidStoreURI, Error); struct BasicDerivation; struct Derivation; -class FSAccessor; +struct SourceAccessor; class NarInfoDiskCache; class Store; @@ -703,7 +703,7 @@ public: /** * @return An object to access files in the Nix store. */ - virtual ref getFSAccessor() = 0; + virtual ref getFSAccessor(bool requireValidPath = true) = 0; /** * Repair the contents of the given path by redownloading it using diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index cdb28a001..a5ac9080a 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -35,8 +35,8 @@ public: static std::set uriSchemes() { return {"unix"}; } - ref getFSAccessor() override - { return LocalFSStore::getFSAccessor(); } + ref getFSAccessor(bool requireValidPath) override + { return LocalFSStore::getFSAccessor(requireValidPath); } void narFromPath(const StorePath & path, Sink & sink) override { LocalFSStore::narFromPath(path, sink); } diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index 5b0c7dd34..e2114e18f 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -10,6 +10,11 @@ SourceAccessor::SourceAccessor() { } +bool SourceAccessor::pathExists(const CanonPath & path) +{ + return maybeLstat(path).has_value(); +} + std::string SourceAccessor::readFile(const CanonPath & path) { StringSink sink; diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index 80bc02b48..1a4e80361 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -40,7 +40,7 @@ struct SourceAccessor Sink & sink, std::function sizeCallback = [](uint64_t size){}); - virtual bool pathExists(const CanonPath & path) = 0; + virtual bool pathExists(const CanonPath & path); enum Type { tRegular, tSymlink, tDirectory, @@ -57,8 +57,24 @@ struct SourceAccessor struct Stat { Type type = tMisc; - //uint64_t fileSize = 0; // regular files only - bool isExecutable = false; // regular files only + + /** + * For regular files only: the size of the file. Not all + * accessors return this since it may be too expensive to + * compute. + */ + std::optional fileSize; + + /** + * For regular files only: whether this is an executable. + */ + bool isExecutable = false; + + /** + * For regular files only: the position of the contents of this + * file in the NAR. Only returned by NAR accessors. + */ + std::optional narOffset; }; Stat lstat(const CanonPath & path); diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 504e35c81..54cc6a17f 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -4,7 +4,6 @@ #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "fs-accessor.hh" #include "eval-inline.hh" using namespace nix; diff --git a/src/nix/cat.cc b/src/nix/cat.cc index b5fe2506f..6e5a736f2 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -1,6 +1,5 @@ #include "command.hh" #include "store-api.hh" -#include "fs-accessor.hh" #include "nar-accessor.hh" using namespace nix; @@ -9,14 +8,12 @@ struct MixCat : virtual Args { std::string path; - void cat(ref accessor) + void cat(ref accessor) { - if (auto st = accessor->stat(path)) { - if (st->type != FSAccessor::Type::tRegular) - throw Error("path '%1%' is not a regular file", path); - writeFull(STDOUT_FILENO, accessor->readFile(path)); - } else - throw Error("path '%1%' does not exist", path); + auto st = accessor->lstat(CanonPath(path)); + if (st.type != SourceAccessor::Type::tRegular) + throw Error("path '%1%' is not a regular file", path); + writeFull(STDOUT_FILENO, accessor->readFile(CanonPath(path))); } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index da978f379..231456c9c 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -1,6 +1,5 @@ #include "command.hh" #include "store-api.hh" -#include "fs-accessor.hh" #include "nar-accessor.hh" #include "common-args.hh" #include @@ -39,63 +38,58 @@ struct MixLs : virtual Args, MixJSON }); } - void listText(ref accessor) + void listText(ref accessor) { - std::function doPath; + std::function doPath; - auto showFile = [&](const Path & curPath, const std::string & relPath) { + auto showFile = [&](const CanonPath & curPath, std::string_view relPath) { if (verbose) { - auto st = accessor->stat(curPath); - assert(st); + auto st = accessor->lstat(curPath); std::string tp = - st->type == FSAccessor::Type::tRegular ? - (st->isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : - st->type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : + st.type == SourceAccessor::Type::tRegular ? + (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : + st.type == SourceAccessor::Type::tSymlink ? "lrwxrwxrwx" : "dr-xr-xr-x"; - auto line = fmt("%s %20d %s", tp, st->fileSize.value_or(0), relPath); - if (st->type == FSAccessor::Type::tSymlink) + auto line = fmt("%s %20d %s", tp, st.fileSize.value_or(0), relPath); + if (st.type == SourceAccessor::Type::tSymlink) line += " -> " + accessor->readLink(curPath); logger->cout(line); - if (recursive && st->type == FSAccessor::Type::tDirectory) - doPath(*st, curPath, relPath, false); + if (recursive && st.type == SourceAccessor::Type::tDirectory) + doPath(st, curPath, relPath, false); } else { logger->cout(relPath); if (recursive) { - auto st = accessor->stat(curPath); - assert(st); - if (st->type == FSAccessor::Type::tDirectory) - doPath(*st, curPath, relPath, false); + auto st = accessor->lstat(curPath); + if (st.type == SourceAccessor::Type::tDirectory) + doPath(st, curPath, relPath, false); } } }; - doPath = [&](const FSAccessor::Stat & st, const Path & curPath, - const std::string & relPath, bool showDirectory) + doPath = [&](const SourceAccessor::Stat & st, const CanonPath & curPath, + std::string_view relPath, bool showDirectory) { - if (st.type == FSAccessor::Type::tDirectory && !showDirectory) { + if (st.type == SourceAccessor::Type::tDirectory && !showDirectory) { auto names = accessor->readDirectory(curPath); for (auto & [name, type] : names) - showFile(curPath + "/" + name, relPath + "/" + name); + showFile(curPath + name, relPath + "/" + name); } else showFile(curPath, relPath); }; - auto st = accessor->stat(path); - if (!st) - throw Error("path '%1%' does not exist", path); - doPath(*st, path, - st->type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)), + auto path2 = CanonPath(path); + auto st = accessor->lstat(path2); + doPath(st, path2, + st.type == SourceAccessor::Type::tDirectory ? "." : path2.baseName().value_or(""), showDirectory); } - void list(ref accessor) + void list(ref accessor) { - if (path == "/") path = ""; - if (json) { if (showDirectory) throw UsageError("'--directory' is useless with '--json'"); - logger->cout("%s", listNar(accessor, path, recursive)); + logger->cout("%s", listNar(accessor, CanonPath(path), recursive)); } else listText(accessor); } diff --git a/src/nix/run.cc b/src/nix/run.cc index 07806283c..1465e8cde 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -6,7 +6,7 @@ #include "derivations.hh" #include "local-store.hh" #include "finally.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "progress-bar.hh" #include "eval.hh" #include "build/personality.hh" @@ -119,9 +119,9 @@ struct CmdShell : InstallablesCommand, MixEnvironment if (true) unixPath.push_front(store->printStorePath(path) + "/bin"); - auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages"; - if (auto st = accessor->stat(propPath); st && st->type == SourceAccessor::tRegular) { - for (auto & p : tokenizeString(readFile(propPath))) + auto propPath = CanonPath(store->printStorePath(path)) + "nix-support" + "propagated-user-env-packages"; + if (auto st = accessor->maybeLstat(propPath); st && st->type == SourceAccessor::tRegular) { + for (auto & p : tokenizeString(accessor->readFile(propPath))) todo.push(store->parseStorePath(p)); } } diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 04c1a0c1c..aecf65922 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "store-api.hh" #include "progress-bar.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "shared.hh" #include @@ -175,7 +175,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions struct BailOut { }; printNode = [&](Node & node, const std::string & firstPad, const std::string & tailPad) { - auto pathS = store->printStorePath(node.path); + CanonPath pathS(store->printStorePath(node.path)); assert(node.dist != inf); if (precise) { @@ -183,7 +183,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions firstPad, node.visited ? "\e[38;5;244m" : "", firstPad != "" ? "→ " : "", - pathS); + pathS.abs()); } if (node.path == dependencyPath && !all @@ -210,25 +210,25 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions contain the reference. */ std::map hits; - std::function visitPath; + std::function visitPath; - visitPath = [&](const Path & p) { - auto st = accessor->stat(p); + visitPath = [&](const CanonPath & p) { + auto st = accessor->maybeLstat(p); assert(st); - auto p2 = p == pathS ? "/" : std::string(p, pathS.size() + 1); + auto p2 = p == pathS ? "/" : p.abs().substr(pathS.abs().size() + 1); auto getColour = [&](const std::string & hash) { return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; }; - if (st->type == FSAccessor::Type::tDirectory) { + if (st->type == SourceAccessor::Type::tDirectory) { auto names = accessor->readDirectory(p); for (auto & [name, type] : names) - visitPath(p + "/" + name); + visitPath(p + name); } - else if (st->type == FSAccessor::Type::tRegular) { + else if (st->type == SourceAccessor::Type::tRegular) { auto contents = accessor->readFile(p); for (auto & hash : hashes) { @@ -246,7 +246,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions } } - else if (st->type == FSAccessor::Type::tSymlink) { + else if (st->type == SourceAccessor::Type::tSymlink) { auto target = accessor->readLink(p); for (auto & hash : hashes) { diff --git a/tests/functional/nar-access.sh b/tests/functional/nar-access.sh index d487d58d2..13d23c342 100644 --- a/tests/functional/nar-access.sh +++ b/tests/functional/nar-access.sh @@ -25,6 +25,12 @@ diff -u baz.cat-nar $storePath/foo/baz nix store cat $storePath/foo/baz > baz.cat-nar diff -u baz.cat-nar $storePath/foo/baz +# Check that 'nix store cat' fails on invalid store paths. +invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo" +mv $storePath $invalidPath +(! nix store cat $invalidPath/foo/baz) +mv $invalidPath $storePath + # Test --json. diff -u \ <(nix nar ls --json $narFile / | jq -S) \ @@ -46,7 +52,7 @@ diff -u \ <(echo '{"type":"regular","size":0}' | jq -S) # Test missing files. -expect 1 nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR' +expect 1 nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist' expect 1 nix store ls $storePath/xyzzy 2>&1 | grep 'does not exist' # Test failure to dump. From 2f5c1a27dc71275c1d4c96cff42beffed0d4d2f7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 17:22:25 +0100 Subject: [PATCH 509/909] LocalStoreAccessor: Reuse PosixSourceAccessor --- src/libstore/local-fs-store.cc | 48 ++++++++-------------------- src/libutil/posix-source-accessor.cc | 3 +- 2 files changed, 15 insertions(+), 36 deletions(-) diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 63497acbd..953f3a264 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,5 +1,5 @@ #include "archive.hh" -#include "source-accessor.hh" +#include "posix-source-accessor.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" @@ -13,7 +13,7 @@ LocalFSStore::LocalFSStore(const Params & params) { } -struct LocalStoreAccessor : public SourceAccessor +struct LocalStoreAccessor : PosixSourceAccessor { ref store; bool requireValidPath; @@ -23,57 +23,35 @@ struct LocalStoreAccessor : public SourceAccessor , requireValidPath(requireValidPath) { } - Path toRealPath(const CanonPath & path) + CanonPath toRealPath(const CanonPath & path) { - auto storePath = store->toStorePath(path.abs()).first; + auto [storePath, rest] = store->toStorePath(path.abs()); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); - return store->getRealStoreDir() + path.abs().substr(store->storeDir.size()); + return CanonPath(store->getRealStoreDir()) + storePath.to_string() + CanonPath(rest); } std::optional maybeLstat(const CanonPath & path) override { - auto realPath = toRealPath(path); - - // FIXME: use PosixSourceAccessor. - struct stat st; - if (::lstat(realPath.c_str(), &st)) { - if (errno == ENOENT || errno == ENOTDIR) return std::nullopt; - throw SysError("getting status of '%1%'", path); - } - - if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) - throw Error("file '%1%' has unsupported type", path); - - return {{ - S_ISREG(st.st_mode) ? Type::tRegular : - S_ISLNK(st.st_mode) ? Type::tSymlink : - Type::tDirectory, - S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0, - S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}}; + return PosixSourceAccessor::maybeLstat(toRealPath(path)); } DirEntries readDirectory(const CanonPath & path) override { - auto realPath = toRealPath(path); - - auto entries = nix::readDirectory(realPath); - - DirEntries res; - for (auto & entry : entries) - res.insert_or_assign(entry.name, std::nullopt); - - return res; + return PosixSourceAccessor::readDirectory(toRealPath(path)); } - std::string readFile(const CanonPath & path) override + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override { - return nix::readFile(toRealPath(path)); + return PosixSourceAccessor::readFile(toRealPath(path), sink, sizeCallback); } std::string readLink(const CanonPath & path) override { - return nix::readLink(toRealPath(path)); + return PosixSourceAccessor::readLink(toRealPath(path)); } }; diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 8a8d64f3f..d5e32d989 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -58,7 +58,8 @@ std::optional PosixSourceAccessor::maybeLstat(const CanonP S_ISDIR(st.st_mode) ? tDirectory : S_ISLNK(st.st_mode) ? tSymlink : tMisc, - .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR + .fileSize = S_ISREG(st.st_mode) ? std::optional(st.st_size) : std::nullopt, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR, }; } From eab92927388bca29027a98199184ebb5e4e3c03a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Wed, 1 Nov 2023 18:10:06 +0100 Subject: [PATCH 510/909] fix: gcc complains about if which doesn't guard the indented statement --- src/libstore/build/local-derivation-goal.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 738e7051e..dcb7dc6bc 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1563,10 +1563,11 @@ void LocalDerivationGoal::addDependency(const StorePath & path) Path source = worker.store.Store::toRealPath(path); Path target = chrootRootDir + worker.store.printStorePath(path); - if (pathExists(target)) + if (pathExists(target)) { // There is a similar debug message in doBind, so only run it in this block to not have double messages. debug("bind-mounting %s -> %s", target, source); throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); + } /* Bind-mount the path into the sandbox. This requires entering its mount namespace, which is not possible From e47984ce0b37cb8e00b66e85703c1ff72de80a73 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Nov 2023 20:19:08 +0100 Subject: [PATCH 511/909] Fix whitespace Co-authored-by: John Ericson --- src/libstore/binary-cache-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index dd9e2f3af..6a52c4c51 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -241,7 +241,7 @@ ref BinaryCacheStore::addToStoreCommon( for (auto & [s2, _type] : narAccessor->readDirectory(dir)) { auto debugPath = dir + s2; - if ( narAccessor->lstat(debugPath).type != SourceAccessor::tRegular + if (narAccessor->lstat(debugPath).type != SourceAccessor::tRegular || !std::regex_match(s2, regex2)) continue; From d7710a40be1a871859d331e9a50cc7f31797d792 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Nov 2023 20:05:23 -0400 Subject: [PATCH 512/909] flake: Temporarily get Nixpkgs ahead of Hydra MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/31ed632c692e6a36cfc18083b88ece892f863ed4' (2023-09-21) → 'github:NixOS/nixpkgs/9eb24edd6a0027fed010ccfe300a9734d029983c' (2023-11-01) --- flake.lock | 8 ++++---- flake.nix | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 56df9c3fb..991cef1ee 100644 --- a/flake.lock +++ b/flake.lock @@ -34,16 +34,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1695283060, - "narHash": "sha256-CJz71xhCLlRkdFUSQEL0pIAAfcnWFXMzd9vXhPrnrEg=", + "lastModified": 1698876495, + "narHash": "sha256-nsQo2/mkDUFeAjuu92p0dEqhRvHHiENhkKVIV1y0/Oo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "31ed632c692e6a36cfc18083b88ece892f863ed4", + "rev": "9eb24edd6a0027fed010ccfe300a9734d029983c", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.05-small", + "ref": "release-23.05", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 398ba10a0..7cc4ed7fe 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,9 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; + # FIXME go back to nixos-23.05-small once + # https://github.com/NixOS/nixpkgs/pull/264875 is included. + inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; From 4ba8b182be350a04caf5b7efff6b804d789570ad Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 25 Oct 2023 04:50:43 +0200 Subject: [PATCH 513/909] document store objects in terms of their constituent parts this also rephrases the introductory sentence to be more general, in order to avoid the same word being repeated in short succession. --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/architecture/architecture.md | 2 +- doc/manual/src/glossary.md | 2 +- doc/manual/src/store/index.md | 5 +++-- doc/manual/src/store/store-object.md | 10 ++++++++++ 5 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 doc/manual/src/store/store-object.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 2fe77d2c6..c728f5296 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -18,6 +18,7 @@ - [Uninstalling Nix](installation/uninstall.md) - [Nix Store](store/index.md) - [File System Object](store/file-system-object.md) + - [Store Object](store/store-object.md) - [Nix Language](language/index.md) - [Data Types](language/values.md) - [Language Constructs](language/constructs.md) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 6e832e1f9..79429508f 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -63,7 +63,7 @@ The command line interface and Nix expressions are what users deal with most. > The Nix language itself does not have a notion of *packages* or *configurations*. > As far as we are concerned here, the inputs and results of a build plan are just data. -Underlying the command line interface and the Nix language evaluator is the [Nix store](../glossary.md#gloss-store), a mechanism to keep track of build plans, data, and references between them. +Underlying the command line interface and the Nix language evaluator is the [Nix store](../store/index.md), a mechanism to keep track of build plans, data, and references between them. It can also execute build plans to produce new data, which are made available to the operating system as files. A build plan itself is a series of *build tasks*, together with their build inputs. diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index ad3cc147b..b6d8a433a 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -59,7 +59,7 @@ - [store]{#gloss-store} A collection of store objects, with operations to manipulate that collection. - See [Nix Store] for details. + See [Nix store](./store/index.md) for details. There are many types of stores. See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) for a complete list. diff --git a/doc/manual/src/store/index.md b/doc/manual/src/store/index.md index 316e04179..8a5305062 100644 --- a/doc/manual/src/store/index.md +++ b/doc/manual/src/store/index.md @@ -1,4 +1,5 @@ # Nix Store -The *Nix store* is an abstraction used by Nix to store immutable filesystem artifacts (such as software packages) that can have dependencies (*references*) between them. -There are multiple implementations of the Nix store, such as the actual filesystem (`/nix/store`) and binary caches. +The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data. + +There are multiple implementations of Nix stores with different capabilities, such as the actual filesystem (`/nix/store`) or binary caches. diff --git a/doc/manual/src/store/store-object.md b/doc/manual/src/store/store-object.md new file mode 100644 index 000000000..0b2b84ea5 --- /dev/null +++ b/doc/manual/src/store/store-object.md @@ -0,0 +1,10 @@ +## Store Object + +A Nix store is a collection of *store objects* with *references* between them. +A store object consists of + + - A [file system object](./file-system-object.md) as data + - A set of [store paths](@docroot@/glossary.md#gloss-store-path) as references to other store objects + +Store objects are [immutable](https://en.wikipedia.org/wiki/Immutable_object): +Once created, they do not change until they are deleted. From d7b7a79f3ef865ebe5f61962a7c2737cdb5d6445 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 23 Oct 2023 01:39:26 +0200 Subject: [PATCH 514/909] document store paths update the glossary to point to the new page. since this is a cross-cutting concern, it warrants its own section in the manual. Co-authored-by: John Ericson --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/glossary.md | 9 ++-- doc/manual/src/store/store-object.md | 2 +- doc/manual/src/store/store-path.md | 69 ++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 4 deletions(-) create mode 100644 doc/manual/src/store/store-path.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c728f5296..794f78a07 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -19,6 +19,7 @@ - [Nix Store](store/index.md) - [File System Object](store/file-system-object.md) - [Store Object](store/store-object.md) + - [Store Path](store/store-path.md) - [Nix Language](language/index.md) - [Data Types](language/values.md) - [Language Constructs](language/constructs.md) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index b6d8a433a..07891175a 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -86,10 +86,13 @@ - [store path]{#gloss-store-path} - The location of a [store object] in the file system, i.e., an - immediate child of the Nix store directory. + The location of a [store object](@docroot@/store/index.md#store-object) in the file system, i.e., an immediate child of the Nix store directory. - Example: `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` + > **Example** + > + > `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` + + See [Store Path](@docroot@/store/store-path.md) for details. [store path]: #gloss-store-path diff --git a/doc/manual/src/store/store-object.md b/doc/manual/src/store/store-object.md index 0b2b84ea5..caf5657d1 100644 --- a/doc/manual/src/store/store-object.md +++ b/doc/manual/src/store/store-object.md @@ -4,7 +4,7 @@ A Nix store is a collection of *store objects* with *references* between them. A store object consists of - A [file system object](./file-system-object.md) as data - - A set of [store paths](@docroot@/glossary.md#gloss-store-path) as references to other store objects + - A set of [store paths](./store-path.md) as references to other store objects Store objects are [immutable](https://en.wikipedia.org/wiki/Immutable_object): Once created, they do not change until they are deleted. diff --git a/doc/manual/src/store/store-path.md b/doc/manual/src/store/store-path.md new file mode 100644 index 000000000..b5ad0c654 --- /dev/null +++ b/doc/manual/src/store/store-path.md @@ -0,0 +1,69 @@ +# Store Path + +Nix implements references to [store objects](./index.md#store-object) as *store paths*. + +Think of a store path as an [opaque], [unique identifier]: +The only way to obtain store path is by adding or building store objects. +A store path will always reference exactly one store object. + +[opaque]: https://en.m.wikipedia.org/wiki/Opaque_data_type +[unique identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier + +Store paths are pairs of + +- A 20-byte digest for identification +- A symbolic name for people to read + +> **Example** +> +> - Digest: `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z` +> - Name: `firefox-33.1` + +To make store objects accessible to operating system processes, stores have to expose store objects through the file system. + +A store path is rendered to a file system path as the concatenation of + +- [Store directory](#store-directory) (typically `/nix/store`) +- Path separator (`/`) +- Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) +- Hyphen (`-`) +- Name + +> **Example** +> +> ``` +> /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 +> |--------| |------------------------------| |----------| +> store directory digest name +> ``` + +## Store Directory + +Every [Nix store](./index.md) has a store directory. + +Not every store can be accessed through the file system. +But if the store has a file system representation, the store directory contains the store’s [file system objects], which can be addressed by [store paths](#store-path). + +[file system objects]: ./file-system-object.md + +This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. + +> **Note** +> +> The store directory defaults to `/nix/store`, but is in principle arbitrary. + +It is important which store a given store object belongs to: +Files in the store object can contain store paths, and processes may read these paths. +Nix can only guarantee referential integrity if store paths do not cross store boundaries. + +Therefore one can only copy store objects to a different store if + +- The source and target stores' directories match + + or + +- The store object in question has no references, that is, contains no store paths + +One cannot copy a store object to a store with a different store directory. +Instead, it has to be rebuilt, together with all its dependencies. +It is in general not enough to replace the store directory string in file contents, as this may render executables unusable by invalidating their internal offsets or checksums. From 55ed09c4f251d87e5aa23c7fb931e87cea63c68d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Thu, 2 Nov 2023 09:22:00 +0100 Subject: [PATCH 515/909] Remove stray executable permissions on source files Noticed because of a warning during an rpm build: *** WARNING: ./usr/src/debug/nix-2.18.1-1.fc40.x86_64/src/nix-copy-closure/nix-copy-closure.cc is executable but has no shebang, removing executable bit *** WARNING: ./usr/src/debug/nix-2.18.1-1.fc40.x86_64/src/nix-channel/nix-channel.cc is executable but has no shebang, removing executable bit --- src/nix-channel/nix-channel.cc | 0 src/nix-copy-closure/nix-copy-closure.cc | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 src/nix-channel/nix-channel.cc mode change 100755 => 100644 src/nix-copy-closure/nix-copy-closure.cc diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc old mode 100755 new mode 100644 diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc old mode 100755 new mode 100644 From d26c317b14bc3f0ce82d5a91acc63e62a8836dee Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Nov 2023 13:40:54 +0100 Subject: [PATCH 516/909] Use expect Co-authored-by: John Ericson --- tests/functional/nar-access.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/nar-access.sh b/tests/functional/nar-access.sh index 13d23c342..218b521fb 100644 --- a/tests/functional/nar-access.sh +++ b/tests/functional/nar-access.sh @@ -28,7 +28,7 @@ diff -u baz.cat-nar $storePath/foo/baz # Check that 'nix store cat' fails on invalid store paths. invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo" mv $storePath $invalidPath -(! nix store cat $invalidPath/foo/baz) +expect 1 nix store cat $invalidPath/foo/baz mv $invalidPath $storePath # Test --json. From b107431816fcbf364aeae6942cc9d1e709635a44 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Nov 2023 11:15:21 -0400 Subject: [PATCH 517/909] Systematize characterization tests a bit more Deduplicating code moreover enforcing the pattern means: - It is easier to write new characterization tests because less boilerplate - It is harder to mess up new tests because there are fewer places to make mistakes. Co-authored-by: Jacek Galowicz --- src/libstore/tests/characterization.hh | 28 ------ src/libstore/tests/common-protocol.cc | 48 ++++------ src/libstore/tests/derivation.cc | 116 ++++++++----------------- src/libstore/tests/libstore.hh | 2 +- src/libstore/tests/protocol.hh | 61 +++++-------- src/libutil/tests/characterization.hh | 111 +++++++++++++++++++++++ 6 files changed, 183 insertions(+), 183 deletions(-) delete mode 100644 src/libstore/tests/characterization.hh create mode 100644 src/libutil/tests/characterization.hh diff --git a/src/libstore/tests/characterization.hh b/src/libstore/tests/characterization.hh deleted file mode 100644 index 46bf4b2e5..000000000 --- a/src/libstore/tests/characterization.hh +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once -///@file - -namespace nix { - -/** - * The path to the `unit-test-data` directory. See the contributing - * guide in the manual for further details. - */ -static Path getUnitTestData() { - return getEnv("_NIX_TEST_UNIT_DATA").value(); -} - -/** - * Whether we should update "golden masters" instead of running tests - * against them. See the contributing guide in the manual for further - * details. - */ -static bool testAccept() { - return getEnv("_NIX_TEST_ACCEPT") == "1"; -} - -constexpr std::string_view cannotReadGoldenMaster = - "Cannot read golden master because another test is also updating it"; - -constexpr std::string_view updatingGoldenMaster = - "Updating golden master"; -} diff --git a/src/libstore/tests/common-protocol.cc b/src/libstore/tests/common-protocol.cc index b3f4977d2..c09ac6a3e 100644 --- a/src/libstore/tests/common-protocol.cc +++ b/src/libstore/tests/common-protocol.cc @@ -20,16 +20,9 @@ public: * Golden test for `T` reading */ template - void readTest(PathView testStem, T value) + void readProtoTest(PathView testStem, const T & expected) { - if (testAccept()) - { - GTEST_SKIP() << cannotReadGoldenMaster; - } - else - { - auto encoded = readFile(goldenMaster(testStem)); - + CharacterizationTest::readTest(testStem, [&](const auto & encoded) { T got = ({ StringSource from { encoded }; CommonProto::Serialise::read( @@ -37,44 +30,33 @@ public: CommonProto::ReadConn { .from = from }); }); - ASSERT_EQ(got, value); - } + ASSERT_EQ(got, expected); + }); } /** * Golden test for `T` write */ template - void writeTest(PathView testStem, const T & value) + void writeProtoTest(PathView testStem, const T & decoded) { - auto file = goldenMaster(testStem); - - StringSink to; - CommonProto::write( - *store, - CommonProto::WriteConn { .to = to }, - value); - - if (testAccept()) - { - createDirs(dirOf(file)); - writeFile(file, to.s); - GTEST_SKIP() << updatingGoldenMaster; - } - else - { - auto expected = readFile(file); - ASSERT_EQ(to.s, expected); - } + CharacterizationTest::writeTest(testStem, [&]() -> std::string { + StringSink to; + CommonProto::Serialise::write( + *store, + CommonProto::WriteConn { .to = to }, + decoded); + return to.s; + }); } }; #define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ TEST_F(CommonProtoTest, NAME ## _read) { \ - readTest(STEM, VALUE); \ + readProtoTest(STEM, VALUE); \ } \ TEST_F(CommonProtoTest, NAME ## _write) { \ - writeTest(STEM, VALUE); \ + writeProtoTest(STEM, VALUE); \ } CHARACTERIZATION_TEST( diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index ca0cdff71..29d5693db 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -11,20 +11,20 @@ namespace nix { using nlohmann::json; -class DerivationTest : public LibStoreTest +class DerivationTest : public CharacterizationTest, public LibStoreTest { + Path unitTestData = getUnitTestData() + "/libstore/derivation"; + public: + Path goldenMaster(std::string_view testStem) const override { + return unitTestData + "/" + testStem; + } + /** * We set these in tests rather than the regular globals so we don't have * to worry about race conditions if the tests run concurrently. */ ExperimentalFeatureSettings mockXpSettings; - - Path unitTestData = getUnitTestData() + "/libstore/derivation"; - - Path goldenMaster(std::string_view testStem) { - return unitTestData + "/" + testStem; - } }; class CaDerivationTest : public DerivationTest @@ -73,14 +73,8 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { #define TEST_JSON(FIXTURE, NAME, VAL, DRV_NAME, OUTPUT_NAME) \ TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \ - if (testAccept()) \ - { \ - GTEST_SKIP() << cannotReadGoldenMaster; \ - } \ - else \ - { \ - auto encoded = json::parse( \ - readFile(goldenMaster("output-" #NAME ".json"))); \ + readTest("output-" #NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ DerivationOutput got = DerivationOutput::fromJSON( \ *store, \ DRV_NAME, \ @@ -89,28 +83,20 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { mockXpSettings); \ DerivationOutput expected { VAL }; \ ASSERT_EQ(got, expected); \ - } \ + }); \ } \ \ TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ - auto file = goldenMaster("output-" #NAME ".json"); \ - \ - json got = DerivationOutput { VAL }.toJSON( \ - *store, \ - DRV_NAME, \ - OUTPUT_NAME); \ - \ - if (testAccept()) \ - { \ - createDirs(dirOf(file)); \ - writeFile(file, got.dump(2) + "\n"); \ - GTEST_SKIP() << updatingGoldenMaster; \ - } \ - else \ - { \ - auto expected = json::parse(readFile(file)); \ - ASSERT_EQ(got, expected); \ - } \ + writeTest("output-" #NAME ".json", [&]() -> json { \ + return DerivationOutput { (VAL) }.toJSON( \ + *store, \ + (DRV_NAME), \ + (OUTPUT_NAME)); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ } TEST_JSON(DerivationTest, inputAddressed, @@ -167,50 +153,30 @@ TEST_JSON(ImpureDerivationTest, impure, #define TEST_JSON(FIXTURE, NAME, VAL) \ TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \ - if (testAccept()) \ - { \ - GTEST_SKIP() << cannotReadGoldenMaster; \ - } \ - else \ - { \ - auto encoded = json::parse( \ - readFile(goldenMaster( #NAME ".json"))); \ + readTest(#NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ Derivation expected { VAL }; \ Derivation got = Derivation::fromJSON( \ *store, \ encoded, \ mockXpSettings); \ ASSERT_EQ(got, expected); \ - } \ + }); \ } \ \ TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \ - auto file = goldenMaster( #NAME ".json"); \ - \ - json got = Derivation { VAL }.toJSON(*store); \ - \ - if (testAccept()) \ - { \ - createDirs(dirOf(file)); \ - writeFile(file, got.dump(2) + "\n"); \ - GTEST_SKIP() << updatingGoldenMaster; \ - } \ - else \ - { \ - auto expected = json::parse(readFile(file)); \ - ASSERT_EQ(got, expected); \ - } \ + writeTest(#NAME ".json", [&]() -> json { \ + return Derivation { VAL }.toJSON(*store); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ } #define TEST_ATERM(FIXTURE, NAME, VAL, DRV_NAME) \ TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \ - if (testAccept()) \ - { \ - GTEST_SKIP() << cannotReadGoldenMaster; \ - } \ - else \ - { \ - auto encoded = readFile(goldenMaster( #NAME ".drv")); \ + readTest(#NAME ".drv", [&](auto encoded) { \ Derivation expected { VAL }; \ auto got = parseDerivation( \ *store, \ @@ -219,25 +185,13 @@ TEST_JSON(ImpureDerivationTest, impure, mockXpSettings); \ ASSERT_EQ(got.toJSON(*store), expected.toJSON(*store)) ; \ ASSERT_EQ(got, expected); \ - } \ + }); \ } \ \ TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \ - auto file = goldenMaster( #NAME ".drv"); \ - \ - auto got = (VAL).unparse(*store, false); \ - \ - if (testAccept()) \ - { \ - createDirs(dirOf(file)); \ - writeFile(file, got); \ - GTEST_SKIP() << updatingGoldenMaster; \ - } \ - else \ - { \ - auto expected = readFile(file); \ - ASSERT_EQ(got, expected); \ - } \ + writeTest(#NAME ".drv", [&]() -> std::string { \ + return (VAL).unparse(*store, false); \ + }); \ } Derivation makeSimpleDrv(const Store & store) { diff --git a/src/libstore/tests/libstore.hh b/src/libstore/tests/libstore.hh index ef93457b5..78b162b95 100644 --- a/src/libstore/tests/libstore.hh +++ b/src/libstore/tests/libstore.hh @@ -8,7 +8,7 @@ namespace nix { -class LibStoreTest : public ::testing::Test { +class LibStoreTest : public virtual ::testing::Test { public: static void SetUpTestSuite() { initLibStore(); diff --git a/src/libstore/tests/protocol.hh b/src/libstore/tests/protocol.hh index 7fdd3e11c..0378b3e1f 100644 --- a/src/libstore/tests/protocol.hh +++ b/src/libstore/tests/protocol.hh @@ -7,12 +7,11 @@ namespace nix { template -class ProtoTest : public LibStoreTest +class ProtoTest : public CharacterizationTest, public LibStoreTest { -protected: Path unitTestData = getUnitTestData() + "/libstore/" + protocolDir; - Path goldenMaster(std::string_view testStem) { + Path goldenMaster(std::string_view testStem) const override { return unitTestData + "/" + testStem + ".bin"; } }; @@ -25,18 +24,11 @@ public: * Golden test for `T` reading */ template - void readTest(PathView testStem, typename Proto::Version version, T value) + void readProtoTest(PathView testStem, typename Proto::Version version, T expected) { - if (testAccept()) - { - GTEST_SKIP() << cannotReadGoldenMaster; - } - else - { - auto expected = readFile(ProtoTest::goldenMaster(testStem)); - + CharacterizationTest::readTest(testStem, [&](const auto & encoded) { T got = ({ - StringSource from { expected }; + StringSource from { encoded }; Proto::template Serialise::read( *LibStoreTest::store, typename Proto::ReadConn { @@ -45,47 +37,36 @@ public: }); }); - ASSERT_EQ(got, value); - } + ASSERT_EQ(got, expected); + }); } /** * Golden test for `T` write */ template - void writeTest(PathView testStem, typename Proto::Version version, const T & value) + void writeProtoTest(PathView testStem, typename Proto::Version version, const T & decoded) { - auto file = ProtoTest::goldenMaster(testStem); - - StringSink to; - Proto::write( - *LibStoreTest::store, - typename Proto::WriteConn { - .to = to, - .version = version, - }, - value); - - if (testAccept()) - { - createDirs(dirOf(file)); - writeFile(file, to.s); - GTEST_SKIP() << updatingGoldenMaster; - } - else - { - auto expected = readFile(file); - ASSERT_EQ(to.s, expected); - } + CharacterizationTest::writeTest(testStem, [&]() { + StringSink to; + Proto::template Serialise::write( + *LibStoreTest::store, + typename Proto::WriteConn { + .to = to, + .version = version, + }, + decoded); + return std::move(to.s); + }); } }; #define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ TEST_F(FIXTURE, NAME ## _read) { \ - readTest(STEM, VERSION, VALUE); \ + readProtoTest(STEM, VERSION, VALUE); \ } \ TEST_F(FIXTURE, NAME ## _write) { \ - writeTest(STEM, VERSION, VALUE); \ + writeProtoTest(STEM, VERSION, VALUE); \ } } diff --git a/src/libutil/tests/characterization.hh b/src/libutil/tests/characterization.hh new file mode 100644 index 000000000..10c8b4f7e --- /dev/null +++ b/src/libutil/tests/characterization.hh @@ -0,0 +1,111 @@ +#pragma once +///@file + +#include + +#include "types.hh" + +namespace nix { + +/** + * The path to the `unit-test-data` directory. See the contributing + * guide in the manual for further details. + */ +static Path getUnitTestData() { + return getEnv("_NIX_TEST_UNIT_DATA").value(); +} + +/** + * Whether we should update "golden masters" instead of running tests + * against them. See the contributing guide in the manual for further + * details. + */ +static bool testAccept() { + return getEnv("_NIX_TEST_ACCEPT") == "1"; +} + +/** + * Mixin class for writing characterization tests + */ +class CharacterizationTest : public virtual ::testing::Test +{ +protected: + /** + * While the "golden master" for this characterization test is + * located. It should not be shared with any other test. + */ + virtual Path goldenMaster(PathView testStem) const = 0; + +public: + /** + * Golden test for reading + * + * @param test hook that takes the contents of the file and does the + * actual work + */ + void readTest(PathView testStem, auto && test) + { + auto file = goldenMaster(testStem); + + if (testAccept()) + { + GTEST_SKIP() + << "Cannot read golden master " + << file + << "because another test is also updating it"; + } + else + { + test(readFile(file)); + } + } + + /** + * Golden test for writing + * + * @param test hook that produces contents of the file and does the + * actual work + */ + template + void writeTest( + PathView testStem, + std::invocable<> auto && test, + std::invocable auto && readFile2, + std::invocable auto && writeFile2) + { + auto file = goldenMaster(testStem); + + T got = test(); + + if (testAccept()) + { + createDirs(dirOf(file)); + writeFile2(file, got); + GTEST_SKIP() + << "Updating golden master " + << file; + } + else + { + T expected = readFile2(file); + ASSERT_EQ(got, expected); + } + } + + /** + * Specialize to `std::string` + */ + void writeTest(PathView testStem, auto && test) + { + writeTest( + testStem, test, + [](const Path & f) -> std::string { + return readFile(f); + }, + [](const Path & f, const std::string & c) { + return writeFile(f, c); + }); + } +}; + +} From d15c3a33e680228c9deaa6d0898d4680cdc8dbc3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Nov 2023 16:11:20 -0400 Subject: [PATCH 518/909] Don't use `std::invocable` C++ concept yet It s not supported on all platforms yet. Can revert this once it is. --- src/libstore/tests/derivation.cc | 4 ++-- src/libutil/tests/characterization.hh | 12 ++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 29d5693db..7becfa5ab 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -87,7 +87,7 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { } \ \ TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ - writeTest("output-" #NAME ".json", [&]() -> json { \ + writeTest("output-" #NAME ".json", [&]() -> json { \ return DerivationOutput { (VAL) }.toJSON( \ *store, \ (DRV_NAME), \ @@ -165,7 +165,7 @@ TEST_JSON(ImpureDerivationTest, impure, } \ \ TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \ - writeTest(#NAME ".json", [&]() -> json { \ + writeTest(#NAME ".json", [&]() -> json { \ return Derivation { VAL }.toJSON(*store); \ }, [](const auto & file) { \ return json::parse(readFile(file)); \ diff --git a/src/libutil/tests/characterization.hh b/src/libutil/tests/characterization.hh index 10c8b4f7e..6698c5239 100644 --- a/src/libutil/tests/characterization.hh +++ b/src/libutil/tests/characterization.hh @@ -66,16 +66,12 @@ public: * @param test hook that produces contents of the file and does the * actual work */ - template void writeTest( - PathView testStem, - std::invocable<> auto && test, - std::invocable auto && readFile2, - std::invocable auto && writeFile2) + PathView testStem, auto && test, auto && readFile2, auto && writeFile2) { auto file = goldenMaster(testStem); - T got = test(); + auto got = test(); if (testAccept()) { @@ -87,7 +83,7 @@ public: } else { - T expected = readFile2(file); + decltype(got) expected = readFile2(file); ASSERT_EQ(got, expected); } } @@ -97,7 +93,7 @@ public: */ void writeTest(PathView testStem, auto && test) { - writeTest( + writeTest( testStem, test, [](const Path & f) -> std::string { return readFile(f); From e5908212e25f2cb7a36ec176a1c7fcb2d522088b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 3 Nov 2023 11:03:58 +0100 Subject: [PATCH 519/909] Fix nar-access test on macOS --- tests/functional/nar-access.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/functional/nar-access.sh b/tests/functional/nar-access.sh index 218b521fb..87981e7d9 100644 --- a/tests/functional/nar-access.sh +++ b/tests/functional/nar-access.sh @@ -27,9 +27,8 @@ diff -u baz.cat-nar $storePath/foo/baz # Check that 'nix store cat' fails on invalid store paths. invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo" -mv $storePath $invalidPath +cp -r $storePath $invalidPath expect 1 nix store cat $invalidPath/foo/baz -mv $invalidPath $storePath # Test --json. diff -u \ From 55dd1244d280d768bfebb8ca2ec93e061d7aa4eb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 3 Nov 2023 11:39:50 +0100 Subject: [PATCH 520/909] parseDerivation(): Fix warning about uninitialized 'version' variable --- src/libstore/derivations.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index efdad18e1..1fecd1c97 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -352,7 +352,7 @@ Derivation parseDerivation( expect(str, "erive("); version = DerivationATermVersion::Traditional; break; - case 'r': + case 'r': { expect(str, "rvWithVersion("); auto versionS = parseString(str); if (versionS == "xp-dyn-drv") { @@ -365,6 +365,9 @@ Derivation parseDerivation( expect(str, ","); break; } + default: + throw Error("derivation does not start with 'Derive' or 'DrvWithVersion'"); + } /* Parse the list of outputs. */ expect(str, "["); From b0455e9931fbcd996b1b240a4513132c36cf852c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 3 Nov 2023 11:58:47 +0100 Subject: [PATCH 521/909] Fix uninitialized variable warnings on i686-linux https://hydra.nixos.org/build/239849607 --- src/libcmd/command.cc | 4 ++-- src/libcmd/installables.cc | 2 +- src/libstore/store-api.cc | 2 +- src/nix-build/nix-build.cc | 2 +- src/nix-env/nix-env.cc | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index a88ba8134..de9f546fc 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -175,7 +175,7 @@ void BuiltPathsCommand::run(ref store, Installables && installables) throw UsageError("'--all' does not expect arguments"); // XXX: Only uses opaque paths, ignores all the realisations for (auto & p : store->queryAllValidPaths()) - paths.push_back(BuiltPath::Opaque{p}); + paths.emplace_back(BuiltPath::Opaque{p}); } else { paths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables); if (recursive) { @@ -188,7 +188,7 @@ void BuiltPathsCommand::run(ref store, Installables && installables) } store->computeFSClosure(pathsRoots, pathsClosure); for (auto & path : pathsClosure) - paths.push_back(BuiltPath::Opaque{path}); + paths.emplace_back(BuiltPath::Opaque{path}); } } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 3aff601e0..bc0b8a988 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -663,7 +663,7 @@ BuiltPaths Installable::toBuiltPaths( BuiltPaths res; for (auto & drvPath : Installable::toDerivations(store, installables, true)) - res.push_back(BuiltPath::Opaque{drvPath}); + res.emplace_back(BuiltPath::Opaque{drvPath}); return res; } } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index ac96e8bb1..646b0ec7d 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -819,7 +819,7 @@ void Store::substitutePaths(const StorePathSet & paths) std::vector paths2; for (auto & path : paths) if (!path.isDerivation()) - paths2.push_back(DerivedPath::Opaque{path}); + paths2.emplace_back(DerivedPath::Opaque{path}); uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; queryMissing(paths2, diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index e62c4f6b1..60bc08146 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -449,7 +449,7 @@ static void main_nix_build(int argc, char * * argv) } } for (const auto & src : drv.inputSrcs) { - pathsToBuild.push_back(DerivedPath::Opaque{src}); + pathsToBuild.emplace_back(DerivedPath::Opaque{src}); pathsToCopy.insert(src); } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 01742daa8..25068f801 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -481,12 +481,12 @@ static void printMissing(EvalState & state, DrvInfos & elems) std::vector targets; for (auto & i : elems) if (auto drvPath = i.queryDrvPath()) - targets.push_back(DerivedPath::Built{ + targets.emplace_back(DerivedPath::Built{ .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, }); else - targets.push_back(DerivedPath::Opaque{ + targets.emplace_back(DerivedPath::Opaque{ .path = i.queryOutPath(), }); From 60b363936d2fd53ac8741d35ba30ff1e4c405a9f Mon Sep 17 00:00:00 2001 From: r-vdp Date: Tue, 31 Oct 2023 17:32:09 +0100 Subject: [PATCH 522/909] libstore/ssh-ng: Fix phase reporting in log files. When doing local builds, we get phase reporting lines in the log file, they look like '@nix {"action":"setPhase","phase":"unpackPhase"}'. With the ssh-ng protocol, we do have access to these messages, but since we are only including messages of type resBuildLogLine in the logs, the phase information does not end up in the log file. The phase reporting could probably be improved altoghether (it looks like it is kind of accidental that these JSON messages for phase reporting show up but others don't, just because they are actually emitted by nixpkgs' stdenv), but as a first step I propose to make ssh-ng behave in the same way as local builds do. --- src/libstore/build/derivation-goal.cc | 23 +++++- tests/nixos/default.nix | 2 + tests/nixos/remote-builds-ssh-ng.nix | 108 ++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 tests/nixos/remote-builds-ssh-ng.nix diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 360c6b70b..0cfa9a148 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1317,9 +1317,26 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data) auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true); // ensure that logs from a builder using `ssh-ng://` as protocol // are also available to `nix log`. - if (s && !isWrittenToLog && logSink && (*json)["type"] == resBuildLogLine) { - auto f = (*json)["fields"]; - (*logSink)((f.size() > 0 ? f.at(0).get() : "") + "\n"); + if (s && !isWrittenToLog && logSink) { + const auto type = (*json)["type"]; + const auto fields = (*json)["fields"]; + if (type == resBuildLogLine) { + (*logSink)((fields.size() > 0 ? fields[0].get() : "") + "\n"); + } else if (type == resSetPhase && ! fields.is_null()) { + const auto phase = fields[0]; + if (! phase.is_null()) { + // nixpkgs' stdenv produces lines in the log to signal + // phase changes. + // We want to get the same lines in case of remote builds. + // The format is: + // @nix { "action": "setPhase", "phase": "$curPhase" } + const auto logLine = nlohmann::json::object({ + {"action", "setPhase"}, + {"phase", phase} + }); + (*logSink)("@nix " + logLine.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace) + "\n"); + } + } } } currentHookLine.clear(); diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index b391d7ef2..4459aa664 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -21,6 +21,8 @@ in remoteBuilds = runNixOSTestFor "x86_64-linux" ./remote-builds.nix; + remoteBuildsSshNg = runNixOSTestFor "x86_64-linux" ./remote-builds-ssh-ng.nix; + nix-copy-closure = runNixOSTestFor "x86_64-linux" ./nix-copy-closure.nix; nix-copy = runNixOSTestFor "x86_64-linux" ./nix-copy.nix; diff --git a/tests/nixos/remote-builds-ssh-ng.nix b/tests/nixos/remote-builds-ssh-ng.nix new file mode 100644 index 000000000..b59dde9bf --- /dev/null +++ b/tests/nixos/remote-builds-ssh-ng.nix @@ -0,0 +1,108 @@ +{ config, lib, hostPkgs, ... }: + +let + pkgs = config.nodes.client.nixpkgs.pkgs; + + # Trivial Nix expression to build remotely. + expr = config: nr: pkgs.writeText "expr.nix" + '' + let utils = builtins.storePath ${config.system.build.extraUtils}; in + derivation { + name = "hello-${toString nr}"; + system = "i686-linux"; + PATH = "''${utils}/bin"; + builder = "''${utils}/bin/sh"; + args = [ "-c" "${ + lib.concatStringsSep "; " [ + ''if [[ -n $NIX_LOG_FD ]]'' + ''then echo '@nix {\"action\":\"setPhase\",\"phase\":\"buildPhase\"}' >&''$NIX_LOG_FD'' + "fi" + "echo Hello" + "mkdir $out" + "cat /proc/sys/kernel/hostname > $out/host" + ] + }" ]; + outputs = [ "out" ]; + } + ''; +in + +{ + name = "remote-builds-ssh-ng"; + + nodes = + { builder = + { config, pkgs, ... }: + { services.openssh.enable = true; + virtualisation.writableStore = true; + nix.settings.sandbox = true; + nix.settings.substituters = lib.mkForce [ ]; + }; + + client = + { config, lib, pkgs, ... }: + { nix.settings.max-jobs = 0; # force remote building + nix.distributedBuilds = true; + nix.buildMachines = + [ { hostName = "builder"; + sshUser = "root"; + sshKey = "/root/.ssh/id_ed25519"; + system = "i686-linux"; + maxJobs = 1; + protocol = "ssh-ng"; + } + ]; + virtualisation.writableStore = true; + virtualisation.additionalPaths = [ config.system.build.extraUtils ]; + nix.settings.substituters = lib.mkForce [ ]; + programs.ssh.extraConfig = "ConnectTimeout 30"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import subprocess + + start_all() + + # Create an SSH key on the client. + subprocess.run([ + "${hostPkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + client.succeed("mkdir -p -m 700 /root/.ssh") + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") + + # Install the SSH key on the builder. + client.wait_for_unit("network.target") + builder.succeed("mkdir -p -m 700 /root/.ssh") + builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + builder.wait_for_unit("sshd") + client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") + + # Perform a build + out = client.succeed("nix-build ${expr nodes.client.config 1} 2> build-output") + + # Verify that the build was done on the builder + builder.succeed(f"test -e {out.strip()}") + + # Print the build log, prefix the log lines to avoid nix intercepting lines starting with @nix + buildOutput = client.succeed("sed -e 's/^/build-output:/' build-output") + print(buildOutput) + + # Make sure that we get the expected build output + client.succeed("grep -qF Hello build-output") + + # We don't want phase reporting in the build output + client.fail("grep -qF '@nix' build-output") + + # Get the log file + client.succeed(f"nix-store --read-log {out.strip()} > log-output") + # Prefix the log lines to avoid nix intercepting lines starting with @nix + logOutput = client.succeed("sed -e 's/^/log-file:/' log-output") + print(logOutput) + + # Check that we get phase reporting in the log file + client.succeed("grep -q '@nix {\"action\":\"setPhase\",\"phase\":\"buildPhase\"}' log-output") + ''; +} From 6df32889a51510dff44c776fa312b7ba61ab8edf Mon Sep 17 00:00:00 2001 From: BootRhetoric <110117466+BootRhetoric@users.noreply.github.com> Date: Fri, 20 Oct 2023 21:16:56 +0200 Subject: [PATCH 523/909] Add git commit verification input attributes This implements the git input attributes `verifyCommit`, `keytype`, `publicKey` and `publicKeys` as experimental feature `verified-fetches`. `publicKeys` should be a json string. This representation was chosen because all attributes must be of type bool, int or string so they can be included in flake uris (see definition of fetchers::Attr). --- src/libfetchers/fetchers.cc | 5 ++ src/libfetchers/fetchers.hh | 9 +++ src/libfetchers/git.cc | 104 +++++++++++++++++++++++++-- src/libutil/experimental-features.cc | 11 ++- src/libutil/experimental-features.hh | 1 + 5 files changed, 124 insertions(+), 6 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 92692d23a..895515327 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -360,4 +360,9 @@ std::optional InputScheme::experimentalFeature() const return {}; } +std::string publicKeys_to_string(const std::vector& publicKeys) +{ + return ((nlohmann::json) publicKeys).dump(); +} + } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 7d768bac1..a056c8939 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -182,4 +182,13 @@ void registerInputScheme(std::shared_ptr && fetcher); nlohmann::json dumpRegisterInputSchemeInfo(); +struct PublicKey +{ + std::string type = "ssh-ed25519"; + std::string key; +}; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(PublicKey, type, key) + +std::string publicKeys_to_string(const std::vector&); + } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index d625fe01e..51e551879 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -143,6 +143,69 @@ struct WorkdirInfo bool hasHead = false; }; +std::vector getPublicKeys(const Attrs & attrs) { + std::vector publicKeys; + if (attrs.contains("publicKeys")) { + nlohmann::json publicKeysJson = nlohmann::json::parse(getStrAttr(attrs, "publicKeys")); + ensureType(publicKeysJson, nlohmann::json::value_t::array); + publicKeys = publicKeysJson.get>(); + } + else { + publicKeys = {}; + } + if (attrs.contains("publicKey")) + publicKeys.push_back(PublicKey{maybeGetStrAttr(attrs, "keytype").value_or("ssh-ed25519"),getStrAttr(attrs, "publicKey")}); + return publicKeys; +} + +void doCommitVerification(const Path repoDir, const Path gitDir, const std::string rev, const std::vector& publicKeys) { + // Create ad-hoc allowedSignersFile and populate it with publicKeys + auto allowedSignersFile = createTempFile().second; + std::string allowedSigners; + for (const PublicKey& k : publicKeys) { + if (k.type != "ssh-dsa" + && k.type != "ssh-ecdsa" + && k.type != "ssh-ecdsa-sk" + && k.type != "ssh-ed25519" + && k.type != "ssh-ed25519-sk" + && k.type != "ssh-rsa") + warn("Unknow keytype: %s\n" + "Please use one of\n" + "- ssh-dsa\n" + "- ssh-ecdsa\n" + "- ssh-ecdsa-sk\n" + "- ssh-ed25519\n" + "- ssh-ed25519-sk\n" + "- ssh-rsa", k.type); + allowedSigners += "* " + k.type + " " + k.key + "\n"; + } + writeFile(allowedSignersFile, allowedSigners); + + // Run verification command + auto [status, output] = runProgram(RunOptions { + .program = "git", + .args = {"-c", "gpg.ssh.allowedSignersFile=" + allowedSignersFile, "-C", repoDir, + "--git-dir", gitDir, "verify-commit", rev}, + .mergeStderrToStdout = true, + }); + + /* Evaluate result through status code and checking if public key fingerprints appear on stderr + * This is neccessary because the git command might also succeed due to the commit being signed by gpg keys + * that are present in the users key agent. */ + std::string re = R"(Good "git" signature for \* with .* key SHA256:[)"; + for (const PublicKey& k : publicKeys){ + // Calculate sha256 fingerprint from public key and escape the regex symbol '+' to match the key literally + auto fingerprint = trim(hashString(htSHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "="); + auto escaped_fingerprint = std::regex_replace(fingerprint, std::regex("\\+"), "\\+" ); + re += "(" + escaped_fingerprint + ")"; + } + re += "]"; + if (status == 0 && std::regex_search(output, std::regex(re))) + printTalkative("Commit signature verification on commit %s succeeded", rev); + else + throw Error("Commit signature verification on commit %s failed: \n%s", rev, output); +} + // Returns whether a git workdir is clean and has commits. WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir) { @@ -272,9 +335,9 @@ struct GitInputScheme : InputScheme attrs.emplace("type", "git"); for (auto & [name, value] : url.query) { - if (name == "rev" || name == "ref") + if (name == "rev" || name == "ref" || name == "keytype" || name == "publicKey" || name == "publicKeys") attrs.emplace(name, value); - else if (name == "shallow" || name == "submodules" || name == "allRefs") + else if (name == "shallow" || name == "submodules" || name == "allRefs" || name == "verifyCommit") attrs.emplace(name, Explicit { value == "1" }); else url2.query.emplace(name, value); @@ -306,14 +369,26 @@ struct GitInputScheme : InputScheme "name", "dirtyRev", "dirtyShortRev", + "verifyCommit", + "keytype", + "publicKey", + "publicKeys", }; } std::optional inputFromAttrs(const Attrs & attrs) const override { + for (auto & [name, _] : attrs) + if (name == "verifyCommit" + || name == "keytype" + || name == "publicKey" + || name == "publicKeys") + experimentalFeatureSettings.require(Xp::VerifiedFetches); + maybeGetBoolAttr(attrs, "shallow"); maybeGetBoolAttr(attrs, "submodules"); maybeGetBoolAttr(attrs, "allRefs"); + maybeGetBoolAttr(attrs, "verifyCommit"); if (auto ref = maybeGetStrAttr(attrs, "ref")) { if (std::regex_search(*ref, badGitRefRegex)) @@ -336,6 +411,15 @@ struct GitInputScheme : InputScheme if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false)) url.query.insert_or_assign("shallow", "1"); + if (maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(false)) + url.query.insert_or_assign("verifyCommit", "1"); + auto publicKeys = getPublicKeys(input.attrs); + if (publicKeys.size() == 1) { + url.query.insert_or_assign("keytype", publicKeys.at(0).type); + url.query.insert_or_assign("publicKey", publicKeys.at(0).key); + } + else if (publicKeys.size() > 1) + url.query.insert_or_assign("publicKeys", publicKeys_to_string(publicKeys)); return url; } @@ -425,6 +509,8 @@ struct GitInputScheme : InputScheme bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); + std::vector publicKeys = getPublicKeys(input.attrs); + bool verifyCommit = maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(!publicKeys.empty()); std::string cacheType = "git"; if (shallow) cacheType += "-shallow"; @@ -445,6 +531,8 @@ struct GitInputScheme : InputScheme {"type", cacheType}, {"name", name}, {"rev", input.getRev()->gitRev()}, + {"verifyCommit", verifyCommit}, + {"publicKeys", publicKeys_to_string(publicKeys)}, }); }; @@ -467,12 +555,15 @@ struct GitInputScheme : InputScheme auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug - /* If this is a local directory and no ref or revision is given, + /* If this is a local directory, no ref or revision is given and no signature verification is needed, allow fetching directly from a dirty workdir. */ if (!input.getRef() && !input.getRev() && isLocal) { auto workdirInfo = getWorkdirInfo(input, actualUrl); if (!workdirInfo.clean) { - return fetchFromWorkdir(store, input, actualUrl, workdirInfo); + if (verifyCommit) + throw Error("Can't fetch from a dirty workdir with commit signature verification enabled."); + else + return fetchFromWorkdir(store, input, actualUrl, workdirInfo); } } @@ -480,6 +571,8 @@ struct GitInputScheme : InputScheme {"type", cacheType}, {"name", name}, {"url", actualUrl}, + {"verifyCommit", verifyCommit}, + {"publicKeys", publicKeys_to_string(publicKeys)}, }); Path repoDir; @@ -637,6 +730,9 @@ struct GitInputScheme : InputScheme ); } + if (verifyCommit) + doCommitVerification(repoDir, gitDir, input.getRev()->gitRev(), publicKeys); + if (submodules) { Path tmpGitDir = createTempDir(); AutoDelete delTmpGitDir(tmpGitDir, true); diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 74af9aae0..47edca3a5 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -227,7 +227,14 @@ constexpr std::array xpFeatureDetails = {{ .description = R"( Allow the use of the [impure-env](@docroot@/command-ref/conf-file.md#conf-impure-env) setting. )", - } + }, + { + .tag = Xp::VerifiedFetches, + .name = "verified-fetches", + .description = R"( + Enables verification of git commit signatures through the [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) built-in. + )", + }, }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index e02f8353e..f005cc9ee 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -32,6 +32,7 @@ enum struct ExperimentalFeature ParseTomlTimestamps, ReadOnlyLocalStore, ConfigurableImpureEnv, + VerifiedFetches, }; /** From 098f0615c9401414a76e66653fbf4c9dd30d55a7 Mon Sep 17 00:00:00 2001 From: BootRhetoric <110117466+BootRhetoric@users.noreply.github.com> Date: Fri, 20 Oct 2023 21:17:14 +0200 Subject: [PATCH 524/909] fetchGit and flake: add publicKeys list input This adds publicKeys as an optional fetcher input attribute to flakes and builtins.fetchGit to provide a nix interface for the json-encoded `publicKeys` attribute of the git fetcher. Co-authored-by: Valentin Gagarin --- doc/manual/src/release-notes/rl-next.md | 3 +- src/libexpr/flake/flake.cc | 10 ++++- src/libexpr/primops/fetchTree.cc | 56 +++++++++++++++++++++++++ src/libfetchers/git.cc | 14 +++---- 4 files changed, 73 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 3cfb53998..8cd69f8fd 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -16,7 +16,6 @@ - `builtins.fetchTree` is now marked as stable. - - The interface for creating and updating lock files has been overhauled: - [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now. @@ -29,3 +28,5 @@ - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. They are superceded by `nix flake update`. + +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 70ae7b584..ded132695 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -8,6 +8,7 @@ #include "fetchers.hh" #include "finally.hh" #include "fetch-settings.hh" +#include "value-to-json.hh" namespace nix { @@ -140,8 +141,13 @@ static FlakeInput parseFlakeInput(EvalState & state, attrs.emplace(state.symbols[attr.name], (long unsigned int)attr.value->integer); break; default: - throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", - state.symbols[attr.name], showType(*attr.value)); + if (attr.name == state.symbols.create("publicKeys")) { + experimentalFeatureSettings.require(Xp::VerifiedFetches); + NixStringContext emptyContext = {}; + attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, emptyContext).dump()); + } else + throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", + state.symbols[attr.name], showType(*attr.value)); } #pragma GCC diagnostic pop } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 767f559be..3717b9022 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -7,6 +7,7 @@ #include "registry.hh" #include "tarball.hh" #include "url.hh" +#include "value-to-json.hh" #include #include @@ -125,6 +126,10 @@ static void fetchTree( attrs.emplace(state.symbols[attr.name], Explicit{attr.value->boolean}); else if (attr.value->type() == nInt) attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer)); + else if (state.symbols[attr.name] == "publicKeys") { + experimentalFeatureSettings.require(Xp::VerifiedFetches); + attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump()); + } else state.debugThrowLastTrace(TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", state.symbols[attr.name], showType(*attr.value))); @@ -427,6 +432,42 @@ static RegisterPrimOp primop_fetchGit({ With this argument being true, it's possible to load a `rev` from *any* `ref` (by default only `rev`s from the specified `ref` are supported). + - `verifyCommit` (default: `true` if `publicKey` or `publicKeys` are provided, otherwise `false`) + + Whether to check `rev` for a signature matching `publicKey` or `publicKeys`. + If `verifyCommit` is enabled, then `fetchGit` cannot use a local repository with uncommitted changes. + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + + - `publicKey` + + The public key against which `rev` is verified if `verifyCommit` is enabled. + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + + - `keytype` (default: `"ssh-ed25519"`) + + The key type of `publicKey`. + Possible values: + - `"ssh-dsa"` + - `"ssh-ecdsa"` + - `"ssh-ecdsa-sk"` + - `"ssh-ed25519"` + - `"ssh-ed25519-sk"` + - `"ssh-rsa"` + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + + - `publicKeys` + + The public keys against which `rev` is verified if `verifyCommit` is enabled. + Must be given as a list of attribute sets with the following form: + ```nix + { + key = ""; + type = ""; # optional, default: "ssh-ed25519" + } + ``` + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + + Here are some examples of how to use `fetchGit`. - To fetch a private repository over SSH: @@ -501,6 +542,21 @@ static RegisterPrimOp primop_fetchGit({ } ``` + - To verify the commit signature: + + ```nix + builtins.fetchGit { + url = "ssh://git@github.com/nixos/nix.git"; + verifyCommit = true; + publicKeys = [ + { + type = "ssh-ed25519"; + key = "AAAAC3NzaC1lZDI1NTE5AAAAIArPKULJOid8eS6XETwUjO48/HKBWl7FTCK0Z//fplDi"; + } + ]; + } + ``` + Nix will refetch the branch according to the [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) setting. This behavior is disabled in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 51e551879..72fba0582 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -169,14 +169,14 @@ void doCommitVerification(const Path repoDir, const Path gitDir, const std::stri && k.type != "ssh-ed25519" && k.type != "ssh-ed25519-sk" && k.type != "ssh-rsa") - warn("Unknow keytype: %s\n" + warn("Unknown keytype: %s\n" "Please use one of\n" "- ssh-dsa\n" - "- ssh-ecdsa\n" - "- ssh-ecdsa-sk\n" - "- ssh-ed25519\n" - "- ssh-ed25519-sk\n" - "- ssh-rsa", k.type); + " ssh-ecdsa\n" + " ssh-ecdsa-sk\n" + " ssh-ed25519\n" + " ssh-ed25519-sk\n" + " ssh-rsa", k.type); allowedSigners += "* " + k.type + " " + k.key + "\n"; } writeFile(allowedSignersFile, allowedSigners); @@ -201,7 +201,7 @@ void doCommitVerification(const Path repoDir, const Path gitDir, const std::stri } re += "]"; if (status == 0 && std::regex_search(output, std::regex(re))) - printTalkative("Commit signature verification on commit %s succeeded", rev); + printTalkative("Signature verification on commit %s succeeded", rev); else throw Error("Commit signature verification on commit %s failed: \n%s", rev, output); } From 271932782dd3d44e0e238bd3234ca1e97996cfea Mon Sep 17 00:00:00 2001 From: BootRhetoric <110117466+BootRhetoric@users.noreply.github.com> Date: Fri, 20 Oct 2023 21:18:01 +0200 Subject: [PATCH 525/909] fetchGit and flake: add commit signature verification tests This adds simple tests of the commit signature verification mechanism of fetchGit and its flake input wrapper. OpenSSH is added to the build dependencies since it's needed to create a key when testing the functionality. It is neither a built- nor a runtime dependency. --- flake.nix | 1 + tests/functional/fetchGitVerification.sh | 76 ++++++++++++++++++++++++ tests/functional/local.mk | 1 + 3 files changed, 78 insertions(+) create mode 100644 tests/functional/fetchGitVerification.sh diff --git a/flake.nix b/flake.nix index 7cc4ed7fe..51d818423 100644 --- a/flake.nix +++ b/flake.nix @@ -185,6 +185,7 @@ buildPackages.git buildPackages.mercurial # FIXME: remove? only needed for tests buildPackages.jq # Also for custom mdBook preprocessor. + buildPackages.openssh # only needed for tests (ssh-keygen) ] ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]; diff --git a/tests/functional/fetchGitVerification.sh b/tests/functional/fetchGitVerification.sh new file mode 100644 index 000000000..4d9209498 --- /dev/null +++ b/tests/functional/fetchGitVerification.sh @@ -0,0 +1,76 @@ +source common.sh + +requireGit +[[ $(type -p ssh-keygen) ]] || skipTest "ssh-keygen not installed" # require ssh-keygen + +enableFeatures "verified-fetches" + +clearStore + +repo="$TEST_ROOT/git" + +# generate signing keys +keysDir=$TEST_ROOT/.ssh +mkdir -p "$keysDir" +ssh-keygen -f "$keysDir/testkey1" -t ed25519 -P "" -C "test key 1" +key1File="$keysDir/testkey1.pub" +publicKey1=$(awk '{print $2}' "$key1File") +ssh-keygen -f "$keysDir/testkey2" -t rsa -P "" -C "test key 2" +key2File="$keysDir/testkey2.pub" +publicKey2=$(awk '{print $2}' "$key2File") + +git init $repo +git -C $repo config user.email "foobar@example.com" +git -C $repo config user.name "Foobar" +git -C $repo config gpg.format ssh + +echo 'hello' > $repo/text +git -C $repo add text +git -C $repo -c "user.signingkey=$key1File" commit -S -m 'initial commit' + +out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = \"file://$repo\"; keytype = \"ssh-rsa\"; publicKey = \"$publicKey2\"; }" 2>&1) || status=$? +[[ $status == 1 ]] +[[ $out =~ 'No principal matched.' ]] +[[ $(nix eval --impure --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; publicKey = \"$publicKey1\"; } + \"/text\")") = 'hello' ]] + +echo 'hello world' > $repo/text +git -C $repo add text +git -C $repo -c "user.signingkey=$key2File" commit -S -m 'second commit' + +[[ $(nix eval --impure --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; publicKeys = [{key = \"$publicKey1\";} {type = \"ssh-rsa\"; key = \"$publicKey2\";}]; } + \"/text\")") = 'hello world' ]] + +# Flake input test +flakeDir="$TEST_ROOT/flake" +mkdir -p "$flakeDir" +cat > "$flakeDir/flake.nix" < "$flakeDir/flake.nix" <&1) || status=$? +[[ $status == 1 ]] +[[ $out =~ 'No principal matched.' ]] \ No newline at end of file diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 3679349f8..fe0d0c4ed 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -55,6 +55,7 @@ nix_tests = \ secure-drv-outputs.sh \ restricted.sh \ fetchGitSubmodules.sh \ + fetchGitVerification.sh \ flakes/search-root.sh \ readfile-context.sh \ nix-channel.sh \ From 9b880e3e29c7a485b0e21495f2d089c5151589cc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 2 Nov 2023 19:39:09 -0400 Subject: [PATCH 526/909] Factor out `MemorySourceAccessor`, implement missing features The new `MemorySourceAccessor` rather than being a slightly lossy flat map is a complete in-memory model of file system objects. Co-authored-by: Eelco Dolstra --- src/libfetchers/input-accessor.hh | 2 +- src/libfetchers/memory-input-accessor.cc | 44 ++------ src/libutil/memory-source-accessor.cc | 124 +++++++++++++++++++++++ src/libutil/memory-source-accessor.hh | 74 ++++++++++++++ 4 files changed, 205 insertions(+), 39 deletions(-) create mode 100644 src/libutil/memory-source-accessor.cc create mode 100644 src/libutil/memory-source-accessor.hh diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 5dc05a363..68fdf07a7 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -14,7 +14,7 @@ struct SourcePath; class StorePath; class Store; -struct InputAccessor : SourceAccessor, std::enable_shared_from_this +struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this { /** * Return the maximum last-modified time of the files in this diff --git a/src/libfetchers/memory-input-accessor.cc b/src/libfetchers/memory-input-accessor.cc index 6468ece41..057f3e37f 100644 --- a/src/libfetchers/memory-input-accessor.cc +++ b/src/libfetchers/memory-input-accessor.cc @@ -1,48 +1,16 @@ #include "memory-input-accessor.hh" +#include "memory-source-accessor.hh" namespace nix { -struct MemoryInputAccessorImpl : MemoryInputAccessor +struct MemoryInputAccessorImpl : MemoryInputAccessor, MemorySourceAccessor { - std::map files; - - std::string readFile(const CanonPath & path) override - { - auto i = files.find(path); - if (i == files.end()) - throw Error("file '%s' does not exist", path); - return i->second; - } - - bool pathExists(const CanonPath & path) override - { - auto i = files.find(path); - return i != files.end(); - } - - std::optional maybeLstat(const CanonPath & path) override - { - auto i = files.find(path); - if (i != files.end()) - return Stat { .type = tRegular, .isExecutable = false }; - return std::nullopt; - } - - DirEntries readDirectory(const CanonPath & path) override - { - return {}; - } - - std::string readLink(const CanonPath & path) override - { - throw UnimplementedError("MemoryInputAccessor::readLink"); - } - SourcePath addFile(CanonPath path, std::string && contents) override { - files.emplace(path, std::move(contents)); - - return {ref(shared_from_this()), std::move(path)}; + return { + ref(shared_from_this()), + MemorySourceAccessor::addFile(path, std::move(contents)) + }; } }; diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc new file mode 100644 index 000000000..f34f6c091 --- /dev/null +++ b/src/libutil/memory-source-accessor.cc @@ -0,0 +1,124 @@ +#include "memory-source-accessor.hh" + +namespace nix { + +MemorySourceAccessor::File * +MemorySourceAccessor::open(const CanonPath & path, std::optional create) +{ + File * cur = &root; + + bool newF = false; + + for (std::string_view name : path) + { + auto * curDirP = std::get_if(&cur->raw); + if (!curDirP) + return nullptr; + auto & curDir = *curDirP; + + auto i = curDir.contents.find(name); + if (i == curDir.contents.end()) { + if (!create) + return nullptr; + else { + newF = true; + i = curDir.contents.insert(i, { + std::string { name }, + File::Directory {}, + }); + } + } + cur = &i->second; + } + + if (newF && create) *cur = std::move(*create); + + return cur; +} + +std::string MemorySourceAccessor::readFile(const CanonPath & path) +{ + auto * f = open(path, std::nullopt); + if (!f) + throw Error("file '%s' does not exist", path); + if (auto * r = std::get_if(&f->raw)) + return r->contents; + else + throw Error("file '%s' is not a regular file", path); +} + +bool MemorySourceAccessor::pathExists(const CanonPath & path) +{ + return open(path, std::nullopt); +} + +MemorySourceAccessor::Stat MemorySourceAccessor::File::lstat() const +{ + return std::visit(overloaded { + [](const Regular & r) { + return Stat { + .type = tRegular, + .fileSize = r.contents.size(), + .isExecutable = r.executable, + }; + }, + [](const Directory &) { + return Stat { + .type = tDirectory, + }; + }, + [](const Symlink &) { + return Stat { + .type = tSymlink, + }; + }, + }, this->raw); +} + +std::optional +MemorySourceAccessor::maybeLstat(const CanonPath & path) +{ + const auto * f = open(path, std::nullopt); + return f ? std::optional { f->lstat() } : std::nullopt; +} + +MemorySourceAccessor::DirEntries MemorySourceAccessor::readDirectory(const CanonPath & path) +{ + auto * f = open(path, std::nullopt); + if (!f) + throw Error("file '%s' does not exist", path); + if (auto * d = std::get_if(&f->raw)) { + DirEntries res; + for (auto & [name, file] : d->contents) + res.insert_or_assign(name, file.lstat().type); + return res; + } else + throw Error("file '%s' is not a directory", path); + return {}; +} + +std::string MemorySourceAccessor::readLink(const CanonPath & path) +{ + auto * f = open(path, std::nullopt); + if (!f) + throw Error("file '%s' does not exist", path); + if (auto * s = std::get_if(&f->raw)) + return s->target; + else + throw Error("file '%s' is not a symbolic link", path); +} + +CanonPath MemorySourceAccessor::addFile(CanonPath path, std::string && contents) +{ + auto * f = open(path, File { File::Regular {} }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + if (auto * r = std::get_if(&f->raw)) + r->contents = std::move(contents); + else + throw Error("file '%s' is not a regular file", path); + + return path; +} + +} diff --git a/src/libutil/memory-source-accessor.hh b/src/libutil/memory-source-accessor.hh new file mode 100644 index 000000000..014fa8098 --- /dev/null +++ b/src/libutil/memory-source-accessor.hh @@ -0,0 +1,74 @@ +#include "source-accessor.hh" +#include "variant-wrapper.hh" + +namespace nix { + +/** + * An source accessor for an in-memory file system. + */ +struct MemorySourceAccessor : virtual SourceAccessor +{ + /** + * In addition to being part of the implementation of + * `MemorySourceAccessor`, this has a side benefit of nicely + * defining what a "file system object" is in Nix. + */ + struct File { + struct Regular { + bool executable = false; + std::string contents; + + GENERATE_CMP(Regular, me->executable, me->contents); + }; + + struct Directory { + using Name = std::string; + + std::map> contents; + + GENERATE_CMP(Directory, me->contents); + }; + + struct Symlink { + std::string target; + + GENERATE_CMP(Symlink, me->target); + }; + + using Raw = std::variant; + Raw raw; + + MAKE_WRAPPER_CONSTRUCTOR(File); + + GENERATE_CMP(File, me->raw); + + Stat lstat() const; + }; + + File root { File::Directory {} }; + + GENERATE_CMP(MemorySourceAccessor, me->root); + + std::string readFile(const CanonPath & path) override; + bool pathExists(const CanonPath & path) override; + std::optional maybeLstat(const CanonPath & path) override; + DirEntries readDirectory(const CanonPath & path) override; + std::string readLink(const CanonPath & path) override; + + /** + * @param create If present, create this file and any parent directories + * that are needed. + * + * Return null if + * + * - `create = false`: File does not exist. + * + * - `create = true`: some parent file was not a dir, so couldn't + * look/create inside. + */ + File * open(const CanonPath & path, std::optional create); + + CanonPath addFile(CanonPath path, std::string && contents); +}; + +} From e97ac09abeab44fa3d10eb539f0b3d51f8575798 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 6 Jan 2023 18:06:03 -0500 Subject: [PATCH 527/909] Factor out `StoreDirConfig` More progress on #5729. --- src/libstore/path.cc | 14 ++-- src/libstore/store-api.cc | 24 +++--- src/libstore/store-api.hh | 103 +------------------------ src/libstore/store-dir-config.hh | 126 +++++++++++++++++++++++++++++++ 4 files changed, 148 insertions(+), 119 deletions(-) create mode 100644 src/libstore/store-dir-config.hh diff --git a/src/libstore/path.cc b/src/libstore/path.cc index ec3e53232..69f6d7356 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -1,4 +1,4 @@ -#include "store-api.hh" +#include "store-dir-config.hh" #include @@ -54,7 +54,7 @@ StorePath StorePath::random(std::string_view name) return StorePath(hash, name); } -StorePath Store::parseStorePath(std::string_view path) const +StorePath StoreDirConfig::parseStorePath(std::string_view path) const { auto p = canonPath(std::string(path)); if (dirOf(p) != storeDir) @@ -62,7 +62,7 @@ StorePath Store::parseStorePath(std::string_view path) const return StorePath(baseNameOf(p)); } -std::optional Store::maybeParseStorePath(std::string_view path) const +std::optional StoreDirConfig::maybeParseStorePath(std::string_view path) const { try { return parseStorePath(path); @@ -71,24 +71,24 @@ std::optional Store::maybeParseStorePath(std::string_view path) const } } -bool Store::isStorePath(std::string_view path) const +bool StoreDirConfig::isStorePath(std::string_view path) const { return (bool) maybeParseStorePath(path); } -StorePathSet Store::parseStorePathSet(const PathSet & paths) const +StorePathSet StoreDirConfig::parseStorePathSet(const PathSet & paths) const { StorePathSet res; for (auto & i : paths) res.insert(parseStorePath(i)); return res; } -std::string Store::printStorePath(const StorePath & path) const +std::string StoreDirConfig::printStorePath(const StorePath & path) const { return (storeDir + "/").append(path.to_string()); } -PathSet Store::printStorePathSet(const StorePathSet & paths) const +PathSet StoreDirConfig::printStorePathSet(const StorePathSet & paths) const { PathSet res; for (auto & i : paths) res.insert(printStorePath(i)); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 646b0ec7d..a681bb6cf 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -23,13 +23,13 @@ using json = nlohmann::json; namespace nix { -bool Store::isInStore(PathView path) const +bool StoreDirConfig::isInStore(PathView path) const { return isInDir(path, storeDir); } -std::pair Store::toStorePath(PathView path) const +std::pair StoreDirConfig::toStorePath(PathView path) const { if (!isInStore(path)) throw Error("path '%1%' is not in the Nix store", path); @@ -143,7 +143,7 @@ StorePath Store::followLinksToStorePath(std::string_view path) const */ -StorePath Store::makeStorePath(std::string_view type, +StorePath StoreDirConfig::makeStorePath(std::string_view type, std::string_view hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ @@ -154,14 +154,14 @@ StorePath Store::makeStorePath(std::string_view type, } -StorePath Store::makeStorePath(std::string_view type, +StorePath StoreDirConfig::makeStorePath(std::string_view type, const Hash & hash, std::string_view name) const { return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name); } -StorePath Store::makeOutputPath(std::string_view id, +StorePath StoreDirConfig::makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const { return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id)); @@ -172,7 +172,7 @@ StorePath Store::makeOutputPath(std::string_view id, hacky, but we can't put them in, say, (per the grammar above) since that would be ambiguous. */ static std::string makeType( - const Store & store, + const StoreDirConfig & store, std::string && type, const StoreReferences & references) { @@ -185,7 +185,7 @@ static std::string makeType( } -StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const +StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", info.references), info.hash, name); @@ -201,7 +201,7 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf } -StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const +StorePath StoreDirConfig::makeTextPath(std::string_view name, const TextInfo & info) const { assert(info.hash.type == htSHA256); return makeStorePath( @@ -214,7 +214,7 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons } -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const +StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const { // New template return std::visit(overloaded { @@ -228,7 +228,7 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA } -std::pair Store::computeStorePathFromDump( +std::pair StoreDirConfig::computeStorePathFromDump( Source & dump, std::string_view name, FileIngestionMethod method, @@ -247,7 +247,7 @@ std::pair Store::computeStorePathFromDump( } -StorePath Store::computeStorePathForText( +StorePath StoreDirConfig::computeStorePathForText( std::string_view name, std::string_view s, const StorePathSet & references) const @@ -1315,7 +1315,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre } -std::string Store::showPaths(const StorePathSet & paths) +std::string StoreDirConfig::showPaths(const StorePathSet & paths) { std::string s; for (auto & i : paths) { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6aa317e3d..bee5ec16c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -14,6 +14,7 @@ #include "config.hh" #include "path-info.hh" #include "repair-flag.hh" +#include "store-dir-config.hh" #include #include @@ -64,7 +65,6 @@ MakeError(InvalidPath, Error); MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); MakeError(SubstituterDisabled, Error); -MakeError(BadStorePath, Error); MakeError(InvalidStoreURI, Error); @@ -97,11 +97,11 @@ struct KeyedBuildResult; typedef std::map> StorePathCAMap; -struct StoreConfig : public Config +struct StoreConfig : public StoreDirConfig { typedef std::map Params; - using Config::Config; + using StoreDirConfig::StoreDirConfig; StoreConfig() = delete; @@ -131,15 +131,6 @@ struct StoreConfig : public Config return std::nullopt; } - const PathSetting storeDir_{this, settings.nixStore, - "store", - R"( - Logical location of the Nix store, usually - `/nix/store`. Note that you can only copy store paths - between stores if they have the same `store` setting. - )"}; - const Path storeDir = storeDir_; - const Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", "Size of the in-memory store path metadata cache."}; @@ -224,45 +215,6 @@ public: virtual std::string getUri() = 0; - StorePath parseStorePath(std::string_view path) const; - - std::optional maybeParseStorePath(std::string_view path) const; - - std::string printStorePath(const StorePath & path) const; - - /** - * Deprecated - * - * \todo remove - */ - StorePathSet parseStorePathSet(const PathSet & paths) const; - - PathSet printStorePathSet(const StorePathSet & path) const; - - /** - * Display a set of paths in human-readable form (i.e., between quotes - * and separated by commas). - */ - std::string showPaths(const StorePathSet & paths); - - /** - * @return true if ‘path’ is in the Nix store (but not the Nix - * store itself). - */ - bool isInStore(PathView path) const; - - /** - * @return true if ‘path’ is a store path, i.e. a direct child of the - * Nix store. - */ - bool isStorePath(std::string_view path) const; - - /** - * Split a path like /nix/store/-/ into - * /nix/store/- and /. - */ - std::pair toStorePath(PathView path) const; - /** * Follow symlinks until we end up with a path in the Nix store. */ @@ -274,55 +226,6 @@ public: */ StorePath followLinksToStorePath(std::string_view path) const; - /** - * Constructs a unique store path name. - */ - StorePath makeStorePath(std::string_view type, - std::string_view hash, std::string_view name) const; - StorePath makeStorePath(std::string_view type, - const Hash & hash, std::string_view name) const; - - StorePath makeOutputPath(std::string_view id, - const Hash & hash, std::string_view name) const; - - StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; - - StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - - StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; - - /** - * Read-only variant of addToStoreFromDump(). It returns the store - * path to which a NAR or flat file would be written. - */ - std::pair computeStorePathFromDump( - Source & dump, - std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, - HashType hashAlgo = htSHA256, - const StorePathSet & references = {}) const; - - /** - * Preparatory part of addTextToStore(). - * - * !!! Computation of the path should take the references given to - * addTextToStore() into account, otherwise we have a (relatively - * minor) security hole: a caller can register a source file with - * bogus references. If there are too many references, the path may - * not be garbage collected when it has to be (not really a problem, - * the caller could create a root anyway), or it may be garbage - * collected when it shouldn't be (more serious). - * - * Hashing the references would solve this (bogus references would - * simply yield a different store path, so other users wouldn't be - * affected), but it has some backwards compatibility issues (the - * hashing scheme changes), so I'm not doing that for now. - */ - StorePath computeStorePathForText( - std::string_view name, - std::string_view s, - const StorePathSet & references) const; - /** * Check whether a path is valid. */ diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh new file mode 100644 index 000000000..53843d663 --- /dev/null +++ b/src/libstore/store-dir-config.hh @@ -0,0 +1,126 @@ +#pragma once + +#include "path.hh" +#include "hash.hh" +#include "content-address.hh" +#include "globals.hh" +#include "config.hh" + +#include +#include +#include + + +namespace nix { + +MakeError(BadStorePath, Error); + +struct StoreDirConfig : public Config +{ + using Config::Config; + + StoreDirConfig() = delete; + + virtual ~StoreDirConfig() = default; + + const PathSetting storeDir_{this, settings.nixStore, + "store", + R"( + Logical location of the Nix store, usually + `/nix/store`. Note that you can only copy store paths + between stores if they have the same `store` setting. + )"}; + const Path storeDir = storeDir_; + + // pure methods + + StorePath parseStorePath(std::string_view path) const; + + std::optional maybeParseStorePath(std::string_view path) const; + + std::string printStorePath(const StorePath & path) const; + + /** + * Deprecated + * + * \todo remove + */ + StorePathSet parseStorePathSet(const PathSet & paths) const; + + PathSet printStorePathSet(const StorePathSet & path) const; + + /** + * Display a set of paths in human-readable form (i.e., between quotes + * and separated by commas). + */ + std::string showPaths(const StorePathSet & paths); + + /** + * @return true if ‘path’ is in the Nix store (but not the Nix + * store itself). + */ + bool isInStore(PathView path) const; + + /** + * @return true if ‘path’ is a store path, i.e. a direct child of the + * Nix store. + */ + bool isStorePath(std::string_view path) const; + + /** + * Split a path like /nix/store/-/ into + * /nix/store/- and /. + */ + std::pair toStorePath(PathView path) const; + + /** + * Constructs a unique store path name. + */ + StorePath makeStorePath(std::string_view type, + std::string_view hash, std::string_view name) const; + StorePath makeStorePath(std::string_view type, + const Hash & hash, std::string_view name) const; + + StorePath makeOutputPath(std::string_view id, + const Hash & hash, std::string_view name) const; + + StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; + + StorePath makeTextPath(std::string_view name, const TextInfo & info) const; + + StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; + + /** + * Read-only variant of addToStoreFromDump(). It returns the store + * path to which a NAR or flat file would be written. + */ + std::pair computeStorePathFromDump( + Source & dump, + std::string_view name, + FileIngestionMethod method = FileIngestionMethod::Recursive, + HashType hashAlgo = htSHA256, + const StorePathSet & references = {}) const; + + /** + * Preparatory part of addTextToStore(). + * + * !!! Computation of the path should take the references given to + * addTextToStore() into account, otherwise we have a (relatively + * minor) security hole: a caller can register a source file with + * bogus references. If there are too many references, the path may + * not be garbage collected when it has to be (not really a problem, + * the caller could create a root anyway), or it may be garbage + * collected when it shouldn't be (more serious). + * + * Hashing the references would solve this (bogus references would + * simply yield a different store path, so other users wouldn't be + * affected), but it has some backwards compatibility issues (the + * hashing scheme changes), so I'm not doing that for now. + */ + StorePath computeStorePathForText( + std::string_view name, + std::string_view s, + const StorePathSet & references) const; +}; + +} From dde1d863388617b3a63db808c125f274c86a3222 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 18 Mar 2022 15:35:45 +0000 Subject: [PATCH 528/909] Restrict some code to `StoreDirConfig` - part of eval cache - part of derivations - derived path - store path with outputs - serializers --- perl/lib/Nix/Store.xs | 1 + src/libcmd/built-path.cc | 8 ++-- src/libcmd/built-path.hh | 18 ++++----- src/libexpr/eval-cache.cc | 6 +-- src/libexpr/primops/fetchClosure.cc | 1 + src/libexpr/value.hh | 1 - src/libstore/builtins/buildenv.cc | 1 + src/libstore/builtins/buildenv.hh | 1 - src/libstore/common-protocol-impl.hh | 4 +- src/libstore/common-protocol.cc | 28 ++++++------- src/libstore/common-protocol.hh | 8 ++-- src/libstore/derivations.cc | 32 +++++++-------- src/libstore/derivations.hh | 26 +++++++------ src/libstore/derived-path.cc | 39 ++++++++++--------- src/libstore/derived-path.hh | 37 ++++++++++-------- .../length-prefixed-protocol-helper.hh | 22 +++++------ src/libstore/local-store.cc | 1 + src/libstore/misc.cc | 1 + src/libstore/path-with-outputs.cc | 4 +- src/libstore/path-with-outputs.hh | 10 +++-- src/libstore/serve-protocol-impl.hh | 8 ++-- src/libstore/serve-protocol.cc | 4 +- src/libstore/serve-protocol.hh | 12 +++--- src/libstore/store-api.cc | 2 + src/libstore/store-api.hh | 11 ++++-- src/libstore/worker-protocol-impl.hh | 8 ++-- src/libstore/worker-protocol.cc | 24 ++++++------ src/libstore/worker-protocol.hh | 12 +++--- src/nix-build/nix-build.cc | 1 + src/nix-copy-closure/nix-copy-closure.cc | 1 + 30 files changed, 175 insertions(+), 157 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 08f812b31..210d50b6e 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -9,6 +9,7 @@ #undef do_close #include "derivations.hh" +#include "realisation.hh" #include "globals.hh" #include "store-api.hh" #include "util.hh" diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index 9a2dce806..8e2efc7c3 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -80,7 +80,7 @@ SingleDerivedPath SingleBuiltPath::discardOutputPath() const ); } -nlohmann::json BuiltPath::Built::toJSON(const Store & store) const +nlohmann::json BuiltPath::Built::toJSON(const StoreDirConfig & store) const { nlohmann::json res; res["drvPath"] = drvPath->toJSON(store); @@ -90,7 +90,7 @@ nlohmann::json BuiltPath::Built::toJSON(const Store & store) const return res; } -nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const +nlohmann::json SingleBuiltPath::Built::toJSON(const StoreDirConfig & store) const { nlohmann::json res; res["drvPath"] = drvPath->toJSON(store); @@ -100,14 +100,14 @@ nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const return res; } -nlohmann::json SingleBuiltPath::toJSON(const Store & store) const +nlohmann::json SingleBuiltPath::toJSON(const StoreDirConfig & store) const { return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw()); } -nlohmann::json BuiltPath::toJSON(const Store & store) const +nlohmann::json BuiltPath::toJSON(const StoreDirConfig & store) const { return std::visit([&](const auto & buildable) { return buildable.toJSON(store); diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index e677bc810..51918f96c 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -11,9 +11,9 @@ struct SingleBuiltPathBuilt { SingleDerivedPathBuilt discardOutputPath() const; - std::string to_string(const Store & store) const; - static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); - nlohmann::json toJSON(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; + static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); + nlohmann::json toJSON(const StoreDirConfig & store) const; DECLARE_CMP(SingleBuiltPathBuilt); }; @@ -38,8 +38,8 @@ struct SingleBuiltPath : _SingleBuiltPathRaw { SingleDerivedPath discardOutputPath() const; - static SingleBuiltPath parse(const Store & store, std::string_view); - nlohmann::json toJSON(const Store & store) const; + static SingleBuiltPath parse(const StoreDirConfig & store, std::string_view); + nlohmann::json toJSON(const StoreDirConfig & store) const; }; static inline ref staticDrv(StorePath drvPath) @@ -56,9 +56,9 @@ struct BuiltPathBuilt { ref drvPath; std::map outputs; - std::string to_string(const Store & store) const; - static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); - nlohmann::json toJSON(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; + static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); + nlohmann::json toJSON(const StoreDirConfig & store) const; DECLARE_CMP(BuiltPathBuilt); }; @@ -86,7 +86,7 @@ struct BuiltPath : _BuiltPathRaw { StorePathSet outPaths() const; RealisedPath::Set toRealisedPaths(Store & store) const; - nlohmann::json toJSON(const Store & store) const; + nlohmann::json toJSON(const StoreDirConfig & store) const; }; typedef std::vector BuiltPaths; diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 10fc799a9..824f94ba1 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -21,7 +21,7 @@ struct AttrDb { std::atomic_bool failed{false}; - const Store & cfg; + const StoreDirConfig & cfg; struct State { @@ -38,7 +38,7 @@ struct AttrDb SymbolTable & symbols; AttrDb( - const Store & cfg, + const StoreDirConfig & cfg, const Hash & fingerprint, SymbolTable & symbols) : cfg(cfg) @@ -322,7 +322,7 @@ struct AttrDb }; static std::shared_ptr makeAttrDb( - const Store & cfg, + const StoreDirConfig & cfg, const Hash & fingerprint, SymbolTable & symbols) { diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index b86ef6b93..27147a5d1 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -1,5 +1,6 @@ #include "primops.hh" #include "store-api.hh" +#include "realisation.hh" #include "make-content-addressed.hh" #include "url.hh" diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 622e613ea..20f268a3e 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -66,7 +66,6 @@ class Symbol; class PosIdx; struct Pos; class StorePath; -class Store; class EvalState; class XMLWriter; diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index c8911d153..9283251ac 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -1,4 +1,5 @@ #include "buildenv.hh" +#include "derivations.hh" #include #include diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh index 0923c2adb..8bebd390d 100644 --- a/src/libstore/builtins/buildenv.hh +++ b/src/libstore/builtins/buildenv.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "derivations.hh" #include "store-api.hh" namespace nix { diff --git a/src/libstore/common-protocol-impl.hh b/src/libstore/common-protocol-impl.hh index 079c182b8..360882c02 100644 --- a/src/libstore/common-protocol-impl.hh +++ b/src/libstore/common-protocol-impl.hh @@ -16,11 +16,11 @@ namespace nix { /* protocol-agnostic templates */ #define COMMON_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ - TEMPLATE T CommonProto::Serialise< T >::read(const Store & store, CommonProto::ReadConn conn) \ + TEMPLATE T CommonProto::Serialise< T >::read(const StoreDirConfig & store, CommonProto::ReadConn conn) \ { \ return LengthPrefixedProtoHelper::read(store, conn); \ } \ - TEMPLATE void CommonProto::Serialise< T >::write(const Store & store, CommonProto::WriteConn conn, const T & t) \ + TEMPLATE void CommonProto::Serialise< T >::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const T & t) \ { \ LengthPrefixedProtoHelper::write(store, conn, t); \ } diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc index f906814bc..c14a6cfcd 100644 --- a/src/libstore/common-protocol.cc +++ b/src/libstore/common-protocol.cc @@ -14,40 +14,40 @@ namespace nix { /* protocol-agnostic definitions */ -std::string CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +std::string CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { return readString(conn.from); } -void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const std::string & str) +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::string & str) { conn.to << str; } -StorePath CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +StorePath CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { return store.parseStorePath(readString(conn.from)); } -void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const StorePath & storePath) +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const StorePath & storePath) { conn.to << store.printStorePath(storePath); } -ContentAddress CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +ContentAddress CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { return ContentAddress::parse(readString(conn.from)); } -void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const ContentAddress & ca) +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const ContentAddress & ca) { conn.to << renderContentAddress(ca); } -Realisation CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +Realisation CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { std::string rawInput = readString(conn.from); return Realisation::fromJSON( @@ -56,41 +56,41 @@ Realisation CommonProto::Serialise::read(const Store & store, Commo ); } -void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const Realisation & realisation) +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation) { conn.to << realisation.toJSON().dump(); } -DrvOutput CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +DrvOutput CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { return DrvOutput::parse(readString(conn.from)); } -void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) { conn.to << drvOutput.to_string(); } -std::optional CommonProto::Serialise>::read(const Store & store, CommonProto::ReadConn conn) +std::optional CommonProto::Serialise>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { auto s = readString(conn.from); return s == "" ? std::optional {} : store.parseStorePath(s); } -void CommonProto::Serialise>::write(const Store & store, CommonProto::WriteConn conn, const std::optional & storePathOpt) +void CommonProto::Serialise>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional & storePathOpt) { conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); } -std::optional CommonProto::Serialise>::read(const Store & store, CommonProto::ReadConn conn) +std::optional CommonProto::Serialise>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { return ContentAddress::parseOpt(readString(conn.from)); } -void CommonProto::Serialise>::write(const Store & store, CommonProto::WriteConn conn, const std::optional & caOpt) +void CommonProto::Serialise>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional & caOpt) { conn.to << (caOpt ? renderContentAddress(*caOpt) : ""); } diff --git a/src/libstore/common-protocol.hh b/src/libstore/common-protocol.hh index f3f28972a..a878e84c9 100644 --- a/src/libstore/common-protocol.hh +++ b/src/libstore/common-protocol.hh @@ -5,7 +5,7 @@ namespace nix { -class Store; +struct StoreDirConfig; struct Source; // items being serialized @@ -48,7 +48,7 @@ struct CommonProto * infer the type instead of having to write it down explicitly. */ template - static void write(const Store & store, WriteConn conn, const T & t) + static void write(const StoreDirConfig & store, WriteConn conn, const T & t) { CommonProto::Serialise::write(store, conn, t); } @@ -57,8 +57,8 @@ struct CommonProto #define DECLARE_COMMON_SERIALISER(T) \ struct CommonProto::Serialise< T > \ { \ - static T read(const Store & store, CommonProto::ReadConn conn); \ - static void write(const Store & store, CommonProto::WriteConn conn, const T & str); \ + static T read(const StoreDirConfig & store, CommonProto::ReadConn conn); \ + static void write(const StoreDirConfig & store, CommonProto::WriteConn conn, const T & str); \ } template<> diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 1fecd1c97..239232c8e 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -11,7 +11,7 @@ namespace nix { -std::optional DerivationOutput::path(const Store & store, std::string_view drvName, OutputNameView outputName) const +std::optional DerivationOutput::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const { return std::visit(overloaded { [](const DerivationOutput::InputAddressed & doi) -> std::optional { @@ -35,7 +35,7 @@ std::optional DerivationOutput::path(const Store & store, std::string } -StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, OutputNameView outputName) const +StorePath DerivationOutput::CAFixed::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const { return store.makeFixedOutputPathFromCA( outputPathName(drvName, outputName), @@ -215,7 +215,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths) static DerivationOutput parseDerivationOutput( - const Store & store, + const StoreDirConfig & store, std::string_view pathS, std::string_view hashAlgo, std::string_view hashS, const ExperimentalFeatureSettings & xpSettings) { @@ -262,7 +262,7 @@ static DerivationOutput parseDerivationOutput( } static DerivationOutput parseDerivationOutput( - const Store & store, std::istringstream & str, + const StoreDirConfig & store, std::istringstream & str, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings) { expect(str, ","); const auto pathS = parseString(str); @@ -291,7 +291,7 @@ enum struct DerivationATermVersion { }; static DerivedPathMap::ChildNode parseDerivedPathMapNode( - const Store & store, + const StoreDirConfig & store, std::istringstream & str, DerivationATermVersion version) { @@ -338,7 +338,7 @@ static DerivedPathMap::ChildNode parseDerivedPathMapNode( Derivation parseDerivation( - const Store & store, std::string && s, std::string_view name, + const StoreDirConfig & store, std::string && s, std::string_view name, const ExperimentalFeatureSettings & xpSettings) { Derivation drv; @@ -471,7 +471,7 @@ static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIt } -static void unparseDerivedPathMapNode(const Store & store, std::string & s, const DerivedPathMap::ChildNode & node) +static void unparseDerivedPathMapNode(const StoreDirConfig & store, std::string & s, const DerivedPathMap::ChildNode & node) { s += ','; if (node.childMap.empty()) { @@ -512,7 +512,7 @@ static bool hasDynamicDrvDep(const Derivation & drv) } -std::string Derivation::unparse(const Store & store, bool maskOutputs, +std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs, DerivedPathMap::ChildNode::Map * actualInputs) const { std::string s; @@ -846,7 +846,7 @@ std::map staticOutputHashes(Store & store, const Derivation & } -static DerivationOutput readDerivationOutput(Source & in, const Store & store) +static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store) { const auto pathS = readString(in); const auto hashAlgo = readString(in); @@ -863,7 +863,7 @@ StringSet BasicDerivation::outputNames() const return names; } -DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const +DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const StoreDirConfig & store) const { DerivationOutputsAndOptPaths outsAndOptPaths; for (auto & [outputName, output] : outputs) @@ -885,7 +885,7 @@ std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) } -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name) +Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivation & drv, std::string_view name) { drv.name = name; @@ -913,7 +913,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, } -void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv) +void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDerivation & drv) { out << drv.outputs.size(); for (auto & i : drv.outputs) { @@ -1154,7 +1154,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const const Hash impureOutputHash = hashString(htSHA256, "impure"); nlohmann::json DerivationOutput::toJSON( - const Store & store, std::string_view drvName, OutputNameView outputName) const + const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const { nlohmann::json res = nlohmann::json::object(); std::visit(overloaded { @@ -1181,7 +1181,7 @@ nlohmann::json DerivationOutput::toJSON( DerivationOutput DerivationOutput::fromJSON( - const Store & store, std::string_view drvName, OutputNameView outputName, + const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName, const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings) { @@ -1250,7 +1250,7 @@ DerivationOutput DerivationOutput::fromJSON( } -nlohmann::json Derivation::toJSON(const Store & store) const +nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const { nlohmann::json res = nlohmann::json::object(); @@ -1303,7 +1303,7 @@ nlohmann::json Derivation::toJSON(const Store & store) const Derivation Derivation::fromJSON( - const Store & store, + const StoreDirConfig & store, const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings) { diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index fa14e7536..219e8e7d7 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -17,7 +17,7 @@ namespace nix { -class Store; +struct StoreDirConfig; /* Abstract syntax of derivations. */ @@ -55,7 +55,7 @@ struct DerivationOutput * @param drvName The name of the derivation this is an output of, without the `.drv`. * @param outputName The name of this output. */ - StorePath path(const Store & store, std::string_view drvName, OutputNameView outputName) const; + StorePath path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; GENERATE_CMP(CAFixed, me->ca); }; @@ -132,17 +132,17 @@ struct DerivationOutput * the safer interface provided by * BasicDerivation::outputsAndOptPaths */ - std::optional path(const Store & store, std::string_view drvName, OutputNameView outputName) const; + std::optional path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; nlohmann::json toJSON( - const Store & store, + const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; /** * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivationOutput fromJSON( - const Store & store, + const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName, const nlohmann::json & json, @@ -304,7 +304,7 @@ struct BasicDerivation * augmented with knowledge of the Store paths they would be written * into. */ - DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const; + DerivationOutputsAndOptPaths outputsAndOptPaths(const StoreDirConfig & store) const; static std::string_view nameFromPath(const StorePath & storePath); @@ -318,6 +318,8 @@ struct BasicDerivation me->name); }; +class Store; + struct Derivation : BasicDerivation { /** @@ -328,7 +330,7 @@ struct Derivation : BasicDerivation /** * Print a derivation. */ - std::string unparse(const Store & store, bool maskOutputs, + std::string unparse(const StoreDirConfig & store, bool maskOutputs, DerivedPathMap::ChildNode::Map * actualInputs = nullptr) const; /** @@ -365,9 +367,9 @@ struct Derivation : BasicDerivation Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } - nlohmann::json toJSON(const Store & store) const; + nlohmann::json toJSON(const StoreDirConfig & store) const; static Derivation fromJSON( - const Store & store, + const StoreDirConfig & store, const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); @@ -391,7 +393,7 @@ StorePath writeDerivation(Store & store, * Read a derivation from a file. */ Derivation parseDerivation( - const Store & store, + const StoreDirConfig & store, std::string && s, std::string_view name, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); @@ -493,8 +495,8 @@ extern Sync drvHashes; struct Source; struct Sink; -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name); -void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); +Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivation & drv, std::string_view name); +void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDerivation & drv); /** * This creates an opaque and almost certainly unique string diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 47d784deb..3105dbc93 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -1,4 +1,5 @@ #include "derived-path.hh" +#include "derivations.hh" #include "store-api.hh" #include @@ -32,7 +33,7 @@ CMP(SingleDerivedPath, DerivedPathBuilt, outputs) #undef CMP #undef CMP_ONE -nlohmann::json DerivedPath::Opaque::toJSON(const Store & store) const +nlohmann::json DerivedPath::Opaque::toJSON(const StoreDirConfig & store) const { return store.printStorePath(path); } @@ -86,50 +87,50 @@ nlohmann::json DerivedPath::toJSON(Store & store) const }, raw()); } -std::string DerivedPath::Opaque::to_string(const Store & store) const +std::string DerivedPath::Opaque::to_string(const StoreDirConfig & store) const { return store.printStorePath(path); } -std::string SingleDerivedPath::Built::to_string(const Store & store) const +std::string SingleDerivedPath::Built::to_string(const StoreDirConfig & store) const { return drvPath->to_string(store) + "^" + output; } -std::string SingleDerivedPath::Built::to_string_legacy(const Store & store) const +std::string SingleDerivedPath::Built::to_string_legacy(const StoreDirConfig & store) const { return drvPath->to_string(store) + "!" + output; } -std::string DerivedPath::Built::to_string(const Store & store) const +std::string DerivedPath::Built::to_string(const StoreDirConfig & store) const { return drvPath->to_string(store) + '^' + outputs.to_string(); } -std::string DerivedPath::Built::to_string_legacy(const Store & store) const +std::string DerivedPath::Built::to_string_legacy(const StoreDirConfig & store) const { return drvPath->to_string_legacy(store) + "!" + outputs.to_string(); } -std::string SingleDerivedPath::to_string(const Store & store) const +std::string SingleDerivedPath::to_string(const StoreDirConfig & store) const { return std::visit( [&](const auto & req) { return req.to_string(store); }, raw()); } -std::string DerivedPath::to_string(const Store & store) const +std::string DerivedPath::to_string(const StoreDirConfig & store) const { return std::visit( [&](const auto & req) { return req.to_string(store); }, raw()); } -std::string SingleDerivedPath::to_string_legacy(const Store & store) const +std::string SingleDerivedPath::to_string_legacy(const StoreDirConfig & store) const { return std::visit(overloaded { [&](const SingleDerivedPath::Built & req) { return req.to_string_legacy(store); }, @@ -137,7 +138,7 @@ std::string SingleDerivedPath::to_string_legacy(const Store & store) const }, this->raw()); } -std::string DerivedPath::to_string_legacy(const Store & store) const +std::string DerivedPath::to_string_legacy(const StoreDirConfig & store) const { return std::visit(overloaded { [&](const DerivedPath::Built & req) { return req.to_string_legacy(store); }, @@ -146,7 +147,7 @@ std::string DerivedPath::to_string_legacy(const Store & store) const } -DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_view s) +DerivedPath::Opaque DerivedPath::Opaque::parse(const StoreDirConfig & store, std::string_view s) { return {store.parseStorePath(s)}; } @@ -166,7 +167,7 @@ void drvRequireExperiment( } SingleDerivedPath::Built SingleDerivedPath::Built::parse( - const Store & store, ref drv, + const StoreDirConfig & store, ref drv, OutputNameView output, const ExperimentalFeatureSettings & xpSettings) { @@ -178,7 +179,7 @@ SingleDerivedPath::Built SingleDerivedPath::Built::parse( } DerivedPath::Built DerivedPath::Built::parse( - const Store & store, ref drv, + const StoreDirConfig & store, ref drv, OutputNameView outputsS, const ExperimentalFeatureSettings & xpSettings) { @@ -190,7 +191,7 @@ DerivedPath::Built DerivedPath::Built::parse( } static SingleDerivedPath parseWithSingle( - const Store & store, std::string_view s, std::string_view separator, + const StoreDirConfig & store, std::string_view s, std::string_view separator, const ExperimentalFeatureSettings & xpSettings) { size_t n = s.rfind(separator); @@ -207,7 +208,7 @@ static SingleDerivedPath parseWithSingle( } SingleDerivedPath SingleDerivedPath::parse( - const Store & store, + const StoreDirConfig & store, std::string_view s, const ExperimentalFeatureSettings & xpSettings) { @@ -215,7 +216,7 @@ SingleDerivedPath SingleDerivedPath::parse( } SingleDerivedPath SingleDerivedPath::parseLegacy( - const Store & store, + const StoreDirConfig & store, std::string_view s, const ExperimentalFeatureSettings & xpSettings) { @@ -223,7 +224,7 @@ SingleDerivedPath SingleDerivedPath::parseLegacy( } static DerivedPath parseWith( - const Store & store, std::string_view s, std::string_view separator, + const StoreDirConfig & store, std::string_view s, std::string_view separator, const ExperimentalFeatureSettings & xpSettings) { size_t n = s.rfind(separator); @@ -240,7 +241,7 @@ static DerivedPath parseWith( } DerivedPath DerivedPath::parse( - const Store & store, + const StoreDirConfig & store, std::string_view s, const ExperimentalFeatureSettings & xpSettings) { @@ -248,7 +249,7 @@ DerivedPath DerivedPath::parse( } DerivedPath DerivedPath::parseLegacy( - const Store & store, + const StoreDirConfig & store, std::string_view s, const ExperimentalFeatureSettings & xpSettings) { diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 4d7033df2..b12f9734a 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -12,6 +12,9 @@ namespace nix { +struct StoreDirConfig; + +// TODO stop needing this, `toJSON` below should be pure class Store; /** @@ -24,9 +27,9 @@ class Store; struct DerivedPathOpaque { StorePath path; - std::string to_string(const Store & store) const; - static DerivedPathOpaque parse(const Store & store, std::string_view); - nlohmann::json toJSON(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; + static DerivedPathOpaque parse(const StoreDirConfig & store, std::string_view); + nlohmann::json toJSON(const StoreDirConfig & store) const; GENERATE_CMP(DerivedPathOpaque, me->path); }; @@ -59,18 +62,18 @@ struct SingleDerivedPathBuilt { /** * Uses `^` as the separator */ - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; /** * Uses `!` as the separator */ - std::string to_string_legacy(const Store & store) const; + std::string to_string_legacy(const StoreDirConfig & store) const; /** * The caller splits on the separator, so it works for both variants. * * @param xpSettings Stop-gap to avoid globals during unit tests. */ static SingleDerivedPathBuilt parse( - const Store & store, ref drvPath, + const StoreDirConfig & store, ref drvPath, OutputNameView outputs, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; @@ -120,18 +123,18 @@ struct SingleDerivedPath : _SingleDerivedPathRaw { /** * Uses `^` as the separator */ - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; /** * Uses `!` as the separator */ - std::string to_string_legacy(const Store & store) const; + std::string to_string_legacy(const StoreDirConfig & store) const; /** * Uses `^` as the separator * * @param xpSettings Stop-gap to avoid globals during unit tests. */ static SingleDerivedPath parse( - const Store & store, + const StoreDirConfig & store, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** @@ -140,7 +143,7 @@ struct SingleDerivedPath : _SingleDerivedPathRaw { * @param xpSettings Stop-gap to avoid globals during unit tests. */ static SingleDerivedPath parseLegacy( - const Store & store, + const StoreDirConfig & store, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; @@ -182,18 +185,18 @@ struct DerivedPathBuilt { /** * Uses `^` as the separator */ - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; /** * Uses `!` as the separator */ - std::string to_string_legacy(const Store & store) const; + std::string to_string_legacy(const StoreDirConfig & store) const; /** * The caller splits on the separator, so it works for both variants. * * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivedPathBuilt parse( - const Store & store, ref, + const StoreDirConfig & store, ref, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; @@ -242,18 +245,18 @@ struct DerivedPath : _DerivedPathRaw { /** * Uses `^` as the separator */ - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; /** * Uses `!` as the separator */ - std::string to_string_legacy(const Store & store) const; + std::string to_string_legacy(const StoreDirConfig & store) const; /** * Uses `^` as the separator * * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivedPath parse( - const Store & store, + const StoreDirConfig & store, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** @@ -262,7 +265,7 @@ struct DerivedPath : _DerivedPathRaw { * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivedPath parseLegacy( - const Store & store, + const StoreDirConfig & store, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); diff --git a/src/libstore/length-prefixed-protocol-helper.hh b/src/libstore/length-prefixed-protocol-helper.hh index 4061b0cd6..0cf950a47 100644 --- a/src/libstore/length-prefixed-protocol-helper.hh +++ b/src/libstore/length-prefixed-protocol-helper.hh @@ -10,7 +10,7 @@ namespace nix { -class Store; +struct StoreDirConfig; /** * Reusable serialisers for serialization container types in a @@ -44,8 +44,8 @@ struct LengthPrefixedProtoHelper; #define LENGTH_PREFIXED_PROTO_HELPER(Inner, T) \ struct LengthPrefixedProtoHelper< Inner, T > \ { \ - static T read(const Store & store, typename Inner::ReadConn conn); \ - static void write(const Store & store, typename Inner::WriteConn conn, const T & str); \ + static T read(const StoreDirConfig & store, typename Inner::ReadConn conn); \ + static void write(const StoreDirConfig & store, typename Inner::WriteConn conn, const T & str); \ private: \ template using S = typename Inner::template Serialise; \ } @@ -67,7 +67,7 @@ LENGTH_PREFIXED_PROTO_HELPER(Inner, _X); template std::vector LengthPrefixedProtoHelper>::read( - const Store & store, typename Inner::ReadConn conn) + const StoreDirConfig & store, typename Inner::ReadConn conn) { std::vector resSet; auto size = readNum(conn.from); @@ -80,7 +80,7 @@ LengthPrefixedProtoHelper>::read( template void LengthPrefixedProtoHelper>::write( - const Store & store, typename Inner::WriteConn conn, const std::vector & resSet) + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::vector & resSet) { conn.to << resSet.size(); for (auto & key : resSet) { @@ -91,7 +91,7 @@ LengthPrefixedProtoHelper>::write( template std::set LengthPrefixedProtoHelper>::read( - const Store & store, typename Inner::ReadConn conn) + const StoreDirConfig & store, typename Inner::ReadConn conn) { std::set resSet; auto size = readNum(conn.from); @@ -104,7 +104,7 @@ LengthPrefixedProtoHelper>::read( template void LengthPrefixedProtoHelper>::write( - const Store & store, typename Inner::WriteConn conn, const std::set & resSet) + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::set & resSet) { conn.to << resSet.size(); for (auto & key : resSet) { @@ -115,7 +115,7 @@ LengthPrefixedProtoHelper>::write( template std::map LengthPrefixedProtoHelper>::read( - const Store & store, typename Inner::ReadConn conn) + const StoreDirConfig & store, typename Inner::ReadConn conn) { std::map resMap; auto size = readNum(conn.from); @@ -130,7 +130,7 @@ LengthPrefixedProtoHelper>::read( template void LengthPrefixedProtoHelper>::write( - const Store & store, typename Inner::WriteConn conn, const std::map & resMap) + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) { conn.to << resMap.size(); for (auto & i : resMap) { @@ -142,7 +142,7 @@ LengthPrefixedProtoHelper>::write( template std::tuple LengthPrefixedProtoHelper>::read( - const Store & store, typename Inner::ReadConn conn) + const StoreDirConfig & store, typename Inner::ReadConn conn) { return std::tuple { S::read(store, conn)..., @@ -152,7 +152,7 @@ LengthPrefixedProtoHelper>::read( template void LengthPrefixedProtoHelper>::write( - const Store & store, typename Inner::WriteConn conn, const std::tuple & res) + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::tuple & res) { std::apply([&](const Us &... args) { (S::write(store, conn, args), ...); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a5e9426f8..e091683dc 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -4,6 +4,7 @@ #include "pathlocks.hh" #include "worker-protocol.hh" #include "derivations.hh" +#include "realisation.hh" #include "nar-info.hh" #include "references.hh" #include "callback.hh" diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 1035691c7..9f63fbbb5 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -4,6 +4,7 @@ #include "local-store.hh" #include "store-api.hh" #include "thread-pool.hh" +#include "realisation.hh" #include "topo-sort.hh" #include "callback.hh" #include "closure.hh" diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index af6837370..026e37647 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -5,7 +5,7 @@ namespace nix { -std::string StorePathWithOutputs::to_string(const Store & store) const +std::string StorePathWithOutputs::to_string(const StoreDirConfig & store) const { return outputs.empty() ? store.printStorePath(path) @@ -85,7 +85,7 @@ std::pair parsePathWithOutputs(std::string_view s) } -StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs) +StorePathWithOutputs parsePathWithOutputs(const StoreDirConfig & store, std::string_view pathWithOutputs) { auto [path, outputs] = parsePathWithOutputs(pathWithOutputs); return StorePathWithOutputs { store.parseStorePath(path), std::move(outputs) }; diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 57e03252d..5f76a583a 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -6,6 +6,8 @@ namespace nix { +struct StoreDirConfig; + /** * This is a deprecated old type just for use by the old CLI, and older * versions of the RPC protocols. In new code don't use it; you want @@ -19,7 +21,7 @@ struct StorePathWithOutputs StorePath path; std::set outputs; - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; DerivedPath toDerivedPath() const; @@ -32,14 +34,14 @@ std::vector toDerivedPaths(const std::vector) std::pair parsePathWithOutputs(std::string_view s); -class Store; - /** * Split a string specifying a derivation and a set of outputs * (/nix/store/hash-foo!out1,out2,...) into the derivation path * and the outputs. */ -StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs); +StorePathWithOutputs parsePathWithOutputs(const StoreDirConfig & store, std::string_view pathWithOutputs); + +class Store; StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); diff --git a/src/libstore/serve-protocol-impl.hh b/src/libstore/serve-protocol-impl.hh index a3ce81026..6f3b177ac 100644 --- a/src/libstore/serve-protocol-impl.hh +++ b/src/libstore/serve-protocol-impl.hh @@ -16,11 +16,11 @@ namespace nix { /* protocol-agnostic templates */ #define SERVE_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ - TEMPLATE T ServeProto::Serialise< T >::read(const Store & store, ServeProto::ReadConn conn) \ + TEMPLATE T ServeProto::Serialise< T >::read(const StoreDirConfig & store, ServeProto::ReadConn conn) \ { \ return LengthPrefixedProtoHelper::read(store, conn); \ } \ - TEMPLATE void ServeProto::Serialise< T >::write(const Store & store, ServeProto::WriteConn conn, const T & t) \ + TEMPLATE void ServeProto::Serialise< T >::write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t) \ { \ LengthPrefixedProtoHelper::write(store, conn, t); \ } @@ -41,12 +41,12 @@ SERVE_USE_LENGTH_PREFIX_SERIALISER( template struct ServeProto::Serialise { - static T read(const Store & store, ServeProto::ReadConn conn) + static T read(const StoreDirConfig & store, ServeProto::ReadConn conn) { return CommonProto::Serialise::read(store, CommonProto::ReadConn { .from = conn.from }); } - static void write(const Store & store, ServeProto::WriteConn conn, const T & t) + static void write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t) { CommonProto::Serialise::write(store, CommonProto::WriteConn { .to = conn.to }, diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index 97a0ddf0e..e0ac80c4e 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -13,7 +13,7 @@ namespace nix { /* protocol-specific definitions */ -BuildResult ServeProto::Serialise::read(const Store & store, ServeProto::ReadConn conn) +BuildResult ServeProto::Serialise::read(const StoreDirConfig & store, ServeProto::ReadConn conn) { BuildResult status; status.status = (BuildResult::Status) readInt(conn.from); @@ -35,7 +35,7 @@ BuildResult ServeProto::Serialise::read(const Store & store, ServeP return status; } -void ServeProto::Serialise::write(const Store & store, ServeProto::WriteConn conn, const BuildResult & status) +void ServeProto::Serialise::write(const StoreDirConfig & store, ServeProto::WriteConn conn, const BuildResult & status) { conn.to << status.status diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index ba159f6e9..6e9d66e2d 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -13,7 +13,7 @@ namespace nix { #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) -class Store; +struct StoreDirConfig; struct Source; // items being serialised @@ -72,8 +72,8 @@ struct ServeProto // See `worker-protocol.hh` for a longer explanation. #if 0 { - static T read(const Store & store, ReadConn conn); - static void write(const Store & store, WriteConn conn, const T & t); + static T read(const StoreDirConfig & store, ReadConn conn); + static void write(const StoreDirConfig & store, WriteConn conn, const T & t); }; #endif @@ -82,7 +82,7 @@ struct ServeProto * infer the type instead of having to write it down explicitly. */ template - static void write(const Store & store, WriteConn conn, const T & t) + static void write(const StoreDirConfig & store, WriteConn conn, const T & t) { ServeProto::Serialise::write(store, conn, t); } @@ -135,8 +135,8 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op) #define DECLARE_SERVE_SERIALISER(T) \ struct ServeProto::Serialise< T > \ { \ - static T read(const Store & store, ServeProto::ReadConn conn); \ - static void write(const Store & store, ServeProto::WriteConn conn, const T & t); \ + static T read(const StoreDirConfig & store, ServeProto::ReadConn conn); \ + static void write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t); \ }; template<> diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index a681bb6cf..e44376fa2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,6 +1,8 @@ #include "crypto.hh" #include "source-accessor.hh" #include "globals.hh" +#include "derived-path.hh" +#include "realisation.hh" #include "derivations.hh" #include "store-api.hh" #include "util.hh" diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index bee5ec16c..4342445ba 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,8 +1,6 @@ #pragma once ///@file -#include "nar-info.hh" -#include "realisation.hh" #include "path.hh" #include "derived-path.hh" #include "hash.hh" @@ -68,8 +66,13 @@ MakeError(SubstituterDisabled, Error); MakeError(InvalidStoreURI, Error); +struct Realisation; +struct RealisedPath; +struct DrvOutput; + struct BasicDerivation; struct Derivation; + struct SourceAccessor; class NarInfoDiskCache; class Store; @@ -811,7 +814,7 @@ void copyStorePath( */ std::map copyPaths( Store & srcStore, Store & dstStore, - const RealisedPath::Set &, + const std::set &, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); @@ -828,7 +831,7 @@ std::map copyPaths( */ void copyClosure( Store & srcStore, Store & dstStore, - const RealisedPath::Set & paths, + const std::set & paths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh index c043588d6..026cc37bc 100644 --- a/src/libstore/worker-protocol-impl.hh +++ b/src/libstore/worker-protocol-impl.hh @@ -16,11 +16,11 @@ namespace nix { /* protocol-agnostic templates */ #define WORKER_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ - TEMPLATE T WorkerProto::Serialise< T >::read(const Store & store, WorkerProto::ReadConn conn) \ + TEMPLATE T WorkerProto::Serialise< T >::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) \ { \ return LengthPrefixedProtoHelper::read(store, conn); \ } \ - TEMPLATE void WorkerProto::Serialise< T >::write(const Store & store, WorkerProto::WriteConn conn, const T & t) \ + TEMPLATE void WorkerProto::Serialise< T >::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t) \ { \ LengthPrefixedProtoHelper::write(store, conn, t); \ } @@ -41,12 +41,12 @@ WORKER_USE_LENGTH_PREFIX_SERIALISER( template struct WorkerProto::Serialise { - static T read(const Store & store, WorkerProto::ReadConn conn) + static T read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { return CommonProto::Serialise::read(store, CommonProto::ReadConn { .from = conn.from }); } - static void write(const Store & store, WorkerProto::WriteConn conn, const T & t) + static void write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t) { CommonProto::Serialise::write(store, CommonProto::WriteConn { .to = conn.to }, diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index d618b9bd8..4edab7894 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -14,7 +14,7 @@ namespace nix { /* protocol-specific definitions */ -std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) +std::optional WorkerProto::Serialise>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { auto temp = readNum(conn.from); switch (temp) { @@ -29,7 +29,7 @@ std::optional WorkerProto::Serialise>::r } } -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & optTrusted) +void WorkerProto::Serialise>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const std::optional & optTrusted) { if (!optTrusted) conn.to << (uint8_t)0; @@ -48,7 +48,7 @@ void WorkerProto::Serialise>::write(const Store & sto } -DerivedPath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) +DerivedPath WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { auto s = readString(conn.from); if (GET_PROTOCOL_MINOR(conn.version) >= 30) { @@ -58,7 +58,7 @@ DerivedPath WorkerProto::Serialise::read(const Store & store, Worke } } -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const DerivedPath & req) +void WorkerProto::Serialise::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const DerivedPath & req) { if (GET_PROTOCOL_MINOR(conn.version) >= 30) { conn.to << req.to_string_legacy(store); @@ -82,7 +82,7 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto } -KeyedBuildResult WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) +KeyedBuildResult WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { auto path = WorkerProto::Serialise::read(store, conn); auto br = WorkerProto::Serialise::read(store, conn); @@ -92,14 +92,14 @@ KeyedBuildResult WorkerProto::Serialise::read(const Store & st }; } -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const KeyedBuildResult & res) +void WorkerProto::Serialise::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const KeyedBuildResult & res) { WorkerProto::write(store, conn, res.path); WorkerProto::write(store, conn, static_cast(res)); } -BuildResult WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) +BuildResult WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { BuildResult res; res.status = (BuildResult::Status) readInt(conn.from); @@ -121,7 +121,7 @@ BuildResult WorkerProto::Serialise::read(const Store & store, Worke return res; } -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const BuildResult & res) +void WorkerProto::Serialise::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const BuildResult & res) { conn.to << res.status @@ -142,7 +142,7 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto } -ValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) +ValidPathInfo WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) { auto path = WorkerProto::Serialise::read(store, conn); return ValidPathInfo { @@ -151,14 +151,14 @@ ValidPathInfo WorkerProto::Serialise::read(const Store & store, R }; } -void WorkerProto::Serialise::write(const Store & store, WriteConn conn, const ValidPathInfo & pathInfo) +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const ValidPathInfo & pathInfo) { WorkerProto::write(store, conn, pathInfo.path); WorkerProto::write(store, conn, static_cast(pathInfo)); } -UnkeyedValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) +UnkeyedValidPathInfo WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) { auto deriver = readString(conn.from); auto narHash = Hash::parseAny(readString(conn.from), htSHA256); @@ -174,7 +174,7 @@ UnkeyedValidPathInfo WorkerProto::Serialise::read(const St return info; } -void WorkerProto::Serialise::write(const Store & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo) +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo) { conn.to << (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "") diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index dcd54ad16..9b02aa2b5 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -24,7 +24,7 @@ namespace nix { #define STDERR_RESULT 0x52534c54 -class Store; +struct StoreDirConfig; struct Source; // items being serialised @@ -100,8 +100,8 @@ struct WorkerProto // This makes for a quicker debug cycle, as desired. #if 0 { - static T read(const Store & store, ReadConn conn); - static void write(const Store & store, WriteConn conn, const T & t); + static T read(const StoreDirConfig & store, ReadConn conn); + static void write(const StoreDirConfig & store, WriteConn conn, const T & t); }; #endif @@ -110,7 +110,7 @@ struct WorkerProto * infer the type instead of having to write it down explicitly. */ template - static void write(const Store & store, WriteConn conn, const T & t) + static void write(const StoreDirConfig & store, WriteConn conn, const T & t) { WorkerProto::Serialise::write(store, conn, t); } @@ -197,8 +197,8 @@ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op) #define DECLARE_WORKER_SERIALISER(T) \ struct WorkerProto::Serialise< T > \ { \ - static T read(const Store & store, WorkerProto::ReadConn conn); \ - static void write(const Store & store, WorkerProto::WriteConn conn, const T & t); \ + static T read(const StoreDirConfig & store, WorkerProto::ReadConn conn); \ + static void write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t); \ }; template<> diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 60bc08146..c46095a14 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -13,6 +13,7 @@ #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" +#include "realisation.hh" #include "derivations.hh" #include "util.hh" #include "shared.hh" diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index 7f2bb93b6..b64af758f 100644 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -1,4 +1,5 @@ #include "shared.hh" +#include "realisation.hh" #include "store-api.hh" #include "legacy.hh" From 2678b51b31febdc6464935e1680d2272a954c3b5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 5 Nov 2023 12:17:33 -0500 Subject: [PATCH 529/909] Narrower scope for `nativeSystem` I don't think we need a CPP defininition and a header entry, and this way allows constant expression elimination. --- src/libstore/build/local-derivation-goal.cc | 2 ++ src/libutil/error.cc | 2 -- src/libutil/util.hh | 6 ------ 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index dcb7dc6bc..e1794139f 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1620,6 +1620,8 @@ void setupSeccomp() seccomp_release(ctx); }); + constexpr std::string_view nativeSystem = SYSTEM; + if (nativeSystem == "x86_64-linux" && seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) throw SysError("unable to add 32-bit seccomp architecture"); diff --git a/src/libutil/error.cc b/src/libutil/error.cc index dd9612471..1badc1069 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -7,8 +7,6 @@ namespace nix { -const std::string nativeSystem = SYSTEM; - void BaseError::addTrace(std::shared_ptr && e, hintformat hint, bool frame) { err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame }); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index b302d6f45..75683f8fe 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -34,12 +34,6 @@ struct Source; void initLibUtil(); -/** - * The system for which Nix is compiled. - */ -extern const std::string nativeSystem; - - /** * @return an environment variable. */ From ac89bb064aeea85a62b82a6daf0ecca7190a28b7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 25 Oct 2023 00:43:36 -0400 Subject: [PATCH 530/909] Split up `util.{hh,cc}` All OS and IO operations should be moved out, leaving only some misc portable pure functions. This is useful to avoid copious CPP when doing things like Windows and Emscripten ports. Newly exposed functions to break cycles: - `restoreSignals` - `updateWindowSize` --- perl/lib/Nix/Store.xs | 1 - src/libcmd/built-path.hh | 3 + src/libcmd/common-eval-args.cc | 1 - src/libcmd/editor-for.cc | 2 +- src/libcmd/installable-attr-path.hh | 1 - src/libcmd/installables.cc | 1 + src/libcmd/installables.hh | 1 - src/libcmd/markdown.cc | 1 + src/libcmd/repl.cc | 3 + src/libexpr/attr-path.cc | 1 - src/libexpr/eval-cache.cc | 1 + src/libexpr/eval-settings.cc | 1 + src/libexpr/eval-settings.hh | 2 + src/libexpr/eval.cc | 1 + src/libexpr/flake/config.cc | 3 +- src/libexpr/flake/flake.cc | 1 + src/libexpr/get-drvs.cc | 1 - src/libexpr/parser.y | 1 + src/libexpr/primops.cc | 1 + src/libexpr/search-path.cc | 1 - src/libexpr/value-to-json.cc | 2 +- src/libexpr/value-to-xml.cc | 2 +- src/libexpr/value/context.cc | 1 + src/libexpr/value/context.hh | 1 - src/libfetchers/cache.cc | 1 + src/libfetchers/fetch-settings.hh | 1 - src/libfetchers/git.cc | 3 +- src/libfetchers/input-accessor.hh | 2 + src/libfetchers/mercurial.cc | 2 + src/libfetchers/registry.cc | 2 +- src/libmain/common-args.cc | 2 + src/libmain/loggers.cc | 2 +- src/libmain/progress-bar.cc | 2 +- src/libmain/shared.cc | 3 +- src/libmain/shared.hh | 2 +- src/libstore/binary-cache-store.cc | 1 + src/libstore/build/child.cc | 37 + src/libstore/build/child.hh | 11 + src/libstore/build/hook-instance.cc | 2 + src/libstore/build/hook-instance.hh | 1 + src/libstore/build/local-derivation-goal.cc | 3 + src/libstore/build/local-derivation-goal.hh | 1 + src/libstore/build/worker.cc | 1 + src/libstore/common-protocol.cc | 1 - src/libstore/crypto.cc | 1 + src/libstore/derived-path-map.cc | 1 + src/libstore/derived-path-map.hh | 1 + src/libstore/derived-path.hh | 2 +- src/libstore/filetransfer.cc | 3 +- src/libstore/gc.cc | 7 + src/libstore/globals.cc | 13 +- src/libstore/globals.hh | 2 +- src/libstore/local-store.cc | 1 + src/libstore/local-store.hh | 1 - src/libstore/lock.cc | 1 + src/libstore/machines.cc | 1 - src/libstore/nar-info-disk-cache.cc | 1 + src/libstore/optimise-store.cc | 2 +- src/libstore/path-references.cc | 1 - src/libstore/path-references.hh | 1 + src/libstore/pathlocks.cc | 1 + src/libstore/pathlocks.hh | 2 +- src/libstore/profiles.cc | 2 +- src/libstore/remote-store-connection.hh | 3 + src/libstore/serve-protocol.cc | 1 - src/libstore/sqlite.cc | 1 + src/libstore/ssh.cc | 3 + src/libstore/ssh.hh | 3 +- src/libstore/store-api.cc | 2 + src/libstore/tests/machines.cc | 2 + src/libstore/tests/protocol.hh | 3 + src/libstore/uds-remote-store.cc | 1 + src/libstore/worker-protocol.cc | 1 - src/libutil/archive.cc | 3 +- src/libutil/args.cc | 3 + src/libutil/args.hh | 5 +- src/libutil/canon-path.cc | 2 +- src/libutil/cgroup.cc | 1 + src/libutil/compression.cc | 2 +- src/libutil/config.cc | 2 + src/libutil/current-process.cc | 110 + src/libutil/current-process.hh | 34 + src/libutil/environment-variables.cc | 49 + src/libutil/environment-variables.hh | 41 + src/libutil/error.cc | 3 + src/libutil/file-descriptor.cc | 254 +++ src/libutil/file-descriptor.hh | 84 + src/libutil/file-system.cc | 647 ++++++ src/libutil/file-system.hh | 238 +++ src/libutil/filesystem.cc | 162 -- src/libutil/fs-sink.hh | 1 + src/libutil/hash.cc | 1 - src/libutil/hash.hh | 1 + src/libutil/logging.cc | 3 + src/libutil/monitor-fd.hh | 2 + src/libutil/namespaces.cc | 69 +- src/libutil/namespaces.hh | 23 + src/libutil/posix-source-accessor.cc | 1 + src/libutil/processes.cc | 421 ++++ src/libutil/processes.hh | 123 ++ src/libutil/references.cc | 1 - src/libutil/serialise.cc | 2 +- src/libutil/serialise.hh | 1 + src/libutil/signals.cc | 188 ++ src/libutil/signals.hh | 104 + src/libutil/suggestions.cc | 4 +- src/libutil/tarfile.cc | 1 + src/libutil/terminal.cc | 108 + src/libutil/terminal.hh | 38 + src/libutil/tests/logging.cc | 1 - src/libutil/tests/tests.cc | 3 + src/libutil/thread-pool.cc | 2 + src/libutil/thread-pool.hh | 2 +- src/libutil/unix-domain-socket.cc | 100 + src/libutil/unix-domain-socket.hh | 31 + src/libutil/users.cc | 116 ++ src/libutil/users.hh | 58 + src/libutil/util.cc | 1816 +---------------- src/libutil/util.hh | 617 ------ src/nix-build/nix-build.cc | 2 +- src/nix-channel/nix-channel.cc | 2 +- .../nix-collect-garbage.cc | 2 + src/nix-env/nix-env.cc | 2 +- src/nix-env/user-env.cc | 1 - src/nix-instantiate/nix-instantiate.cc | 1 - src/nix-store/dotgraph.cc | 1 - src/nix-store/graphml.cc | 1 - src/nix-store/nix-store.cc | 1 - src/nix/daemon.cc | 3 +- src/nix/develop.cc | 1 - src/nix/doctor.cc | 1 - src/nix/edit.cc | 1 + src/nix/flake.cc | 1 + src/nix/main.cc | 2 + src/nix/run.cc | 1 + src/nix/sigs.cc | 1 + src/nix/upgrade-nix.cc | 1 + src/nix/verify.cc | 1 + 138 files changed, 3028 insertions(+), 2654 deletions(-) create mode 100644 src/libstore/build/child.cc create mode 100644 src/libstore/build/child.hh create mode 100644 src/libutil/current-process.cc create mode 100644 src/libutil/current-process.hh create mode 100644 src/libutil/environment-variables.cc create mode 100644 src/libutil/environment-variables.hh create mode 100644 src/libutil/file-descriptor.cc create mode 100644 src/libutil/file-descriptor.hh create mode 100644 src/libutil/file-system.cc create mode 100644 src/libutil/file-system.hh delete mode 100644 src/libutil/filesystem.cc create mode 100644 src/libutil/processes.cc create mode 100644 src/libutil/processes.hh create mode 100644 src/libutil/signals.cc create mode 100644 src/libutil/signals.hh create mode 100644 src/libutil/terminal.cc create mode 100644 src/libutil/terminal.hh create mode 100644 src/libutil/unix-domain-socket.cc create mode 100644 src/libutil/unix-domain-socket.hh create mode 100644 src/libutil/users.cc create mode 100644 src/libutil/users.hh diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 08f812b31..f89ac4077 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -11,7 +11,6 @@ #include "derivations.hh" #include "globals.hh" #include "store-api.hh" -#include "util.hh" #include "crypto.hh" #include diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index e677bc810..7154cc504 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "derived-path.hh" #include "realisation.hh" diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index e53bc4c01..91fa881b1 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -2,7 +2,6 @@ #include "common-eval-args.hh" #include "shared.hh" #include "filetransfer.hh" -#include "util.hh" #include "eval.hh" #include "fetchers.hh" #include "registry.hh" diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc index a17c6f12a..619d3673f 100644 --- a/src/libcmd/editor-for.cc +++ b/src/libcmd/editor-for.cc @@ -1,5 +1,5 @@ -#include "util.hh" #include "editor-for.hh" +#include "environment-variables.hh" namespace nix { diff --git a/src/libcmd/installable-attr-path.hh b/src/libcmd/installable-attr-path.hh index e9f0c33da..86c2f8219 100644 --- a/src/libcmd/installable-attr-path.hh +++ b/src/libcmd/installable-attr-path.hh @@ -4,7 +4,6 @@ #include "globals.hh" #include "installable-value.hh" #include "outputs-spec.hh" -#include "util.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index bc0b8a988..e7f58556f 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -4,6 +4,7 @@ #include "installable-attr-path.hh" #include "installable-flake.hh" #include "outputs-spec.hh" +#include "users.hh" #include "util.hh" #include "command.hh" #include "attr-path.hh" diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index b0dc0dc02..e087f935c 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "util.hh" #include "path.hh" #include "outputs-spec.hh" #include "derived-path.hh" diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 668a07763..8b3bbc1b5 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,6 +1,7 @@ #include "markdown.hh" #include "util.hh" #include "finally.hh" +#include "terminal.hh" #include #include diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 2e17a29a7..bf5643a5c 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -22,6 +22,7 @@ extern "C" { #include "repl.hh" #include "ansicolor.hh" +#include "signals.hh" #include "shared.hh" #include "eval.hh" #include "eval-cache.hh" @@ -36,6 +37,8 @@ extern "C" { #include "globals.hh" #include "flake/flake.hh" #include "flake/lockfile.hh" +#include "users.hh" +#include "terminal.hh" #include "editor-for.hh" #include "finally.hh" #include "markdown.hh" diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index d12345710..7481a2232 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -1,6 +1,5 @@ #include "attr-path.hh" #include "eval-inline.hh" -#include "util.hh" namespace nix { diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 10fc799a9..6c0e33709 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -1,3 +1,4 @@ +#include "users.hh" #include "eval-cache.hh" #include "sqlite.hh" #include "eval.hh" diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 93b4a5289..444a7d7d6 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -1,3 +1,4 @@ +#include "users.hh" #include "globals.hh" #include "profiles.hh" #include "eval.hh" diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 5473d688e..db2971acb 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -1,4 +1,6 @@ #pragma once +///@file + #include "config.hh" namespace nix { diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d26cde423..dfe81cbf7 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -14,6 +14,7 @@ #include "print.hh" #include "fs-input-accessor.hh" #include "memory-input-accessor.hh" +#include "signals.hh" #include #include diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index e89014862..3c7ed5d8a 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -1,6 +1,7 @@ -#include "flake.hh" +#include "users.hh" #include "globals.hh" #include "fetch-settings.hh" +#include "flake.hh" #include diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index ded132695..54de53e0b 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -1,3 +1,4 @@ +#include "terminal.hh" #include "flake.hh" #include "eval.hh" #include "eval-settings.hh" diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index fe3e6f7ee..d4e946d81 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -1,5 +1,4 @@ #include "get-drvs.hh" -#include "util.hh" #include "eval-inline.hh" #include "derivations.hh" #include "store-api.hh" diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 607795937..b86cef217 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -19,6 +19,7 @@ #include #include "util.hh" +#include "users.hh" #include "nixexpr.hh" #include "eval.hh" diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e3c775d90..36340d0f9 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -10,6 +10,7 @@ #include "path-references.hh" #include "store-api.hh" #include "util.hh" +#include "processes.hh" #include "value-to-json.hh" #include "value-to-xml.hh" #include "primops.hh" diff --git a/src/libexpr/search-path.cc b/src/libexpr/search-path.cc index 180d5f8b1..a25767496 100644 --- a/src/libexpr/search-path.cc +++ b/src/libexpr/search-path.cc @@ -1,5 +1,4 @@ #include "search-path.hh" -#include "util.hh" namespace nix { diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index cbc91f509..74b3ebf13 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -1,7 +1,7 @@ #include "value-to-json.hh" #include "eval-inline.hh" -#include "util.hh" #include "store-api.hh" +#include "signals.hh" #include #include diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index bd7a4ae30..5032115bb 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -1,7 +1,7 @@ #include "value-to-xml.hh" #include "xml-writer.hh" #include "eval-inline.hh" -#include "util.hh" +#include "signals.hh" #include diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index 22361d8fa..6d9633268 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -1,3 +1,4 @@ +#include "util.hh" #include "value/context.hh" #include diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 9f1d59317..51fd30a44 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "util.hh" #include "comparator.hh" #include "derived-path.hh" #include "variant-wrapper.hh" diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 0c8ecac9d..b72a464e8 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -1,4 +1,5 @@ #include "cache.hh" +#include "users.hh" #include "sqlite.hh" #include "sync.hh" #include "store-api.hh" diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index 6108a179c..f095963a8 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -3,7 +3,6 @@ #include "types.hh" #include "config.hh" -#include "util.hh" #include #include diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 72fba0582..cc735996b 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1,11 +1,12 @@ #include "fetchers.hh" +#include "users.hh" #include "cache.hh" #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" #include "url-parts.hh" #include "pathlocks.hh" -#include "util.hh" +#include "processes.hh" #include "git.hh" #include "fetch-settings.hh" diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 5dc05a363..6857ce156 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -1,8 +1,10 @@ #pragma once +///@file #include "source-accessor.hh" #include "ref.hh" #include "types.hh" +#include "file-system.hh" #include "repair-flag.hh" #include "content-address.hh" diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index eda33dfe7..9244acf39 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -1,4 +1,6 @@ #include "fetchers.hh" +#include "processes.hh" +#include "users.hh" #include "cache.hh" #include "globals.hh" #include "tarfile.hh" diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index a0fff9ceb..9c7bc0cfe 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,6 +1,6 @@ #include "registry.hh" #include "tarball.hh" -#include "util.hh" +#include "users.hh" #include "globals.hh" #include "store-api.hh" #include "local-fs-store.hh" diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 205b77808..5b49aaabc 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,7 +1,9 @@ #include "common-args.hh" #include "args/root.hh" #include "globals.hh" +#include "logging.hh" #include "loggers.hh" +#include "util.hh" namespace nix { diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index cda5cb939..9829859de 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -1,6 +1,6 @@ #include "loggers.hh" +#include "environment-variables.hh" #include "progress-bar.hh" -#include "util.hh" namespace nix { diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 45b1fdfd1..a7aee47c3 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -1,5 +1,5 @@ #include "progress-bar.hh" -#include "util.hh" +#include "terminal.hh" #include "sync.hh" #include "store-api.hh" #include "names.hh" diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 9c2ad039a..862ef355b 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -1,10 +1,11 @@ #include "globals.hh" +#include "current-process.hh" #include "shared.hh" #include "store-api.hh" #include "gc-store.hh" -#include "util.hh" #include "loggers.hh" #include "progress-bar.hh" +#include "signals.hh" #include #include diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 3159fe479..c68f6cd83 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "util.hh" +#include "processes.hh" #include "args.hh" #include "args/root.hh" #include "common-args.hh" diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 6a52c4c51..ae483c95e 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -11,6 +11,7 @@ #include "nar-accessor.hh" #include "thread-pool.hh" #include "callback.hh" +#include "signals.hh" #include #include diff --git a/src/libstore/build/child.cc b/src/libstore/build/child.cc new file mode 100644 index 000000000..aa31c3caf --- /dev/null +++ b/src/libstore/build/child.cc @@ -0,0 +1,37 @@ +#include "child.hh" +#include "current-process.hh" +#include "logging.hh" + +#include +#include + +namespace nix { + +void commonChildInit() +{ + logger = makeSimpleLogger(); + + const static std::string pathNullDevice = "/dev/null"; + restoreProcessContext(false); + + /* Put the child in a separate session (and thus a separate + process group) so that it has no controlling terminal (meaning + that e.g. ssh cannot open /dev/tty) and it doesn't receive + terminal signals. */ + if (setsid() == -1) + throw SysError("creating a new session"); + + /* Dup stderr to stdout. */ + if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) + throw SysError("cannot dup stderr into stdout"); + + /* Reroute stdin to /dev/null. */ + int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); + if (fdDevNull == -1) + throw SysError("cannot open '%1%'", pathNullDevice); + if (dup2(fdDevNull, STDIN_FILENO) == -1) + throw SysError("cannot dup null device into stdin"); + close(fdDevNull); +} + +} diff --git a/src/libstore/build/child.hh b/src/libstore/build/child.hh new file mode 100644 index 000000000..3dfc552b9 --- /dev/null +++ b/src/libstore/build/child.hh @@ -0,0 +1,11 @@ +#pragma once +///@file + +namespace nix { + +/** + * Common initialisation performed in child processes. + */ +void commonChildInit(); + +} diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index 337c60bd4..5d045ec3d 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -1,5 +1,7 @@ #include "globals.hh" #include "hook-instance.hh" +#include "file-system.hh" +#include "child.hh" namespace nix { diff --git a/src/libstore/build/hook-instance.hh b/src/libstore/build/hook-instance.hh index d84f62877..61cf534f4 100644 --- a/src/libstore/build/hook-instance.hh +++ b/src/libstore/build/hook-instance.hh @@ -3,6 +3,7 @@ #include "logging.hh" #include "serialise.hh" +#include "processes.hh" namespace nix { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index e1794139f..adb011e30 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -15,7 +15,10 @@ #include "json-utils.hh" #include "cgroup.hh" #include "personality.hh" +#include "current-process.hh" #include "namespaces.hh" +#include "child.hh" +#include "unix-domain-socket.hh" #include #include diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 1cb68a869..88152a645 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -3,6 +3,7 @@ #include "derivation-goal.hh" #include "local-store.hh" +#include "processes.hh" namespace nix { diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 37cb86b91..01914e2d6 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -4,6 +4,7 @@ #include "drv-output-substitution-goal.hh" #include "local-derivation-goal.hh" #include "hook-instance.hh" +#include "signals.hh" #include diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc index f906814bc..68445258f 100644 --- a/src/libstore/common-protocol.cc +++ b/src/libstore/common-protocol.cc @@ -1,5 +1,4 @@ #include "serialise.hh" -#include "util.hh" #include "path-with-outputs.hh" #include "store-api.hh" #include "build-result.hh" diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 1027469c9..1b705733c 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -1,4 +1,5 @@ #include "crypto.hh" +#include "file-system.hh" #include "util.hh" #include "globals.hh" diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index 5982c04b3..4c1ea417a 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -1,4 +1,5 @@ #include "derived-path-map.hh" +#include "util.hh" namespace nix { diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh index 4d72b301e..393cdedf7 100644 --- a/src/libstore/derived-path-map.hh +++ b/src/libstore/derived-path-map.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "derived-path.hh" diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 4d7033df2..6c5dfeed9 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -1,10 +1,10 @@ #pragma once ///@file -#include "util.hh" #include "path.hh" #include "outputs-spec.hh" #include "comparator.hh" +#include "config.hh" #include diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index a283af5a2..dcbec4acd 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1,11 +1,12 @@ #include "filetransfer.hh" -#include "util.hh" +#include "namespaces.hh" #include "globals.hh" #include "store-api.hh" #include "s3.hh" #include "compression.hh" #include "finally.hh" #include "callback.hh" +#include "signals.hh" #if ENABLE_S3 #include diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index fb7895817..8d05ae4bd 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -2,6 +2,13 @@ #include "globals.hh" #include "local-store.hh" #include "finally.hh" +#include "unix-domain-socket.hh" +#include "signals.hh" + +#if !defined(__linux__) +// For shelling out to lsof +# include "processes.hh" +#endif #include #include diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 9c25d9868..cc416a4d6 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,7 +1,8 @@ #include "globals.hh" -#include "util.hh" +#include "current-process.hh" #include "archive.hh" #include "args.hh" +#include "users.hh" #include "abstract-setting-to-json.hh" #include "compute-levels.hh" @@ -17,9 +18,13 @@ #include #ifdef __GLIBC__ -#include -#include -#include +# include +# include +# include +#endif + +#if __APPLE__ +# include "processes.hh" #endif #include "config-impl.hh" diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 12fb48d93..8e034f5a9 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -3,7 +3,7 @@ #include "types.hh" #include "config.hh" -#include "util.hh" +#include "environment-variables.hh" #include "experimental-features.hh" #include diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a5e9426f8..2a3582ad8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -10,6 +10,7 @@ #include "topo-sort.hh" #include "finally.hh" #include "compression.hh" +#include "signals.hh" #include #include diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index fe26a0f27..6d589bee5 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -7,7 +7,6 @@ #include "store-api.hh" #include "indirect-root-store.hh" #include "sync.hh" -#include "util.hh" #include #include diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 165e4969f..87f55ce49 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -1,4 +1,5 @@ #include "lock.hh" +#include "file-system.hh" #include "globals.hh" #include "pathlocks.hh" diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index e87f46980..512115893 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -1,5 +1,4 @@ #include "machines.hh" -#include "util.hh" #include "globals.hh" #include "store-api.hh" diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index cdbcf7e74..e50c15939 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -1,4 +1,5 @@ #include "nar-info-disk-cache.hh" +#include "users.hh" #include "sync.hh" #include "sqlite.hh" #include "globals.hh" diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 23c6a41e4..a4ac413b3 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -1,6 +1,6 @@ -#include "util.hh" #include "local-store.hh" #include "globals.hh" +#include "signals.hh" #include #include diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc index 33cf66ce3..274b596c0 100644 --- a/src/libstore/path-references.cc +++ b/src/libstore/path-references.cc @@ -1,6 +1,5 @@ #include "path-references.hh" #include "hash.hh" -#include "util.hh" #include "archive.hh" #include diff --git a/src/libstore/path-references.hh b/src/libstore/path-references.hh index 7b44e3261..0553003f8 100644 --- a/src/libstore/path-references.hh +++ b/src/libstore/path-references.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "references.hh" #include "path.hh" diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index adc763e6a..2b5b8dfe7 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -1,6 +1,7 @@ #include "pathlocks.hh" #include "util.hh" #include "sync.hh" +#include "signals.hh" #include #include diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 4921df352..7fcfa2e40 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "util.hh" +#include "file-descriptor.hh" namespace nix { diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 239047dd6..e8b88693d 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -1,7 +1,7 @@ #include "profiles.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "util.hh" +#include "users.hh" #include #include diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh index e4a9cacb9..44328b06b 100644 --- a/src/libstore/remote-store-connection.hh +++ b/src/libstore/remote-store-connection.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "remote-store.hh" #include "worker-protocol.hh" #include "pool.hh" diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index 97a0ddf0e..9bfcc279c 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -1,5 +1,4 @@ #include "serialise.hh" -#include "util.hh" #include "path-with-outputs.hh" #include "store-api.hh" #include "build-result.hh" diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 7c8decb74..d7432a305 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -2,6 +2,7 @@ #include "globals.hh" #include "util.hh" #include "url.hh" +#include "signals.hh" #include diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index da32f1b79..03b2f0be9 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,5 +1,8 @@ #include "ssh.hh" #include "finally.hh" +#include "current-process.hh" +#include "environment-variables.hh" +#include "util.hh" namespace nix { diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index 94b952af9..bfcd6f21c 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -1,8 +1,9 @@ #pragma once ///@file -#include "util.hh" #include "sync.hh" +#include "processes.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 646b0ec7d..c9ebb6c14 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -14,6 +14,8 @@ // FIXME this should not be here, see TODO below on // `addMultipleToStore`. #include "worker-protocol.hh" +#include "signals.hh" +#include "users.hh" #include #include diff --git a/src/libstore/tests/machines.cc b/src/libstore/tests/machines.cc index f51052b14..fede328ea 100644 --- a/src/libstore/tests/machines.cc +++ b/src/libstore/tests/machines.cc @@ -1,5 +1,7 @@ #include "machines.hh" #include "globals.hh" +#include "file-system.hh" +#include "util.hh" #include diff --git a/src/libstore/tests/protocol.hh b/src/libstore/tests/protocol.hh index 0378b3e1f..466032a79 100644 --- a/src/libstore/tests/protocol.hh +++ b/src/libstore/tests/protocol.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include #include diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 99589f8b2..226cdf717 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,4 +1,5 @@ #include "uds-remote-store.hh" +#include "unix-domain-socket.hh" #include "worker-protocol.hh" #include diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index d618b9bd8..1d202f8d1 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -1,5 +1,4 @@ #include "serialise.hh" -#include "util.hh" #include "path-with-outputs.hh" #include "store-api.hh" #include "build-result.hh" diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 4ca84d357..465df2073 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -6,9 +6,10 @@ #include // for strcasecmp #include "archive.hh" -#include "util.hh" #include "config.hh" #include "posix-source-accessor.hh" +#include "file-system.hh" +#include "signals.hh" namespace nix { diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 811353c18..0b65519a3 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -1,6 +1,9 @@ #include "args.hh" #include "args/root.hh" #include "hash.hh" +#include "environment-variables.hh" +#include "signals.hh" +#include "users.hh" #include "json-utils.hh" #include diff --git a/src/libutil/args.hh b/src/libutil/args.hh index e3b41313f..45fd678e7 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -2,12 +2,15 @@ ///@file #include +#include #include #include +#include #include -#include "util.hh" +#include "types.hh" +#include "experimental-features.hh" namespace nix { diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc index 040464532..f678fae94 100644 --- a/src/libutil/canon-path.cc +++ b/src/libutil/canon-path.cc @@ -1,5 +1,5 @@ #include "canon-path.hh" -#include "util.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libutil/cgroup.cc b/src/libutil/cgroup.cc index a008481ca..4c2bf31ff 100644 --- a/src/libutil/cgroup.cc +++ b/src/libutil/cgroup.cc @@ -2,6 +2,7 @@ #include "cgroup.hh" #include "util.hh" +#include "file-system.hh" #include "finally.hh" #include diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index ba0847cde..d06f1f87b 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,6 +1,6 @@ #include "compression.hh" +#include "signals.hh" #include "tarfile.hh" -#include "util.hh" #include "finally.hh" #include "logging.hh" diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8e06273ee..0bf36c987 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -2,6 +2,8 @@ #include "args.hh" #include "abstract-setting-to-json.hh" #include "experimental-features.hh" +#include "util.hh" +#include "file-system.hh" #include "config-impl.hh" diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc new file mode 100644 index 000000000..352a6a0fb --- /dev/null +++ b/src/libutil/current-process.cc @@ -0,0 +1,110 @@ +#include "current-process.hh" +#include "namespaces.hh" +#include "util.hh" +#include "finally.hh" +#include "file-system.hh" +#include "processes.hh" +#include "signals.hh" + +#ifdef __APPLE__ +# include +#endif + +#if __linux__ +# include +# include +# include "cgroup.hh" +#endif + +#include + +namespace nix { + +unsigned int getMaxCPU() +{ + #if __linux__ + try { + auto cgroupFS = getCgroupFS(); + if (!cgroupFS) return 0; + + auto cgroups = getCgroups("/proc/self/cgroup"); + auto cgroup = cgroups[""]; + if (cgroup == "") return 0; + + auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; + + auto cpuMax = readFile(cpuFile); + auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); + auto quota = cpuMaxParts[0]; + auto period = cpuMaxParts[1]; + if (quota != "max") + return std::ceil(std::stoi(quota) / std::stof(period)); + } catch (Error &) { ignoreException(lvlDebug); } + #endif + + return 0; +} + + +////////////////////////////////////////////////////////////////////// + + +#if __linux__ +rlim_t savedStackSize = 0; +#endif + +void setStackSize(size_t stackSize) +{ + #if __linux__ + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { + savedStackSize = limit.rlim_cur; + limit.rlim_cur = stackSize; + setrlimit(RLIMIT_STACK, &limit); + } + #endif +} + +void restoreProcessContext(bool restoreMounts) +{ + restoreSignals(); + if (restoreMounts) { + restoreMountNamespace(); + } + + #if __linux__ + if (savedStackSize) { + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0) { + limit.rlim_cur = savedStackSize; + setrlimit(RLIMIT_STACK, &limit); + } + } + #endif +} + + +////////////////////////////////////////////////////////////////////// + + +std::optional getSelfExe() +{ + static auto cached = []() -> std::optional + { + #if __linux__ + return readLink("/proc/self/exe"); + #elif __APPLE__ + char buf[1024]; + uint32_t size = sizeof(buf); + if (_NSGetExecutablePath(buf, &size) == 0) + return buf; + else + return std::nullopt; + #else + return std::nullopt; + #endif + }(); + return cached; +} + +} diff --git a/src/libutil/current-process.hh b/src/libutil/current-process.hh new file mode 100644 index 000000000..826d6fe20 --- /dev/null +++ b/src/libutil/current-process.hh @@ -0,0 +1,34 @@ +#pragma once +///@file + +#include + +#include "types.hh" + +namespace nix { + +/** + * If cgroups are active, attempt to calculate the number of CPUs available. + * If cgroups are unavailable or if cpu.max is set to "max", return 0. + */ +unsigned int getMaxCPU(); + +/** + * Change the stack size. + */ +void setStackSize(size_t stackSize); + +/** + * Restore the original inherited Unix process context (such as signal + * masks, stack size). + + * See startSignalHandlerThread(), saveSignalMask(). + */ +void restoreProcessContext(bool restoreMounts = true); + +/** + * @return the path of the current executable. + */ +std::optional getSelfExe(); + +} diff --git a/src/libutil/environment-variables.cc b/src/libutil/environment-variables.cc new file mode 100644 index 000000000..6618d7872 --- /dev/null +++ b/src/libutil/environment-variables.cc @@ -0,0 +1,49 @@ +#include "util.hh" +#include "environment-variables.hh" + +extern char * * environ __attribute__((weak)); + +namespace nix { + +std::optional getEnv(const std::string & key) +{ + char * value = getenv(key.c_str()); + if (!value) return {}; + return std::string(value); +} + +std::optional getEnvNonEmpty(const std::string & key) { + auto value = getEnv(key); + if (value == "") return {}; + return value; +} + +std::map getEnv() +{ + std::map env; + for (size_t i = 0; environ[i]; ++i) { + auto s = environ[i]; + auto eq = strchr(s, '='); + if (!eq) + // invalid env, just keep going + continue; + env.emplace(std::string(s, eq), std::string(eq + 1)); + } + return env; +} + + +void clearEnv() +{ + for (auto & name : getEnv()) + unsetenv(name.first.c_str()); +} + +void replaceEnv(const std::map & newEnv) +{ + clearEnv(); + for (auto & newEnvVar : newEnv) + setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); +} + +} diff --git a/src/libutil/environment-variables.hh b/src/libutil/environment-variables.hh new file mode 100644 index 000000000..21eb4619b --- /dev/null +++ b/src/libutil/environment-variables.hh @@ -0,0 +1,41 @@ +#pragma once +/** + * @file + * + * Utilities for working with the current process's environment + * variables. + */ + +#include + +#include "types.hh" + +namespace nix { + +/** + * @return an environment variable. + */ +std::optional getEnv(const std::string & key); + +/** + * @return a non empty environment variable. Returns nullopt if the env + * variable is set to "" + */ +std::optional getEnvNonEmpty(const std::string & key); + +/** + * Get the entire environment. + */ +std::map getEnv(); + +/** + * Clear the environment. + */ +void clearEnv(); + +/** + * Replace the entire environment with the given one. + */ +void replaceEnv(const std::map & newEnv); + +} diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 1badc1069..8488e7e21 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -1,4 +1,7 @@ #include "error.hh" +#include "environment-variables.hh" +#include "signals.hh" +#include "terminal.hh" #include #include diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc new file mode 100644 index 000000000..38dd70c8e --- /dev/null +++ b/src/libutil/file-descriptor.cc @@ -0,0 +1,254 @@ +#include "file-system.hh" +#include "signals.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include + +namespace nix { + +std::string readFile(int fd) +{ + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("statting file"); + + return drainFD(fd, true, st.st_size); +} + + +void readFull(int fd, char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + ssize_t res = read(fd, buf, count); + if (res == -1) { + if (errno == EINTR) continue; + throw SysError("reading from file"); + } + if (res == 0) throw EndOfFile("unexpected end-of-file"); + count -= res; + buf += res; + } +} + + +void writeFull(int fd, std::string_view s, bool allowInterrupts) +{ + while (!s.empty()) { + if (allowInterrupts) checkInterrupt(); + ssize_t res = write(fd, s.data(), s.size()); + if (res == -1 && errno != EINTR) + throw SysError("writing to file"); + if (res > 0) + s.remove_prefix(res); + } +} + + +std::string readLine(int fd) +{ + std::string s; + while (1) { + checkInterrupt(); + char ch; + // FIXME: inefficient + ssize_t rd = read(fd, &ch, 1); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading a line"); + } else if (rd == 0) + throw EndOfFile("unexpected EOF reading a line"); + else { + if (ch == '\n') return s; + s += ch; + } + } +} + + +void writeLine(int fd, std::string s) +{ + s += '\n'; + writeFull(fd, s); +} + + +std::string drainFD(int fd, bool block, const size_t reserveSize) +{ + // the parser needs two extra bytes to append terminating characters, other users will + // not care very much about the extra memory. + StringSink sink(reserveSize + 2); + drainFD(fd, sink, block); + return std::move(sink.s); +} + + +void drainFD(int fd, Sink & sink, bool block) +{ + // silence GCC maybe-uninitialized warning in finally + int saved = 0; + + if (!block) { + saved = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) + throw SysError("making file descriptor non-blocking"); + } + + Finally finally([&]() { + if (!block) { + if (fcntl(fd, F_SETFL, saved) == -1) + throw SysError("making file descriptor blocking"); + } + }); + + std::vector buf(64 * 1024); + while (1) { + checkInterrupt(); + ssize_t rd = read(fd, buf.data(), buf.size()); + if (rd == -1) { + if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) + break; + if (errno != EINTR) + throw SysError("reading from file"); + } + else if (rd == 0) break; + else sink({(char *) buf.data(), (size_t) rd}); + } +} + +////////////////////////////////////////////////////////////////////// + +AutoCloseFD::AutoCloseFD() : fd{-1} {} + + +AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} + + +AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} +{ + that.fd = -1; +} + + +AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) +{ + close(); + fd = that.fd; + that.fd = -1; + return *this; +} + + +AutoCloseFD::~AutoCloseFD() +{ + try { + close(); + } catch (...) { + ignoreException(); + } +} + + +int AutoCloseFD::get() const +{ + return fd; +} + + +void AutoCloseFD::close() +{ + if (fd != -1) { + if (::close(fd) == -1) + /* This should never happen. */ + throw SysError("closing file descriptor %1%", fd); + fd = -1; + } +} + +void AutoCloseFD::fsync() +{ + if (fd != -1) { + int result; +#if __APPLE__ + result = ::fcntl(fd, F_FULLFSYNC); +#else + result = ::fsync(fd); +#endif + if (result == -1) + throw SysError("fsync file descriptor %1%", fd); + } +} + + +AutoCloseFD::operator bool() const +{ + return fd != -1; +} + + +int AutoCloseFD::release() +{ + int oldFD = fd; + fd = -1; + return oldFD; +} + + +void Pipe::create() +{ + int fds[2]; +#if HAVE_PIPE2 + if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); +#else + if (pipe(fds) != 0) throw SysError("creating pipe"); + closeOnExec(fds[0]); + closeOnExec(fds[1]); +#endif + readSide = fds[0]; + writeSide = fds[1]; +} + + +void Pipe::close() +{ + readSide.close(); + writeSide.close(); +} + +////////////////////////////////////////////////////////////////////// + +void closeMostFDs(const std::set & exceptions) +{ +#if __linux__ + try { + for (auto & s : readDirectory("/proc/self/fd")) { + auto fd = std::stoi(s.name); + if (!exceptions.count(fd)) { + debug("closing leaked FD %d", fd); + close(fd); + } + } + return; + } catch (SysError &) { + } +#endif + + int maxFD = 0; + maxFD = sysconf(_SC_OPEN_MAX); + for (int fd = 0; fd < maxFD; ++fd) + if (!exceptions.count(fd)) + close(fd); /* ignore result */ +} + + +void closeOnExec(int fd) +{ + int prev; + if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || + fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) + throw SysError("setting close-on-exec flag"); +} + +} diff --git a/src/libutil/file-descriptor.hh b/src/libutil/file-descriptor.hh new file mode 100644 index 000000000..80ec86135 --- /dev/null +++ b/src/libutil/file-descriptor.hh @@ -0,0 +1,84 @@ +#pragma once +///@file + +#include "types.hh" +#include "error.hh" + +namespace nix { + +struct Sink; +struct Source; + +/** + * Read the contents of a resource into a string. + */ +std::string readFile(int fd); + +/** + * Wrappers arount read()/write() that read/write exactly the + * requested number of bytes. + */ +void readFull(int fd, char * buf, size_t count); + +void writeFull(int fd, std::string_view s, bool allowInterrupts = true); + +/** + * Read a line from a file descriptor. + */ +std::string readLine(int fd); + +/** + * Write a line to a file descriptor. + */ +void writeLine(int fd, std::string s); + +/** + * Read a file descriptor until EOF occurs. + */ +std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); + +void drainFD(int fd, Sink & sink, bool block = true); + +/** + * Automatic cleanup of resources. + */ +class AutoCloseFD +{ + int fd; +public: + AutoCloseFD(); + AutoCloseFD(int fd); + AutoCloseFD(const AutoCloseFD & fd) = delete; + AutoCloseFD(AutoCloseFD&& fd); + ~AutoCloseFD(); + AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; + AutoCloseFD& operator =(AutoCloseFD&& fd); + int get() const; + explicit operator bool() const; + int release(); + void close(); + void fsync(); +}; + +class Pipe +{ +public: + AutoCloseFD readSide, writeSide; + void create(); + void close(); +}; + +/** + * Close all file descriptors except those listed in the given set. + * Good practice in child processes. + */ +void closeMostFDs(const std::set & exceptions); + +/** + * Set the close-on-exec flag for the given file descriptor. + */ +void closeOnExec(int fd); + +MakeError(EndOfFile, Error); + +} diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc new file mode 100644 index 000000000..c96effff9 --- /dev/null +++ b/src/libutil/file-system.cc @@ -0,0 +1,647 @@ +#include "environment-variables.hh" +#include "file-system.hh" +#include "signals.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace nix { + +Path absPath(Path path, std::optional dir, bool resolveSymlinks) +{ + if (path[0] != '/') { + if (!dir) { +#ifdef __GNU__ + /* GNU (aka. GNU/Hurd) doesn't have any limitation on path + lengths and doesn't define `PATH_MAX'. */ + char *buf = getcwd(NULL, 0); + if (buf == NULL) +#else + char buf[PATH_MAX]; + if (!getcwd(buf, sizeof(buf))) +#endif + throw SysError("cannot get cwd"); + path = concatStrings(buf, "/", path); +#ifdef __GNU__ + free(buf); +#endif + } else + path = concatStrings(*dir, "/", path); + } + return canonPath(path, resolveSymlinks); +} + + +Path canonPath(PathView path, bool resolveSymlinks) +{ + assert(path != ""); + + std::string s; + s.reserve(256); + + if (path[0] != '/') + throw Error("not an absolute path: '%1%'", path); + + std::string temp; + + /* Count the number of times we follow a symlink and stop at some + arbitrary (but high) limit to prevent infinite loops. */ + unsigned int followCount = 0, maxFollow = 1024; + + while (1) { + + /* Skip slashes. */ + while (!path.empty() && path[0] == '/') path.remove_prefix(1); + if (path.empty()) break; + + /* Ignore `.'. */ + if (path == "." || path.substr(0, 2) == "./") + path.remove_prefix(1); + + /* If `..', delete the last component. */ + else if (path == ".." || path.substr(0, 3) == "../") + { + if (!s.empty()) s.erase(s.rfind('/')); + path.remove_prefix(2); + } + + /* Normal component; copy it. */ + else { + s += '/'; + if (const auto slash = path.find('/'); slash == std::string::npos) { + s += path; + path = {}; + } else { + s += path.substr(0, slash); + path = path.substr(slash); + } + + /* If s points to a symlink, resolve it and continue from there */ + if (resolveSymlinks && isLink(s)) { + if (++followCount >= maxFollow) + throw Error("infinite symlink recursion in path '%1%'", path); + temp = concatStrings(readLink(s), path); + path = temp; + if (!temp.empty() && temp[0] == '/') { + s.clear(); /* restart for symlinks pointing to absolute path */ + } else { + s = dirOf(s); + if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = / + s.clear(); + } + } + } + } + } + + return s.empty() ? "/" : std::move(s); +} + + +Path dirOf(const PathView path) +{ + Path::size_type pos = path.rfind('/'); + if (pos == std::string::npos) + return "."; + return pos == 0 ? "/" : Path(path, 0, pos); +} + + +std::string_view baseNameOf(std::string_view path) +{ + if (path.empty()) + return ""; + + auto last = path.size() - 1; + if (path[last] == '/' && last > 0) + last -= 1; + + auto pos = path.rfind('/', last); + if (pos == std::string::npos) + pos = 0; + else + pos += 1; + + return path.substr(pos, last - pos + 1); +} + + +bool isInDir(std::string_view path, std::string_view dir) +{ + return path.substr(0, 1) == "/" + && path.substr(0, dir.size()) == dir + && path.size() >= dir.size() + 2 + && path[dir.size()] == '/'; +} + + +bool isDirOrInDir(std::string_view path, std::string_view dir) +{ + return path == dir || isInDir(path, dir); +} + + +struct stat stat(const Path & path) +{ + struct stat st; + if (stat(path.c_str(), &st)) + throw SysError("getting status of '%1%'", path); + return st; +} + + +struct stat lstat(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError("getting status of '%1%'", path); + return st; +} + + +bool pathExists(const Path & path) +{ + int res; + struct stat st; + res = lstat(path.c_str(), &st); + if (!res) return true; + if (errno != ENOENT && errno != ENOTDIR) + throw SysError("getting status of %1%", path); + return false; +} + +bool pathAccessible(const Path & path) +{ + try { + return pathExists(path); + } catch (SysError & e) { + // swallow EPERM + if (e.errNo == EPERM) return false; + throw; + } +} + + +Path readLink(const Path & path) +{ + checkInterrupt(); + std::vector buf; + for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { + buf.resize(bufSize); + ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); + if (rlSize == -1) + if (errno == EINVAL) + throw Error("'%1%' is not a symlink", path); + else + throw SysError("reading symbolic link '%1%'", path); + else if (rlSize < bufSize) + return std::string(buf.data(), rlSize); + } +} + + +bool isLink(const Path & path) +{ + struct stat st = lstat(path); + return S_ISLNK(st.st_mode); +} + + +DirEntries readDirectory(DIR *dir, const Path & path) +{ + DirEntries entries; + entries.reserve(64); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { /* sic */ + checkInterrupt(); + std::string name = dirent->d_name; + if (name == "." || name == "..") continue; + entries.emplace_back(name, dirent->d_ino, +#ifdef HAVE_STRUCT_DIRENT_D_TYPE + dirent->d_type +#else + DT_UNKNOWN +#endif + ); + } + if (errno) throw SysError("reading directory '%1%'", path); + + return entries; +} + +DirEntries readDirectory(const Path & path) +{ + AutoCloseDir dir(opendir(path.c_str())); + if (!dir) throw SysError("opening directory '%1%'", path); + + return readDirectory(dir.get(), path); +} + + +unsigned char getFileType(const Path & path) +{ + struct stat st = lstat(path); + if (S_ISDIR(st.st_mode)) return DT_DIR; + if (S_ISLNK(st.st_mode)) return DT_LNK; + if (S_ISREG(st.st_mode)) return DT_REG; + return DT_UNKNOWN; +} + + +std::string readFile(const Path & path) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%1%'", path); + return readFile(fd.get()); +} + + +void readFile(const Path & path, Sink & sink) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%s'", path); + drainFD(fd.get(), sink); +} + + +void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + if (!fd) + throw SysError("opening file '%1%'", path); + try { + writeFull(fd.get(), s); + } catch (Error & e) { + e.addTrace({}, "writing file '%1%'", path); + throw; + } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); +} + + +void writeFile(const Path & path, Source & source, mode_t mode, bool sync) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + if (!fd) + throw SysError("opening file '%1%'", path); + + std::vector buf(64 * 1024); + + try { + while (true) { + try { + auto n = source.read(buf.data(), buf.size()); + writeFull(fd.get(), {buf.data(), n}); + } catch (EndOfFile &) { break; } + } + } catch (Error & e) { + e.addTrace({}, "writing file '%1%'", path); + throw; + } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); +} + +void syncParent(const Path & path) +{ + AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0); + if (!fd) + throw SysError("opening file '%1%'", path); + fd.fsync(); +} + + +static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) +{ + checkInterrupt(); + + std::string name(baseNameOf(path)); + + struct stat st; + if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { + if (errno == ENOENT) return; + throw SysError("getting status of '%1%'", path); + } + + if (!S_ISDIR(st.st_mode)) { + /* We are about to delete a file. Will it likely free space? */ + + switch (st.st_nlink) { + /* Yes: last link. */ + case 1: + bytesFreed += st.st_size; + break; + /* Maybe: yes, if 'auto-optimise-store' or manual optimisation + was performed. Instead of checking for real let's assume + it's an optimised file and space will be freed. + + In worst case we will double count on freed space for files + with exactly two hardlinks for unoptimised packages. + */ + case 2: + bytesFreed += st.st_size; + break; + /* No: 3+ links. */ + default: + break; + } + } + + if (S_ISDIR(st.st_mode)) { + /* Make the directory accessible. */ + const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; + if ((st.st_mode & PERM_MASK) != PERM_MASK) { + if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) + throw SysError("chmod '%1%'", path); + } + + int fd = openat(parentfd, path.c_str(), O_RDONLY); + if (fd == -1) + throw SysError("opening directory '%1%'", path); + AutoCloseDir dir(fdopendir(fd)); + if (!dir) + throw SysError("opening directory '%1%'", path); + for (auto & i : readDirectory(dir.get(), path)) + _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); + } + + int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; + if (unlinkat(parentfd, name.c_str(), flags) == -1) { + if (errno == ENOENT) return; + throw SysError("cannot unlink '%1%'", path); + } +} + +static void _deletePath(const Path & path, uint64_t & bytesFreed) +{ + Path dir = dirOf(path); + if (dir == "") + dir = "/"; + + AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)}; + if (!dirfd) { + if (errno == ENOENT) return; + throw SysError("opening directory '%1%'", path); + } + + _deletePath(dirfd.get(), path, bytesFreed); +} + + +void deletePath(const Path & path) +{ + uint64_t dummy; + deletePath(path, dummy); +} + + +Paths createDirs(const Path & path) +{ + Paths created; + if (path == "/") return created; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + created = createDirs(dirOf(path)); + if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) + throw SysError("creating directory '%1%'", path); + st = lstat(path); + created.push_back(path); + } + + if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) + throw SysError("statting symlink '%1%'", path); + + if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); + + return created; +} + + +void deletePath(const Path & path, uint64_t & bytesFreed) +{ + //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); + bytesFreed = 0; + _deletePath(path, bytesFreed); +} + + +////////////////////////////////////////////////////////////////////// + +AutoDelete::AutoDelete() : del{false} {} + +AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p) +{ + del = true; + this->recursive = recursive; +} + +AutoDelete::~AutoDelete() +{ + try { + if (del) { + if (recursive) + deletePath(path); + else { + if (remove(path.c_str()) == -1) + throw SysError("cannot unlink '%1%'", path); + } + } + } catch (...) { + ignoreException(); + } +} + +void AutoDelete::cancel() +{ + del = false; +} + +void AutoDelete::reset(const Path & p, bool recursive) { + path = p; + this->recursive = recursive; + del = true; +} + +////////////////////////////////////////////////////////////////////// + +////////////////////////////////////////////////////////////////////// + +static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, + std::atomic & counter) +{ + tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); + if (includePid) + return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++); + else + return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++); +} + +Path createTempDir(const Path & tmpRoot, const Path & prefix, + bool includePid, bool useGlobalCounter, mode_t mode) +{ + static std::atomic globalCounter = 0; + std::atomic localCounter = 0; + auto & counter(useGlobalCounter ? globalCounter : localCounter); + + while (1) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str(), mode) == 0) { +#if __FreeBSD__ + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + throw SysError("setting group of directory '%1%'", tmpDir); +#endif + return tmpDir; + } + if (errno != EEXIST) + throw SysError("creating directory '%1%'", tmpDir); + } +} + + +std::pair createTempFile(const Path & prefix) +{ + Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + // FIXME: use O_TMPFILE. + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + closeOnExec(fd.get()); + return {std::move(fd), tmpl}; +} + +void createSymlink(const Path & target, const Path & link) +{ + if (symlink(target.c_str(), link.c_str())) + throw SysError("creating symlink from '%1%' to '%2%'", link, target); +} + +void replaceSymlink(const Path & target, const Path & link) +{ + for (unsigned int n = 0; true; n++) { + Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); + + try { + createSymlink(target, tmp); + } catch (SysError & e) { + if (e.errNo == EEXIST) continue; + throw; + } + + renameFile(tmp, link); + + break; + } +} + +void setWriteTime(const fs::path & p, const struct stat & st) +{ + struct timeval times[2]; + times[0] = { + .tv_sec = st.st_atime, + .tv_usec = 0, + }; + times[1] = { + .tv_sec = st.st_mtime, + .tv_usec = 0, + }; + if (lutimes(p.c_str(), times) != 0) + throw SysError("changing modification time of '%s'", p); +} + +void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) +{ + // TODO: Rewrite the `is_*` to use `symlink_status()` + auto statOfFrom = lstat(from.path().c_str()); + auto fromStatus = from.symlink_status(); + + // Mark the directory as writable so that we can delete its children + if (andDelete && fs::is_directory(fromStatus)) { + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + } + + + if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { + fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); + } else if (fs::is_directory(fromStatus)) { + fs::create_directory(to); + for (auto & entry : fs::directory_iterator(from.path())) { + copy(entry, to / entry.path().filename(), andDelete); + } + } else { + throw Error("file '%s' has an unsupported type", from.path()); + } + + setWriteTime(to, statOfFrom); + if (andDelete) { + if (!fs::is_symlink(fromStatus)) + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + fs::remove(from.path()); + } +} + +void renameFile(const Path & oldName, const Path & newName) +{ + fs::rename(oldName, newName); +} + +void moveFile(const Path & oldName, const Path & newName) +{ + try { + renameFile(oldName, newName); + } catch (fs::filesystem_error & e) { + auto oldPath = fs::path(oldName); + auto newPath = fs::path(newName); + // For the move to be as atomic as possible, copy to a temporary + // directory + fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); + Finally removeTemp = [&]() { fs::remove(temp); }; + auto tempCopyTarget = temp / "copy-target"; + if (e.code().value() == EXDEV) { + fs::remove(newPath); + warn("Can’t rename %s as %s, copying instead", oldName, newName); + copy(fs::directory_entry(oldPath), tempCopyTarget, true); + renameFile(tempCopyTarget, newPath); + } + } +} + +////////////////////////////////////////////////////////////////////// + +} diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh new file mode 100644 index 000000000..4637507b3 --- /dev/null +++ b/src/libutil/file-system.hh @@ -0,0 +1,238 @@ +#pragma once +/** + * @file + * + * Utiltities for working with the file sytem and file paths. + */ + +#include "types.hh" +#include "error.hh" +#include "logging.hh" +#include "file-descriptor.hh" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +#ifndef HAVE_STRUCT_DIRENT_D_TYPE +#define DT_UNKNOWN 0 +#define DT_REG 1 +#define DT_LNK 2 +#define DT_DIR 3 +#endif + +namespace nix { + +struct Sink; +struct Source; + +/** + * @return An absolutized path, resolving paths relative to the + * specified directory, or the current directory otherwise. The path + * is also canonicalised. + */ +Path absPath(Path path, + std::optional dir = {}, + bool resolveSymlinks = false); + +/** + * Canonicalise a path by removing all `.` or `..` components and + * double or trailing slashes. Optionally resolves all symlink + * components such that each component of the resulting path is *not* + * a symbolic link. + */ +Path canonPath(PathView path, bool resolveSymlinks = false); + +/** + * @return The directory part of the given canonical path, i.e., + * everything before the final `/`. If the path is the root or an + * immediate child thereof (e.g., `/foo`), this means `/` + * is returned. + */ +Path dirOf(const PathView path); + +/** + * @return the base name of the given canonical path, i.e., everything + * following the final `/` (trailing slashes are removed). + */ +std::string_view baseNameOf(std::string_view path); + +/** + * Check whether 'path' is a descendant of 'dir'. Both paths must be + * canonicalized. + */ +bool isInDir(std::string_view path, std::string_view dir); + +/** + * Check whether 'path' is equal to 'dir' or a descendant of + * 'dir'. Both paths must be canonicalized. + */ +bool isDirOrInDir(std::string_view path, std::string_view dir); + +/** + * Get status of `path`. + */ +struct stat stat(const Path & path); +struct stat lstat(const Path & path); + +/** + * @return true iff the given path exists. + */ +bool pathExists(const Path & path); + +/** + * A version of pathExists that returns false on a permission error. + * Useful for inferring default paths across directories that might not + * be readable. + * @return true iff the given path can be accessed and exists + */ +bool pathAccessible(const Path & path); + +/** + * Read the contents (target) of a symbolic link. The result is not + * in any way canonicalised. + */ +Path readLink(const Path & path); + +bool isLink(const Path & path); + +/** + * Read the contents of a directory. The entries `.` and `..` are + * removed. + */ +struct DirEntry +{ + std::string name; + ino_t ino; + /** + * one of DT_* + */ + unsigned char type; + DirEntry(std::string name, ino_t ino, unsigned char type) + : name(std::move(name)), ino(ino), type(type) { } +}; + +typedef std::vector DirEntries; + +DirEntries readDirectory(const Path & path); + +unsigned char getFileType(const Path & path); + +/** + * Read the contents of a file into a string. + */ +std::string readFile(const Path & path); +void readFile(const Path & path, Sink & sink); + +/** + * Write a string to a file. + */ +void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); + +void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); + +/** + * Flush a file's parent directory to disk + */ +void syncParent(const Path & path); + +/** + * Delete a path; i.e., in the case of a directory, it is deleted + * recursively. It's not an error if the path does not exist. The + * second variant returns the number of bytes and blocks freed. + */ +void deletePath(const Path & path); + +void deletePath(const Path & path, uint64_t & bytesFreed); + +/** + * Create a directory and all its parents, if necessary. Returns the + * list of created directories, in order of creation. + */ +Paths createDirs(const Path & path); +inline Paths createDirs(PathView path) +{ + return createDirs(Path(path)); +} + +/** + * Create a symlink. + */ +void createSymlink(const Path & target, const Path & link); + +/** + * Atomically create or replace a symlink. + */ +void replaceSymlink(const Path & target, const Path & link); + +void renameFile(const Path & src, const Path & dst); + +/** + * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` + * are on a different filesystem. + * + * Beware that this might not be atomic because of the copy that happens behind + * the scenes + */ +void moveFile(const Path & src, const Path & dst); + + +/** + * Automatic cleanup of resources. + */ +class AutoDelete +{ + Path path; + bool del; + bool recursive; +public: + AutoDelete(); + AutoDelete(const Path & p, bool recursive = true); + ~AutoDelete(); + void cancel(); + void reset(const Path & p, bool recursive = true); + operator Path() const { return path; } + operator PathView() const { return path; } +}; + + +struct DIRDeleter +{ + void operator()(DIR * dir) const { + closedir(dir); + } +}; + +typedef std::unique_ptr AutoCloseDir; + + +/** + * Create a temporary directory. + */ +Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); + +/** + * Create a temporary file, returning a file handle and its path. + */ +std::pair createTempFile(const Path & prefix = "nix"); + + +/** + * Used in various places. + */ +typedef std::function PathFilter; + +extern PathFilter defaultPathFilter; + +} diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc deleted file mode 100644 index 11cc0c0e7..000000000 --- a/src/libutil/filesystem.cc +++ /dev/null @@ -1,162 +0,0 @@ -#include -#include -#include - -#include "finally.hh" -#include "util.hh" -#include "types.hh" - -namespace fs = std::filesystem; - -namespace nix { - -static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, - std::atomic & counter) -{ - tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); - if (includePid) - return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++); - else - return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++); -} - -Path createTempDir(const Path & tmpRoot, const Path & prefix, - bool includePid, bool useGlobalCounter, mode_t mode) -{ - static std::atomic globalCounter = 0; - std::atomic localCounter = 0; - auto & counter(useGlobalCounter ? globalCounter : localCounter); - - while (1) { - checkInterrupt(); - Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); - if (mkdir(tmpDir.c_str(), mode) == 0) { -#if __FreeBSD__ - /* Explicitly set the group of the directory. This is to - work around around problems caused by BSD's group - ownership semantics (directories inherit the group of - the parent). For instance, the group of /tmp on - FreeBSD is "wheel", so all directories created in /tmp - will be owned by "wheel"; but if the user is not in - "wheel", then "tar" will fail to unpack archives that - have the setgid bit set on directories. */ - if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) - throw SysError("setting group of directory '%1%'", tmpDir); -#endif - return tmpDir; - } - if (errno != EEXIST) - throw SysError("creating directory '%1%'", tmpDir); - } -} - - -std::pair createTempFile(const Path & prefix) -{ - Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); - // Strictly speaking, this is UB, but who cares... - // FIXME: use O_TMPFILE. - AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); - if (!fd) - throw SysError("creating temporary file '%s'", tmpl); - closeOnExec(fd.get()); - return {std::move(fd), tmpl}; -} - -void createSymlink(const Path & target, const Path & link) -{ - if (symlink(target.c_str(), link.c_str())) - throw SysError("creating symlink from '%1%' to '%2%'", link, target); -} - -void replaceSymlink(const Path & target, const Path & link) -{ - for (unsigned int n = 0; true; n++) { - Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); - - try { - createSymlink(target, tmp); - } catch (SysError & e) { - if (e.errNo == EEXIST) continue; - throw; - } - - renameFile(tmp, link); - - break; - } -} - -void setWriteTime(const fs::path & p, const struct stat & st) -{ - struct timeval times[2]; - times[0] = { - .tv_sec = st.st_atime, - .tv_usec = 0, - }; - times[1] = { - .tv_sec = st.st_mtime, - .tv_usec = 0, - }; - if (lutimes(p.c_str(), times) != 0) - throw SysError("changing modification time of '%s'", p); -} - -void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) -{ - // TODO: Rewrite the `is_*` to use `symlink_status()` - auto statOfFrom = lstat(from.path().c_str()); - auto fromStatus = from.symlink_status(); - - // Mark the directory as writable so that we can delete its children - if (andDelete && fs::is_directory(fromStatus)) { - fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); - } - - - if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { - fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); - } else if (fs::is_directory(fromStatus)) { - fs::create_directory(to); - for (auto & entry : fs::directory_iterator(from.path())) { - copy(entry, to / entry.path().filename(), andDelete); - } - } else { - throw Error("file '%s' has an unsupported type", from.path()); - } - - setWriteTime(to, statOfFrom); - if (andDelete) { - if (!fs::is_symlink(fromStatus)) - fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); - fs::remove(from.path()); - } -} - -void renameFile(const Path & oldName, const Path & newName) -{ - fs::rename(oldName, newName); -} - -void moveFile(const Path & oldName, const Path & newName) -{ - try { - renameFile(oldName, newName); - } catch (fs::filesystem_error & e) { - auto oldPath = fs::path(oldName); - auto newPath = fs::path(newName); - // For the move to be as atomic as possible, copy to a temporary - // directory - fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); - Finally removeTemp = [&]() { fs::remove(temp); }; - auto tempCopyTarget = temp / "copy-target"; - if (e.code().value() == EXDEV) { - fs::remove(newPath); - warn("Can’t rename %s as %s, copying instead", oldName, newName); - copy(fs::directory_entry(oldPath), tempCopyTarget, true); - renameFile(tempCopyTarget, newPath); - } - } -} - -} diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index c22edd390..bf54b7301 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -4,6 +4,7 @@ #include "types.hh" #include "serialise.hh" #include "source-accessor.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index e297c245b..144f7ae7e 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -9,7 +9,6 @@ #include "hash.hh" #include "archive.hh" #include "split.hh" -#include "util.hh" #include #include diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index cab3e6eca..6ade6555c 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "serialise.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 9d7a141b3..60b0865bf 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -1,4 +1,7 @@ #include "logging.hh" +#include "file-descriptor.hh" +#include "environment-variables.hh" +#include "terminal.hh" #include "util.hh" #include "config.hh" diff --git a/src/libutil/monitor-fd.hh b/src/libutil/monitor-fd.hh index 86d0115fc..228fb13f8 100644 --- a/src/libutil/monitor-fd.hh +++ b/src/libutil/monitor-fd.hh @@ -10,6 +10,8 @@ #include #include +#include "signals.hh" + namespace nix { diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index f66accb10..a789b321e 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -1,13 +1,22 @@ -#if __linux__ - -#include "namespaces.hh" +#include "current-process.hh" #include "util.hh" #include "finally.hh" +#include "file-system.hh" +#include "processes.hh" +#include "signals.hh" + +#if __linux__ +# include +# include +# include "cgroup.hh" +#endif #include namespace nix { +#if __linux__ + bool userNamespacesSupported() { static auto res = [&]() -> bool @@ -92,6 +101,60 @@ bool mountAndPidNamespacesSupported() return res; } +#endif + + +////////////////////////////////////////////////////////////////////// + +#if __linux__ +static AutoCloseFD fdSavedMountNamespace; +static AutoCloseFD fdSavedRoot; +#endif + +void saveMountNamespace() +{ +#if __linux__ + static std::once_flag done; + std::call_once(done, []() { + fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); + if (!fdSavedMountNamespace) + throw SysError("saving parent mount namespace"); + + fdSavedRoot = open("/proc/self/root", O_RDONLY); + }); +#endif } +void restoreMountNamespace() +{ +#if __linux__ + try { + auto savedCwd = absPath("."); + + if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + throw SysError("restoring parent mount namespace"); + + if (fdSavedRoot) { + if (fchdir(fdSavedRoot.get())) + throw SysError("chdir into saved root"); + if (chroot(".")) + throw SysError("chroot into saved root"); + } + + if (chdir(savedCwd.c_str()) == -1) + throw SysError("restoring cwd"); + } catch (Error & e) { + debug(e.msg()); + } #endif +} + +void unshareFilesystem() +{ +#ifdef __linux__ + if (unshare(CLONE_FS) != 0 && errno != EPERM) + throw SysError("unsharing filesystem state in download thread"); +#endif +} + +} diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh index 0b7eeb66c..7e4e921a8 100644 --- a/src/libutil/namespaces.hh +++ b/src/libutil/namespaces.hh @@ -1,8 +1,31 @@ #pragma once ///@file +#include + +#include "types.hh" + namespace nix { +/** + * Save the current mount namespace. Ignored if called more than + * once. + */ +void saveMountNamespace(); + +/** + * Restore the mount namespace saved by saveMountNamespace(). Ignored + * if saveMountNamespace() was never called. + */ +void restoreMountNamespace(); + +/** + * Cause this thread to not share any FS attributes with the main + * thread, because this causes setns() in restoreMountNamespace() to + * fail. + */ +void unshareFilesystem(); + #if __linux__ bool userNamespacesSupported(); diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index d5e32d989..dc96f84e5 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -1,4 +1,5 @@ #include "posix-source-accessor.hh" +#include "signals.hh" namespace nix { diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc new file mode 100644 index 000000000..91a0ea66f --- /dev/null +++ b/src/libutil/processes.cc @@ -0,0 +1,421 @@ +#include "current-process.hh" +#include "environment-variables.hh" +#include "signals.hh" +#include "processes.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#ifdef __APPLE__ +# include +#endif + +#ifdef __linux__ +# include +# include +#endif + + +namespace nix { + +Pid::Pid() +{ +} + + +Pid::Pid(pid_t pid) + : pid(pid) +{ +} + + +Pid::~Pid() +{ + if (pid != -1) kill(); +} + + +void Pid::operator =(pid_t pid) +{ + if (this->pid != -1 && this->pid != pid) kill(); + this->pid = pid; + killSignal = SIGKILL; // reset signal to default +} + + +Pid::operator pid_t() +{ + return pid; +} + + +int Pid::kill() +{ + assert(pid != -1); + + debug("killing process %1%", pid); + + /* Send the requested signal to the child. If it has its own + process group, send the signal to every process in the child + process group (which hopefully includes *all* its children). */ + if (::kill(separatePG ? -pid : pid, killSignal) != 0) { + /* On BSDs, killing a process group will return EPERM if all + processes in the group are zombies (or something like + that). So try to detect and ignore that situation. */ +#if __FreeBSD__ || __APPLE__ + if (errno != EPERM || ::kill(pid, 0) != 0) +#endif + logError(SysError("killing process %d", pid).info()); + } + + return wait(); +} + + +int Pid::wait() +{ + assert(pid != -1); + while (1) { + int status; + int res = waitpid(pid, &status, 0); + if (res == pid) { + pid = -1; + return status; + } + if (errno != EINTR) + throw SysError("cannot get exit status of PID %d", pid); + checkInterrupt(); + } +} + + +void Pid::setSeparatePG(bool separatePG) +{ + this->separatePG = separatePG; +} + + +void Pid::setKillSignal(int signal) +{ + this->killSignal = signal; +} + + +pid_t Pid::release() +{ + pid_t p = pid; + pid = -1; + return p; +} + + +void killUser(uid_t uid) +{ + debug("killing all processes running under uid '%1%'", uid); + + assert(uid != 0); /* just to be safe... */ + + /* The system call kill(-1, sig) sends the signal `sig' to all + users to which the current process can send signals. So we + fork a process, switch to uid, and send a mass kill. */ + + Pid pid = startProcess([&]() { + + if (setuid(uid) == -1) + throw SysError("setting uid"); + + while (true) { +#ifdef __APPLE__ + /* OSX's kill syscall takes a third parameter that, among + other things, determines if kill(-1, signo) affects the + calling process. In the OSX libc, it's set to true, + which means "follow POSIX", which we don't want here + */ + if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; +#else + if (kill(-1, SIGKILL) == 0) break; +#endif + if (errno == ESRCH || errno == EPERM) break; /* no more processes */ + if (errno != EINTR) + throw SysError("cannot kill processes for uid '%1%'", uid); + } + + _exit(0); + }); + + int status = pid.wait(); + if (status != 0) + throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status)); + + /* !!! We should really do some check to make sure that there are + no processes left running under `uid', but there is no portable + way to do so (I think). The most reliable way may be `ps -eo + uid | grep -q $uid'. */ +} + + +////////////////////////////////////////////////////////////////////// + + +/* Wrapper around vfork to prevent the child process from clobbering + the caller's stack frame in the parent. */ +static pid_t doFork(bool allowVfork, std::function fun) __attribute__((noinline)); +static pid_t doFork(bool allowVfork, std::function fun) +{ +#ifdef __linux__ + pid_t pid = allowVfork ? vfork() : fork(); +#else + pid_t pid = fork(); +#endif + if (pid != 0) return pid; + fun(); + abort(); +} + + +#if __linux__ +static int childEntry(void * arg) +{ + auto main = (std::function *) arg; + (*main)(); + return 1; +} +#endif + + +pid_t startProcess(std::function fun, const ProcessOptions & options) +{ + std::function wrapper = [&]() { + if (!options.allowVfork) + logger = makeSimpleLogger(); + try { +#if __linux__ + if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) + throw SysError("setting death signal"); +#endif + fun(); + } catch (std::exception & e) { + try { + std::cerr << options.errorPrefix << e.what() << "\n"; + } catch (...) { } + } catch (...) { } + if (options.runExitHandlers) + exit(1); + else + _exit(1); + }; + + pid_t pid = -1; + + if (options.cloneFlags) { + #ifdef __linux__ + // Not supported, since then we don't know when to free the stack. + assert(!(options.cloneFlags & CLONE_VM)); + + size_t stackSize = 1 * 1024 * 1024; + auto stack = (char *) mmap(0, stackSize, + PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (stack == MAP_FAILED) throw SysError("allocating stack"); + + Finally freeStack([&]() { munmap(stack, stackSize); }); + + pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper); + #else + throw Error("clone flags are only supported on Linux"); + #endif + } else + pid = doFork(options.allowVfork, wrapper); + + if (pid == -1) throw SysError("unable to fork"); + + return pid; +} + + +std::string runProgram(Path program, bool searchPath, const Strings & args, + const std::optional & input, bool isInteractive) +{ + auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive}); + + if (!statusOk(res.first)) + throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); + + return res.second; +} + +// Output = error code + "standard out" output stream +std::pair runProgram(RunOptions && options) +{ + StringSink sink; + options.standardOut = &sink; + + int status = 0; + + try { + runProgram2(options); + } catch (ExecError & e) { + status = e.status; + } + + return {status, std::move(sink.s)}; +} + +void runProgram2(const RunOptions & options) +{ + checkInterrupt(); + + assert(!(options.standardIn && options.input)); + + std::unique_ptr source_; + Source * source = options.standardIn; + + if (options.input) { + source_ = std::make_unique(*options.input); + source = source_.get(); + } + + /* Create a pipe. */ + Pipe out, in; + if (options.standardOut) out.create(); + if (source) in.create(); + + ProcessOptions processOptions; + // vfork implies that the environment of the main process and the fork will + // be shared (technically this is undefined, but in practice that's the + // case), so we can't use it if we alter the environment + processOptions.allowVfork = !options.environment; + + std::optional>> resumeLoggerDefer; + if (options.isInteractive) { + logger->pause(); + resumeLoggerDefer.emplace( + []() { + logger->resume(); + } + ); + } + + /* Fork. */ + Pid pid = startProcess([&]() { + if (options.environment) + replaceEnv(*options.environment); + if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + if (options.mergeStderrToStdout) + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); + if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) + throw SysError("dupping stdin"); + + if (options.chdir && chdir((*options.chdir).c_str()) == -1) + throw SysError("chdir failed"); + if (options.gid && setgid(*options.gid) == -1) + throw SysError("setgid failed"); + /* Drop all other groups if we're setgid. */ + if (options.gid && setgroups(0, 0) == -1) + throw SysError("setgroups failed"); + if (options.uid && setuid(*options.uid) == -1) + throw SysError("setuid failed"); + + Strings args_(options.args); + args_.push_front(options.program); + + restoreProcessContext(); + + if (options.searchPath) + execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); + // This allows you to refer to a program with a pathname relative + // to the PATH variable. + else + execv(options.program.c_str(), stringsToCharPtrs(args_).data()); + + throw SysError("executing '%1%'", options.program); + }, processOptions); + + out.writeSide.close(); + + std::thread writerThread; + + std::promise promise; + + Finally doJoin([&]() { + if (writerThread.joinable()) + writerThread.join(); + }); + + + if (source) { + in.readSide.close(); + writerThread = std::thread([&]() { + try { + std::vector buf(8 * 1024); + while (true) { + size_t n; + try { + n = source->read(buf.data(), buf.size()); + } catch (EndOfFile &) { + break; + } + writeFull(in.writeSide.get(), {buf.data(), n}); + } + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + in.writeSide.close(); + }); + } + + if (options.standardOut) + drainFD(out.readSide.get(), *options.standardOut); + + /* Wait for the child to finish. */ + int status = pid.wait(); + + /* Wait for the writer thread to finish. */ + if (source) promise.get_future().get(); + + if (status) + throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); +} + +////////////////////////////////////////////////////////////////////// + +std::string statusToString(int status) +{ + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + if (WIFEXITED(status)) + return fmt("failed with exit code %1%", WEXITSTATUS(status)); + else if (WIFSIGNALED(status)) { + int sig = WTERMSIG(status); +#if HAVE_STRSIGNAL + const char * description = strsignal(sig); + return fmt("failed due to signal %1% (%2%)", sig, description); +#else + return fmt("failed due to signal %1%", sig); +#endif + } + else + return "died abnormally"; + } else return "succeeded"; +} + + +bool statusOk(int status) +{ + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +} diff --git a/src/libutil/processes.hh b/src/libutil/processes.hh new file mode 100644 index 000000000..978c37105 --- /dev/null +++ b/src/libutil/processes.hh @@ -0,0 +1,123 @@ +#pragma once +///@file + +#include "types.hh" +#include "error.hh" +#include "logging.hh" +#include "ansicolor.hh" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace nix { + +struct Sink; +struct Source; + +class Pid +{ + pid_t pid = -1; + bool separatePG = false; + int killSignal = SIGKILL; +public: + Pid(); + Pid(pid_t pid); + ~Pid(); + void operator =(pid_t pid); + operator pid_t(); + int kill(); + int wait(); + + void setSeparatePG(bool separatePG); + void setKillSignal(int signal); + pid_t release(); +}; + + +/** + * Kill all processes running under the specified uid by sending them + * a SIGKILL. + */ +void killUser(uid_t uid); + + +/** + * Fork a process that runs the given function, and return the child + * pid to the caller. + */ +struct ProcessOptions +{ + std::string errorPrefix = ""; + bool dieWithParent = true; + bool runExitHandlers = false; + bool allowVfork = false; + /** + * use clone() with the specified flags (Linux only) + */ + int cloneFlags = 0; +}; + +pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); + + +/** + * Run a program and return its stdout in a string (i.e., like the + * shell backtick operator). + */ +std::string runProgram(Path program, bool searchPath = false, + const Strings & args = Strings(), + const std::optional & input = {}, bool isInteractive = false); + +struct RunOptions +{ + Path program; + bool searchPath = true; + Strings args; + std::optional uid; + std::optional gid; + std::optional chdir; + std::optional> environment; + std::optional input; + Source * standardIn = nullptr; + Sink * standardOut = nullptr; + bool mergeStderrToStdout = false; + bool isInteractive = false; +}; + +std::pair runProgram(RunOptions && options); + +void runProgram2(const RunOptions & options); + + +class ExecError : public Error +{ +public: + int status; + + template + ExecError(int status, const Args & ... args) + : Error(args...), status(status) + { } +}; + + +/** + * Convert the exit status of a child as returned by wait() into an + * error string. + */ +std::string statusToString(int status); + +bool statusOk(int status); + +} diff --git a/src/libutil/references.cc b/src/libutil/references.cc index 7f59b4c09..9d75606ef 100644 --- a/src/libutil/references.cc +++ b/src/libutil/references.cc @@ -1,6 +1,5 @@ #include "references.hh" #include "hash.hh" -#include "util.hh" #include "archive.hh" #include diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 3d5121a19..725ddbb8d 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -1,5 +1,5 @@ #include "serialise.hh" -#include "util.hh" +#include "signals.hh" #include #include diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 333c254ea..9e07226bf 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -5,6 +5,7 @@ #include "types.hh" #include "util.hh" +#include "file-descriptor.hh" namespace boost::context { struct stack_context; } diff --git a/src/libutil/signals.cc b/src/libutil/signals.cc new file mode 100644 index 000000000..4632aa319 --- /dev/null +++ b/src/libutil/signals.cc @@ -0,0 +1,188 @@ +#include "signals.hh" +#include "util.hh" +#include "error.hh" +#include "sync.hh" +#include "terminal.hh" + +#include + +namespace nix { + +std::atomic _isInterrupted = false; + +static thread_local bool interruptThrown = false; +thread_local std::function interruptCheck; + +void setInterruptThrown() +{ + interruptThrown = true; +} + +void _interrupted() +{ + /* Block user interrupts while an exception is being handled. + Throwing an exception while another exception is being handled + kills the program! */ + if (!interruptThrown && !std::uncaught_exceptions()) { + interruptThrown = true; + throw Interrupted("interrupted by the user"); + } +} + + +////////////////////////////////////////////////////////////////////// + + +/* We keep track of interrupt callbacks using integer tokens, so we can iterate + safely without having to lock the data structure while executing arbitrary + functions. + */ +struct InterruptCallbacks { + typedef int64_t Token; + + /* We use unique tokens so that we can't accidentally delete the wrong + handler because of an erroneous double delete. */ + Token nextToken = 0; + + /* Used as a list, see InterruptCallbacks comment. */ + std::map> callbacks; +}; + +static Sync _interruptCallbacks; + +static void signalHandlerThread(sigset_t set) +{ + while (true) { + int signal = 0; + sigwait(&set, &signal); + + if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) + triggerInterrupt(); + + else if (signal == SIGWINCH) { + updateWindowSize(); + } + } +} + +void triggerInterrupt() +{ + _isInterrupted = true; + + { + InterruptCallbacks::Token i = 0; + while (true) { + std::function callback; + { + auto interruptCallbacks(_interruptCallbacks.lock()); + auto lb = interruptCallbacks->callbacks.lower_bound(i); + if (lb == interruptCallbacks->callbacks.end()) + break; + + callback = lb->second; + i = lb->first + 1; + } + + try { + callback(); + } catch (...) { + ignoreException(); + } + } + } +} + + +static sigset_t savedSignalMask; +static bool savedSignalMaskIsSet = false; + +void setChildSignalMask(sigset_t * sigs) +{ + assert(sigs); // C style function, but think of sigs as a reference + +#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE + sigemptyset(&savedSignalMask); + // There's no "assign" or "copy" function, so we rely on (math) idempotence + // of the or operator: a or a = a. + sigorset(&savedSignalMask, sigs, sigs); +#else + // Without sigorset, our best bet is to assume that sigset_t is a type that + // can be assigned directly, such as is the case for a sigset_t defined as + // an integer type. + savedSignalMask = *sigs; +#endif + + savedSignalMaskIsSet = true; +} + +void saveSignalMask() { + if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) + throw SysError("querying signal mask"); + + savedSignalMaskIsSet = true; +} + +void startSignalHandlerThread() +{ + updateWindowSize(); + + saveSignalMask(); + + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGINT); + sigaddset(&set, SIGTERM); + sigaddset(&set, SIGHUP); + sigaddset(&set, SIGPIPE); + sigaddset(&set, SIGWINCH); + if (pthread_sigmask(SIG_BLOCK, &set, nullptr)) + throw SysError("blocking signals"); + + std::thread(signalHandlerThread, set).detach(); +} + +void restoreSignals() +{ + // If startSignalHandlerThread wasn't called, that means we're not running + // in a proper libmain process, but a process that presumably manages its + // own signal handlers. Such a process should call either + // - initNix(), to be a proper libmain process + // - startSignalHandlerThread(), to resemble libmain regarding signal + // handling only + // - saveSignalMask(), for processes that define their own signal handling + // thread + // TODO: Warn about this? Have a default signal mask? The latter depends on + // whether we should generally inherit signal masks from the caller. + // I don't know what the larger unix ecosystem expects from us here. + if (!savedSignalMaskIsSet) + return; + + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); +} + + +/* RAII helper to automatically deregister a callback. */ +struct InterruptCallbackImpl : InterruptCallback +{ + InterruptCallbacks::Token token; + ~InterruptCallbackImpl() override + { + auto interruptCallbacks(_interruptCallbacks.lock()); + interruptCallbacks->callbacks.erase(token); + } +}; + +std::unique_ptr createInterruptCallback(std::function callback) +{ + auto interruptCallbacks(_interruptCallbacks.lock()); + auto token = interruptCallbacks->nextToken++; + interruptCallbacks->callbacks.emplace(token, callback); + + auto res = std::make_unique(); + res->token = token; + + return std::unique_ptr(res.release()); +} + +} diff --git a/src/libutil/signals.hh b/src/libutil/signals.hh new file mode 100644 index 000000000..7e8beff33 --- /dev/null +++ b/src/libutil/signals.hh @@ -0,0 +1,104 @@ +#pragma once +///@file + +#include "types.hh" +#include "error.hh" +#include "logging.hh" +#include "ansicolor.hh" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace nix { + +/* User interruption. */ + +extern std::atomic _isInterrupted; + +extern thread_local std::function interruptCheck; + +void setInterruptThrown(); + +void _interrupted(); + +void inline checkInterrupt() +{ + if (_isInterrupted || (interruptCheck && interruptCheck())) + _interrupted(); +} + +MakeError(Interrupted, BaseError); + + +/** + * Start a thread that handles various signals. Also block those signals + * on the current thread (and thus any threads created by it). + * Saves the signal mask before changing the mask to block those signals. + * See saveSignalMask(). + */ +void startSignalHandlerThread(); + +/** + * Saves the signal mask, which is the signal mask that nix will restore + * before creating child processes. + * See setChildSignalMask() to set an arbitrary signal mask instead of the + * current mask. + */ +void saveSignalMask(); + +/** + * To use in a process that already called `startSignalHandlerThread()` + * or `saveSignalMask()` first. + */ +void restoreSignals(); + +/** + * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't + * necessarily match the current thread's mask. + * See saveSignalMask() to set the saved mask to the current mask. + */ +void setChildSignalMask(sigset_t *sigs); + +struct InterruptCallback +{ + virtual ~InterruptCallback() { }; +}; + +/** + * Register a function that gets called on SIGINT (in a non-signal + * context). + */ +std::unique_ptr createInterruptCallback( + std::function callback); + +void triggerInterrupt(); + +/** + * A RAII class that causes the current thread to receive SIGUSR1 when + * the signal handler thread receives SIGINT. That is, this allows + * SIGINT to be multiplexed to multiple threads. + */ +struct ReceiveInterrupts +{ + pthread_t target; + std::unique_ptr callback; + + ReceiveInterrupts() + : target(pthread_self()) + , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) + { } +}; + + +} diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc index 9510a5f0c..e67e986fb 100644 --- a/src/libutil/suggestions.cc +++ b/src/libutil/suggestions.cc @@ -1,7 +1,9 @@ #include "suggestions.hh" #include "ansicolor.hh" -#include "util.hh" +#include "terminal.hh" + #include +#include namespace nix { diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 5060a8f24..1733c791c 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -3,6 +3,7 @@ #include "serialise.hh" #include "tarfile.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc new file mode 100644 index 000000000..8febc8771 --- /dev/null +++ b/src/libutil/terminal.cc @@ -0,0 +1,108 @@ +#include "terminal.hh" +#include "environment-variables.hh" +#include "sync.hh" + +#include +#include + +namespace nix { + +bool shouldANSI() +{ + return isatty(STDERR_FILENO) + && getEnv("TERM").value_or("dumb") != "dumb" + && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value()); +} + +std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) +{ + std::string t, e; + size_t w = 0; + auto i = s.begin(); + + while (w < (size_t) width && i != s.end()) { + + if (*i == '\e') { + std::string e; + e += *i++; + char last = 0; + + if (i != s.end() && *i == '[') { + e += *i++; + // eat parameter bytes + while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; + // eat intermediate bytes + while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; + // eat final byte + if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; + } else { + if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; + } + + if (!filterAll && last == 'm') + t += e; + } + + else if (*i == '\t') { + i++; t += ' '; w++; + while (w < (size_t) width && w % 8) { + t += ' '; w++; + } + } + + else if (*i == '\r' || *i == '\a') + // do nothing for now + i++; + + else { + w++; + // Copy one UTF-8 character. + if ((*i & 0xe0) == 0xc0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } else if ((*i & 0xf0) == 0xe0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } + } else if ((*i & 0xf8) == 0xf0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } + } + } else + t += *i++; + } + } + + return t; +} + + +////////////////////////////////////////////////////////////////////// + +static Sync> windowSize{{0, 0}}; + + +void updateWindowSize() +{ + struct winsize ws; + if (ioctl(2, TIOCGWINSZ, &ws) == 0) { + auto windowSize_(windowSize.lock()); + windowSize_->first = ws.ws_row; + windowSize_->second = ws.ws_col; + } +} + + +std::pair getWindowSize() +{ + return *windowSize.lock(); +} + +} diff --git a/src/libutil/terminal.hh b/src/libutil/terminal.hh new file mode 100644 index 000000000..9cb191308 --- /dev/null +++ b/src/libutil/terminal.hh @@ -0,0 +1,38 @@ +#pragma once +///@file + +#include "types.hh" + +namespace nix { +/** + * Determine whether ANSI escape sequences are appropriate for the + * present output. + */ +bool shouldANSI(); + +/** + * Truncate a string to 'width' printable characters. If 'filterAll' + * is true, all ANSI escape sequences are filtered out. Otherwise, + * some escape sequences (such as colour setting) are copied but not + * included in the character count. Also, tabs are expanded to + * spaces. + */ +std::string filterANSIEscapes(std::string_view s, + bool filterAll = false, + unsigned int width = std::numeric_limits::max()); + +/** + * Recalculate the window size, updating a global variable. Used in the + * `SIGWINCH` signal handler. + */ +void updateWindowSize(); + +/** + * @return the number of rows and columns of the terminal. + * + * The value is cached so this is quick. The cached result is computed + * by `updateWindowSize()`. + */ +std::pair getWindowSize(); + +} diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index 2ffdc2e9b..c6dfe63d3 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -2,7 +2,6 @@ #include "logging.hh" #include "nixexpr.hh" -#include "util.hh" #include #include diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index f3c1e8248..568f03f70 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -1,5 +1,8 @@ #include "util.hh" #include "types.hh" +#include "file-system.hh" +#include "processes.hh" +#include "terminal.hh" #include #include diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index dc4067f1b..c5e735617 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -1,4 +1,6 @@ #include "thread-pool.hh" +#include "signals.hh" +#include "util.hh" namespace nix { diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index 0e09fae97..02765badc 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -1,8 +1,8 @@ #pragma once ///@file +#include "error.hh" #include "sync.hh" -#include "util.hh" #include #include diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc new file mode 100644 index 000000000..8949461d2 --- /dev/null +++ b/src/libutil/unix-domain-socket.cc @@ -0,0 +1,100 @@ +#include "file-system.hh" +#include "processes.hh" +#include "unix-domain-socket.hh" + +#include +#include +#include + +namespace nix { + +AutoCloseFD createUnixDomainSocket() +{ + AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM + #ifdef SOCK_CLOEXEC + | SOCK_CLOEXEC + #endif + , 0); + if (!fdSocket) + throw SysError("cannot create Unix domain socket"); + closeOnExec(fdSocket.get()); + return fdSocket; +} + + +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) +{ + auto fdSocket = nix::createUnixDomainSocket(); + + bind(fdSocket.get(), path); + + if (chmod(path.c_str(), mode) == -1) + throw SysError("changing permissions on '%1%'", path); + + if (listen(fdSocket.get(), 100) == -1) + throw SysError("cannot listen on socket '%1%'", path); + + return fdSocket; +} + + +void bind(int fd, const std::string & path) +{ + unlink(path.c_str()); + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + + if (path.size() + 1 >= sizeof(addr.sun_path)) { + Pid pid = startProcess([&]() { + Path dir = dirOf(path); + if (chdir(dir.c_str()) == -1) + throw SysError("chdir to '%s' failed", dir); + std::string base(baseNameOf(path)); + if (base.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%s' is too long", base); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); + if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot bind to socket '%s'", path); + _exit(0); + }); + int status = pid.wait(); + if (status != 0) + throw Error("cannot bind to socket '%s'", path); + } else { + memcpy(addr.sun_path, path.c_str(), path.size() + 1); + if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot bind to socket '%s'", path); + } +} + + +void connect(int fd, const std::string & path) +{ + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + + if (path.size() + 1 >= sizeof(addr.sun_path)) { + Pid pid = startProcess([&]() { + Path dir = dirOf(path); + if (chdir(dir.c_str()) == -1) + throw SysError("chdir to '%s' failed", dir); + std::string base(baseNameOf(path)); + if (base.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%s' is too long", base); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); + if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot connect to socket at '%s'", path); + _exit(0); + }); + int status = pid.wait(); + if (status != 0) + throw Error("cannot connect to socket at '%s'", path); + } else { + memcpy(addr.sun_path, path.c_str(), path.size() + 1); + if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot connect to socket at '%s'", path); + } +} + +} diff --git a/src/libutil/unix-domain-socket.hh b/src/libutil/unix-domain-socket.hh new file mode 100644 index 000000000..b78feb454 --- /dev/null +++ b/src/libutil/unix-domain-socket.hh @@ -0,0 +1,31 @@ +#pragma once +///@file + +#include "types.hh" +#include "file-descriptor.hh" + +#include + +namespace nix { + +/** + * Create a Unix domain socket. + */ +AutoCloseFD createUnixDomainSocket(); + +/** + * Create a Unix domain socket in listen mode. + */ +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); + +/** + * Bind a Unix domain socket to a path. + */ +void bind(int fd, const std::string & path); + +/** + * Connect to a Unix domain socket. + */ +void connect(int fd, const std::string & path); + +} diff --git a/src/libutil/users.cc b/src/libutil/users.cc new file mode 100644 index 000000000..95a641322 --- /dev/null +++ b/src/libutil/users.cc @@ -0,0 +1,116 @@ +#include "util.hh" +#include "users.hh" +#include "environment-variables.hh" +#include "file-system.hh" + +#include +#include +#include + +namespace nix { + +std::string getUserName() +{ + auto pw = getpwuid(geteuid()); + std::string name = pw ? pw->pw_name : getEnv("USER").value_or(""); + if (name.empty()) + throw Error("cannot figure out user name"); + return name; +} + +Path getHomeOf(uid_t userId) +{ + std::vector buf(16384); + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + return pw->pw_dir; +} + +Path getHome() +{ + static Path homeDir = []() + { + std::optional unownedUserHomeDir = {}; + auto homeDir = getEnv("HOME"); + if (homeDir) { + // Only use $HOME if doesn't exist or is owned by the current user. + struct stat st; + int result = stat(homeDir->c_str(), &st); + if (result != 0) { + if (errno != ENOENT) { + warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); + homeDir.reset(); + } + } else if (st.st_uid != geteuid()) { + unownedUserHomeDir.swap(homeDir); + } + } + if (!homeDir) { + homeDir = getHomeOf(geteuid()); + if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { + warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); + } + } + return *homeDir; + }(); + return homeDir; +} + + +Path getCacheDir() +{ + auto cacheDir = getEnv("XDG_CACHE_HOME"); + return cacheDir ? *cacheDir : getHome() + "/.cache"; +} + + +Path getConfigDir() +{ + auto configDir = getEnv("XDG_CONFIG_HOME"); + return configDir ? *configDir : getHome() + "/.config"; +} + +std::vector getConfigDirs() +{ + Path configHome = getConfigDir(); + auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); + std::vector result = tokenizeString>(configDirs, ":"); + result.insert(result.begin(), configHome); + return result; +} + + +Path getDataDir() +{ + auto dataDir = getEnv("XDG_DATA_HOME"); + return dataDir ? *dataDir : getHome() + "/.local/share"; +} + +Path getStateDir() +{ + auto stateDir = getEnv("XDG_STATE_HOME"); + return stateDir ? *stateDir : getHome() + "/.local/state"; +} + +Path createNixStateDir() +{ + Path dir = getStateDir() + "/nix"; + createDirs(dir); + return dir; +} + + +std::string expandTilde(std::string_view path) +{ + // TODO: expand ~user ? + auto tilde = path.substr(0, 2); + if (tilde == "~/" || tilde == "~") + return getHome() + std::string(path.substr(1)); + else + return std::string(path); +} + +} diff --git a/src/libutil/users.hh b/src/libutil/users.hh new file mode 100644 index 000000000..cecbb8bfb --- /dev/null +++ b/src/libutil/users.hh @@ -0,0 +1,58 @@ +#pragma once +///@file + +#include "types.hh" + +#include + +namespace nix { + +std::string getUserName(); + +/** + * @return the given user's home directory from /etc/passwd. + */ +Path getHomeOf(uid_t userId); + +/** + * @return $HOME or the user's home directory from /etc/passwd. + */ +Path getHome(); + +/** + * @return $XDG_CACHE_HOME or $HOME/.cache. + */ +Path getCacheDir(); + +/** + * @return $XDG_CONFIG_HOME or $HOME/.config. + */ +Path getConfigDir(); + +/** + * @return the directories to search for user configuration files + */ +std::vector getConfigDirs(); + +/** + * @return $XDG_DATA_HOME or $HOME/.local/share. + */ +Path getDataDir(); + +/** + * @return $XDG_STATE_HOME or $HOME/.local/state. + */ +Path getStateDir(); + +/** + * Create the Nix state directory and return the path to it. + */ +Path createNixStateDir(); + +/** + * Perform tilde expansion on a path, replacing tilde with the user's + * home directory. + */ +std::string expandTilde(std::string_view path); + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 3b4c181e5..ee7a22849 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,49 +1,10 @@ #include "util.hh" -#include "sync.hh" -#include "finally.hh" -#include "serialise.hh" -#include "cgroup.hh" +#include "fmt.hh" #include #include -#include -#include -#include -#include -#include -#include #include -#include -#include -#include - -#include #include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#include -#endif - -#ifdef __linux__ -#include -#include -#include - -#include -#endif - - -extern char * * environ __attribute__((weak)); - namespace nix { @@ -67,1099 +28,8 @@ void initLibUtil() { assert(caught); } -std::optional getEnv(const std::string & key) -{ - char * value = getenv(key.c_str()); - if (!value) return {}; - return std::string(value); -} - -std::optional getEnvNonEmpty(const std::string & key) { - auto value = getEnv(key); - if (value == "") return {}; - return value; -} - -std::map getEnv() -{ - std::map env; - for (size_t i = 0; environ[i]; ++i) { - auto s = environ[i]; - auto eq = strchr(s, '='); - if (!eq) - // invalid env, just keep going - continue; - env.emplace(std::string(s, eq), std::string(eq + 1)); - } - return env; -} - - -void clearEnv() -{ - for (auto & name : getEnv()) - unsetenv(name.first.c_str()); -} - -void replaceEnv(const std::map & newEnv) -{ - clearEnv(); - for (auto & newEnvVar : newEnv) - setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); -} - - -Path absPath(Path path, std::optional dir, bool resolveSymlinks) -{ - if (path[0] != '/') { - if (!dir) { -#ifdef __GNU__ - /* GNU (aka. GNU/Hurd) doesn't have any limitation on path - lengths and doesn't define `PATH_MAX'. */ - char *buf = getcwd(NULL, 0); - if (buf == NULL) -#else - char buf[PATH_MAX]; - if (!getcwd(buf, sizeof(buf))) -#endif - throw SysError("cannot get cwd"); - path = concatStrings(buf, "/", path); -#ifdef __GNU__ - free(buf); -#endif - } else - path = concatStrings(*dir, "/", path); - } - return canonPath(path, resolveSymlinks); -} - - -Path canonPath(PathView path, bool resolveSymlinks) -{ - assert(path != ""); - - std::string s; - s.reserve(256); - - if (path[0] != '/') - throw Error("not an absolute path: '%1%'", path); - - std::string temp; - - /* Count the number of times we follow a symlink and stop at some - arbitrary (but high) limit to prevent infinite loops. */ - unsigned int followCount = 0, maxFollow = 1024; - - while (1) { - - /* Skip slashes. */ - while (!path.empty() && path[0] == '/') path.remove_prefix(1); - if (path.empty()) break; - - /* Ignore `.'. */ - if (path == "." || path.substr(0, 2) == "./") - path.remove_prefix(1); - - /* If `..', delete the last component. */ - else if (path == ".." || path.substr(0, 3) == "../") - { - if (!s.empty()) s.erase(s.rfind('/')); - path.remove_prefix(2); - } - - /* Normal component; copy it. */ - else { - s += '/'; - if (const auto slash = path.find('/'); slash == std::string::npos) { - s += path; - path = {}; - } else { - s += path.substr(0, slash); - path = path.substr(slash); - } - - /* If s points to a symlink, resolve it and continue from there */ - if (resolveSymlinks && isLink(s)) { - if (++followCount >= maxFollow) - throw Error("infinite symlink recursion in path '%1%'", path); - temp = concatStrings(readLink(s), path); - path = temp; - if (!temp.empty() && temp[0] == '/') { - s.clear(); /* restart for symlinks pointing to absolute path */ - } else { - s = dirOf(s); - if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = / - s.clear(); - } - } - } - } - } - - return s.empty() ? "/" : std::move(s); -} - - -Path dirOf(const PathView path) -{ - Path::size_type pos = path.rfind('/'); - if (pos == std::string::npos) - return "."; - return pos == 0 ? "/" : Path(path, 0, pos); -} - - -std::string_view baseNameOf(std::string_view path) -{ - if (path.empty()) - return ""; - - auto last = path.size() - 1; - if (path[last] == '/' && last > 0) - last -= 1; - - auto pos = path.rfind('/', last); - if (pos == std::string::npos) - pos = 0; - else - pos += 1; - - return path.substr(pos, last - pos + 1); -} - - -std::string expandTilde(std::string_view path) -{ - // TODO: expand ~user ? - auto tilde = path.substr(0, 2); - if (tilde == "~/" || tilde == "~") - return getHome() + std::string(path.substr(1)); - else - return std::string(path); -} - - -bool isInDir(std::string_view path, std::string_view dir) -{ - return path.substr(0, 1) == "/" - && path.substr(0, dir.size()) == dir - && path.size() >= dir.size() + 2 - && path[dir.size()] == '/'; -} - - -bool isDirOrInDir(std::string_view path, std::string_view dir) -{ - return path == dir || isInDir(path, dir); -} - - -struct stat stat(const Path & path) -{ - struct stat st; - if (stat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); - return st; -} - - -struct stat lstat(const Path & path) -{ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); - return st; -} - - -bool pathExists(const Path & path) -{ - int res; - struct stat st; - res = lstat(path.c_str(), &st); - if (!res) return true; - if (errno != ENOENT && errno != ENOTDIR) - throw SysError("getting status of %1%", path); - return false; -} - -bool pathAccessible(const Path & path) -{ - try { - return pathExists(path); - } catch (SysError & e) { - // swallow EPERM - if (e.errNo == EPERM) return false; - throw; - } -} - - -Path readLink(const Path & path) -{ - checkInterrupt(); - std::vector buf; - for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { - buf.resize(bufSize); - ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); - if (rlSize == -1) - if (errno == EINVAL) - throw Error("'%1%' is not a symlink", path); - else - throw SysError("reading symbolic link '%1%'", path); - else if (rlSize < bufSize) - return std::string(buf.data(), rlSize); - } -} - - -bool isLink(const Path & path) -{ - struct stat st = lstat(path); - return S_ISLNK(st.st_mode); -} - - -DirEntries readDirectory(DIR *dir, const Path & path) -{ - DirEntries entries; - entries.reserve(64); - - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { /* sic */ - checkInterrupt(); - std::string name = dirent->d_name; - if (name == "." || name == "..") continue; - entries.emplace_back(name, dirent->d_ino, -#ifdef HAVE_STRUCT_DIRENT_D_TYPE - dirent->d_type -#else - DT_UNKNOWN -#endif - ); - } - if (errno) throw SysError("reading directory '%1%'", path); - - return entries; -} - -DirEntries readDirectory(const Path & path) -{ - AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError("opening directory '%1%'", path); - - return readDirectory(dir.get(), path); -} - - -unsigned char getFileType(const Path & path) -{ - struct stat st = lstat(path); - if (S_ISDIR(st.st_mode)) return DT_DIR; - if (S_ISLNK(st.st_mode)) return DT_LNK; - if (S_ISREG(st.st_mode)) return DT_REG; - return DT_UNKNOWN; -} - - -std::string readFile(int fd) -{ - struct stat st; - if (fstat(fd, &st) == -1) - throw SysError("statting file"); - - return drainFD(fd, true, st.st_size); -} - - -std::string readFile(const Path & path) -{ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) - throw SysError("opening file '%1%'", path); - return readFile(fd.get()); -} - - -void readFile(const Path & path, Sink & sink) -{ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) - throw SysError("opening file '%s'", path); - drainFD(fd.get(), sink); -} - - -void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) -{ - AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); - if (!fd) - throw SysError("opening file '%1%'", path); - try { - writeFull(fd.get(), s); - } catch (Error & e) { - e.addTrace({}, "writing file '%1%'", path); - throw; - } - if (sync) - fd.fsync(); - // Explicitly close to make sure exceptions are propagated. - fd.close(); - if (sync) - syncParent(path); -} - - -void writeFile(const Path & path, Source & source, mode_t mode, bool sync) -{ - AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); - if (!fd) - throw SysError("opening file '%1%'", path); - - std::vector buf(64 * 1024); - - try { - while (true) { - try { - auto n = source.read(buf.data(), buf.size()); - writeFull(fd.get(), {buf.data(), n}); - } catch (EndOfFile &) { break; } - } - } catch (Error & e) { - e.addTrace({}, "writing file '%1%'", path); - throw; - } - if (sync) - fd.fsync(); - // Explicitly close to make sure exceptions are propagated. - fd.close(); - if (sync) - syncParent(path); -} - -void syncParent(const Path & path) -{ - AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0); - if (!fd) - throw SysError("opening file '%1%'", path); - fd.fsync(); -} - -std::string readLine(int fd) -{ - std::string s; - while (1) { - checkInterrupt(); - char ch; - // FIXME: inefficient - ssize_t rd = read(fd, &ch, 1); - if (rd == -1) { - if (errno != EINTR) - throw SysError("reading a line"); - } else if (rd == 0) - throw EndOfFile("unexpected EOF reading a line"); - else { - if (ch == '\n') return s; - s += ch; - } - } -} - - -void writeLine(int fd, std::string s) -{ - s += '\n'; - writeFull(fd, s); -} - - -static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) -{ - checkInterrupt(); - - std::string name(baseNameOf(path)); - - struct stat st; - if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { - if (errno == ENOENT) return; - throw SysError("getting status of '%1%'", path); - } - - if (!S_ISDIR(st.st_mode)) { - /* We are about to delete a file. Will it likely free space? */ - - switch (st.st_nlink) { - /* Yes: last link. */ - case 1: - bytesFreed += st.st_size; - break; - /* Maybe: yes, if 'auto-optimise-store' or manual optimisation - was performed. Instead of checking for real let's assume - it's an optimised file and space will be freed. - - In worst case we will double count on freed space for files - with exactly two hardlinks for unoptimised packages. - */ - case 2: - bytesFreed += st.st_size; - break; - /* No: 3+ links. */ - default: - break; - } - } - - if (S_ISDIR(st.st_mode)) { - /* Make the directory accessible. */ - const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; - if ((st.st_mode & PERM_MASK) != PERM_MASK) { - if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) - throw SysError("chmod '%1%'", path); - } - - int fd = openat(parentfd, path.c_str(), O_RDONLY); - if (fd == -1) - throw SysError("opening directory '%1%'", path); - AutoCloseDir dir(fdopendir(fd)); - if (!dir) - throw SysError("opening directory '%1%'", path); - for (auto & i : readDirectory(dir.get(), path)) - _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); - } - - int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; - if (unlinkat(parentfd, name.c_str(), flags) == -1) { - if (errno == ENOENT) return; - throw SysError("cannot unlink '%1%'", path); - } -} - -static void _deletePath(const Path & path, uint64_t & bytesFreed) -{ - Path dir = dirOf(path); - if (dir == "") - dir = "/"; - - AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)}; - if (!dirfd) { - if (errno == ENOENT) return; - throw SysError("opening directory '%1%'", path); - } - - _deletePath(dirfd.get(), path, bytesFreed); -} - - -void deletePath(const Path & path) -{ - uint64_t dummy; - deletePath(path, dummy); -} - - -void deletePath(const Path & path, uint64_t & bytesFreed) -{ - //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); - bytesFreed = 0; - _deletePath(path, bytesFreed); -} - - -std::string getUserName() -{ - auto pw = getpwuid(geteuid()); - std::string name = pw ? pw->pw_name : getEnv("USER").value_or(""); - if (name.empty()) - throw Error("cannot figure out user name"); - return name; -} - -Path getHomeOf(uid_t userId) -{ - std::vector buf(16384); - struct passwd pwbuf; - struct passwd * pw; - if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 - || !pw || !pw->pw_dir || !pw->pw_dir[0]) - throw Error("cannot determine user's home directory"); - return pw->pw_dir; -} - -Path getHome() -{ - static Path homeDir = []() - { - std::optional unownedUserHomeDir = {}; - auto homeDir = getEnv("HOME"); - if (homeDir) { - // Only use $HOME if doesn't exist or is owned by the current user. - struct stat st; - int result = stat(homeDir->c_str(), &st); - if (result != 0) { - if (errno != ENOENT) { - warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); - homeDir.reset(); - } - } else if (st.st_uid != geteuid()) { - unownedUserHomeDir.swap(homeDir); - } - } - if (!homeDir) { - homeDir = getHomeOf(geteuid()); - if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { - warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); - } - } - return *homeDir; - }(); - return homeDir; -} - - -Path getCacheDir() -{ - auto cacheDir = getEnv("XDG_CACHE_HOME"); - return cacheDir ? *cacheDir : getHome() + "/.cache"; -} - - -Path getConfigDir() -{ - auto configDir = getEnv("XDG_CONFIG_HOME"); - return configDir ? *configDir : getHome() + "/.config"; -} - -std::vector getConfigDirs() -{ - Path configHome = getConfigDir(); - auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); - std::vector result = tokenizeString>(configDirs, ":"); - result.insert(result.begin(), configHome); - return result; -} - - -Path getDataDir() -{ - auto dataDir = getEnv("XDG_DATA_HOME"); - return dataDir ? *dataDir : getHome() + "/.local/share"; -} - -Path getStateDir() -{ - auto stateDir = getEnv("XDG_STATE_HOME"); - return stateDir ? *stateDir : getHome() + "/.local/state"; -} - -Path createNixStateDir() -{ - Path dir = getStateDir() + "/nix"; - createDirs(dir); - return dir; -} - - -std::optional getSelfExe() -{ - static auto cached = []() -> std::optional - { - #if __linux__ - return readLink("/proc/self/exe"); - #elif __APPLE__ - char buf[1024]; - uint32_t size = sizeof(buf); - if (_NSGetExecutablePath(buf, &size) == 0) - return buf; - else - return std::nullopt; - #else - return std::nullopt; - #endif - }(); - return cached; -} - - -Paths createDirs(const Path & path) -{ - Paths created; - if (path == "/") return created; - - struct stat st; - if (lstat(path.c_str(), &st) == -1) { - created = createDirs(dirOf(path)); - if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) - throw SysError("creating directory '%1%'", path); - st = lstat(path); - created.push_back(path); - } - - if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) - throw SysError("statting symlink '%1%'", path); - - if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); - - return created; -} - - -void readFull(int fd, char * buf, size_t count) -{ - while (count) { - checkInterrupt(); - ssize_t res = read(fd, buf, count); - if (res == -1) { - if (errno == EINTR) continue; - throw SysError("reading from file"); - } - if (res == 0) throw EndOfFile("unexpected end-of-file"); - count -= res; - buf += res; - } -} - - -void writeFull(int fd, std::string_view s, bool allowInterrupts) -{ - while (!s.empty()) { - if (allowInterrupts) checkInterrupt(); - ssize_t res = write(fd, s.data(), s.size()); - if (res == -1 && errno != EINTR) - throw SysError("writing to file"); - if (res > 0) - s.remove_prefix(res); - } -} - - -std::string drainFD(int fd, bool block, const size_t reserveSize) -{ - // the parser needs two extra bytes to append terminating characters, other users will - // not care very much about the extra memory. - StringSink sink(reserveSize + 2); - drainFD(fd, sink, block); - return std::move(sink.s); -} - - -void drainFD(int fd, Sink & sink, bool block) -{ - // silence GCC maybe-uninitialized warning in finally - int saved = 0; - - if (!block) { - saved = fcntl(fd, F_GETFL); - if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) - throw SysError("making file descriptor non-blocking"); - } - - Finally finally([&]() { - if (!block) { - if (fcntl(fd, F_SETFL, saved) == -1) - throw SysError("making file descriptor blocking"); - } - }); - - std::vector buf(64 * 1024); - while (1) { - checkInterrupt(); - ssize_t rd = read(fd, buf.data(), buf.size()); - if (rd == -1) { - if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) - break; - if (errno != EINTR) - throw SysError("reading from file"); - } - else if (rd == 0) break; - else sink({(char *) buf.data(), (size_t) rd}); - } -} - ////////////////////////////////////////////////////////////////////// -unsigned int getMaxCPU() -{ - #if __linux__ - try { - auto cgroupFS = getCgroupFS(); - if (!cgroupFS) return 0; - - auto cgroups = getCgroups("/proc/self/cgroup"); - auto cgroup = cgroups[""]; - if (cgroup == "") return 0; - - auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; - - auto cpuMax = readFile(cpuFile); - auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); - auto quota = cpuMaxParts[0]; - auto period = cpuMaxParts[1]; - if (quota != "max") - return std::ceil(std::stoi(quota) / std::stof(period)); - } catch (Error &) { ignoreException(lvlDebug); } - #endif - - return 0; -} - -////////////////////////////////////////////////////////////////////// - - -AutoDelete::AutoDelete() : del{false} {} - -AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p) -{ - del = true; - this->recursive = recursive; -} - -AutoDelete::~AutoDelete() -{ - try { - if (del) { - if (recursive) - deletePath(path); - else { - if (remove(path.c_str()) == -1) - throw SysError("cannot unlink '%1%'", path); - } - } - } catch (...) { - ignoreException(); - } -} - -void AutoDelete::cancel() -{ - del = false; -} - -void AutoDelete::reset(const Path & p, bool recursive) { - path = p; - this->recursive = recursive; - del = true; -} - - - -////////////////////////////////////////////////////////////////////// - - -AutoCloseFD::AutoCloseFD() : fd{-1} {} - - -AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} - - -AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} -{ - that.fd = -1; -} - - -AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) -{ - close(); - fd = that.fd; - that.fd = -1; - return *this; -} - - -AutoCloseFD::~AutoCloseFD() -{ - try { - close(); - } catch (...) { - ignoreException(); - } -} - - -int AutoCloseFD::get() const -{ - return fd; -} - - -void AutoCloseFD::close() -{ - if (fd != -1) { - if (::close(fd) == -1) - /* This should never happen. */ - throw SysError("closing file descriptor %1%", fd); - fd = -1; - } -} - -void AutoCloseFD::fsync() -{ - if (fd != -1) { - int result; -#if __APPLE__ - result = ::fcntl(fd, F_FULLFSYNC); -#else - result = ::fsync(fd); -#endif - if (result == -1) - throw SysError("fsync file descriptor %1%", fd); - } -} - - -AutoCloseFD::operator bool() const -{ - return fd != -1; -} - - -int AutoCloseFD::release() -{ - int oldFD = fd; - fd = -1; - return oldFD; -} - - -void Pipe::create() -{ - int fds[2]; -#if HAVE_PIPE2 - if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); -#else - if (pipe(fds) != 0) throw SysError("creating pipe"); - closeOnExec(fds[0]); - closeOnExec(fds[1]); -#endif - readSide = fds[0]; - writeSide = fds[1]; -} - - -void Pipe::close() -{ - readSide.close(); - writeSide.close(); -} - - -////////////////////////////////////////////////////////////////////// - - -Pid::Pid() -{ -} - - -Pid::Pid(pid_t pid) - : pid(pid) -{ -} - - -Pid::~Pid() -{ - if (pid != -1) kill(); -} - - -void Pid::operator =(pid_t pid) -{ - if (this->pid != -1 && this->pid != pid) kill(); - this->pid = pid; - killSignal = SIGKILL; // reset signal to default -} - - -Pid::operator pid_t() -{ - return pid; -} - - -int Pid::kill() -{ - assert(pid != -1); - - debug("killing process %1%", pid); - - /* Send the requested signal to the child. If it has its own - process group, send the signal to every process in the child - process group (which hopefully includes *all* its children). */ - if (::kill(separatePG ? -pid : pid, killSignal) != 0) { - /* On BSDs, killing a process group will return EPERM if all - processes in the group are zombies (or something like - that). So try to detect and ignore that situation. */ -#if __FreeBSD__ || __APPLE__ - if (errno != EPERM || ::kill(pid, 0) != 0) -#endif - logError(SysError("killing process %d", pid).info()); - } - - return wait(); -} - - -int Pid::wait() -{ - assert(pid != -1); - while (1) { - int status; - int res = waitpid(pid, &status, 0); - if (res == pid) { - pid = -1; - return status; - } - if (errno != EINTR) - throw SysError("cannot get exit status of PID %d", pid); - checkInterrupt(); - } -} - - -void Pid::setSeparatePG(bool separatePG) -{ - this->separatePG = separatePG; -} - - -void Pid::setKillSignal(int signal) -{ - this->killSignal = signal; -} - - -pid_t Pid::release() -{ - pid_t p = pid; - pid = -1; - return p; -} - - -void killUser(uid_t uid) -{ - debug("killing all processes running under uid '%1%'", uid); - - assert(uid != 0); /* just to be safe... */ - - /* The system call kill(-1, sig) sends the signal `sig' to all - users to which the current process can send signals. So we - fork a process, switch to uid, and send a mass kill. */ - - Pid pid = startProcess([&]() { - - if (setuid(uid) == -1) - throw SysError("setting uid"); - - while (true) { -#ifdef __APPLE__ - /* OSX's kill syscall takes a third parameter that, among - other things, determines if kill(-1, signo) affects the - calling process. In the OSX libc, it's set to true, - which means "follow POSIX", which we don't want here - */ - if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; -#else - if (kill(-1, SIGKILL) == 0) break; -#endif - if (errno == ESRCH || errno == EPERM) break; /* no more processes */ - if (errno != EINTR) - throw SysError("cannot kill processes for uid '%1%'", uid); - } - - _exit(0); - }); - - int status = pid.wait(); - if (status != 0) - throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status)); - - /* !!! We should really do some check to make sure that there are - no processes left running under `uid', but there is no portable - way to do so (I think). The most reliable way may be `ps -eo - uid | grep -q $uid'. */ -} - - -////////////////////////////////////////////////////////////////////// - - -/* Wrapper around vfork to prevent the child process from clobbering - the caller's stack frame in the parent. */ -static pid_t doFork(bool allowVfork, std::function fun) __attribute__((noinline)); -static pid_t doFork(bool allowVfork, std::function fun) -{ -#ifdef __linux__ - pid_t pid = allowVfork ? vfork() : fork(); -#else - pid_t pid = fork(); -#endif - if (pid != 0) return pid; - fun(); - abort(); -} - - -#if __linux__ -static int childEntry(void * arg) -{ - auto main = (std::function *) arg; - (*main)(); - return 1; -} -#endif - - -pid_t startProcess(std::function fun, const ProcessOptions & options) -{ - std::function wrapper = [&]() { - if (!options.allowVfork) - logger = makeSimpleLogger(); - try { -#if __linux__ - if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) - throw SysError("setting death signal"); -#endif - fun(); - } catch (std::exception & e) { - try { - std::cerr << options.errorPrefix << e.what() << "\n"; - } catch (...) { } - } catch (...) { } - if (options.runExitHandlers) - exit(1); - else - _exit(1); - }; - - pid_t pid = -1; - - if (options.cloneFlags) { - #ifdef __linux__ - // Not supported, since then we don't know when to free the stack. - assert(!(options.cloneFlags & CLONE_VM)); - - size_t stackSize = 1 * 1024 * 1024; - auto stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - Finally freeStack([&]() { munmap(stack, stackSize); }); - - pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper); - #else - throw Error("clone flags are only supported on Linux"); - #endif - } else - pid = doFork(options.allowVfork, wrapper); - - if (pid == -1) throw SysError("unable to fork"); - - return pid; -} - - std::vector stringsToCharPtrs(const Strings & ss) { std::vector res; @@ -1168,211 +38,6 @@ std::vector stringsToCharPtrs(const Strings & ss) return res; } -std::string runProgram(Path program, bool searchPath, const Strings & args, - const std::optional & input, bool isInteractive) -{ - auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive}); - - if (!statusOk(res.first)) - throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); - - return res.second; -} - -// Output = error code + "standard out" output stream -std::pair runProgram(RunOptions && options) -{ - StringSink sink; - options.standardOut = &sink; - - int status = 0; - - try { - runProgram2(options); - } catch (ExecError & e) { - status = e.status; - } - - return {status, std::move(sink.s)}; -} - -void runProgram2(const RunOptions & options) -{ - checkInterrupt(); - - assert(!(options.standardIn && options.input)); - - std::unique_ptr source_; - Source * source = options.standardIn; - - if (options.input) { - source_ = std::make_unique(*options.input); - source = source_.get(); - } - - /* Create a pipe. */ - Pipe out, in; - if (options.standardOut) out.create(); - if (source) in.create(); - - ProcessOptions processOptions; - // vfork implies that the environment of the main process and the fork will - // be shared (technically this is undefined, but in practice that's the - // case), so we can't use it if we alter the environment - processOptions.allowVfork = !options.environment; - - std::optional>> resumeLoggerDefer; - if (options.isInteractive) { - logger->pause(); - resumeLoggerDefer.emplace( - []() { - logger->resume(); - } - ); - } - - /* Fork. */ - Pid pid = startProcess([&]() { - if (options.environment) - replaceEnv(*options.environment); - if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - if (options.mergeStderrToStdout) - if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) - throw SysError("cannot dup stdout into stderr"); - if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - - if (options.chdir && chdir((*options.chdir).c_str()) == -1) - throw SysError("chdir failed"); - if (options.gid && setgid(*options.gid) == -1) - throw SysError("setgid failed"); - /* Drop all other groups if we're setgid. */ - if (options.gid && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - if (options.uid && setuid(*options.uid) == -1) - throw SysError("setuid failed"); - - Strings args_(options.args); - args_.push_front(options.program); - - restoreProcessContext(); - - if (options.searchPath) - execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); - // This allows you to refer to a program with a pathname relative - // to the PATH variable. - else - execv(options.program.c_str(), stringsToCharPtrs(args_).data()); - - throw SysError("executing '%1%'", options.program); - }, processOptions); - - out.writeSide.close(); - - std::thread writerThread; - - std::promise promise; - - Finally doJoin([&]() { - if (writerThread.joinable()) - writerThread.join(); - }); - - - if (source) { - in.readSide.close(); - writerThread = std::thread([&]() { - try { - std::vector buf(8 * 1024); - while (true) { - size_t n; - try { - n = source->read(buf.data(), buf.size()); - } catch (EndOfFile &) { - break; - } - writeFull(in.writeSide.get(), {buf.data(), n}); - } - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - in.writeSide.close(); - }); - } - - if (options.standardOut) - drainFD(out.readSide.get(), *options.standardOut); - - /* Wait for the child to finish. */ - int status = pid.wait(); - - /* Wait for the writer thread to finish. */ - if (source) promise.get_future().get(); - - if (status) - throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); -} - - -void closeMostFDs(const std::set & exceptions) -{ -#if __linux__ - try { - for (auto & s : readDirectory("/proc/self/fd")) { - auto fd = std::stoi(s.name); - if (!exceptions.count(fd)) { - debug("closing leaked FD %d", fd); - close(fd); - } - } - return; - } catch (SysError &) { - } -#endif - - int maxFD = 0; - maxFD = sysconf(_SC_OPEN_MAX); - for (int fd = 0; fd < maxFD; ++fd) - if (!exceptions.count(fd)) - close(fd); /* ignore result */ -} - - -void closeOnExec(int fd) -{ - int prev; - if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || - fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) - throw SysError("setting close-on-exec flag"); -} - - -////////////////////////////////////////////////////////////////////// - - -std::atomic _isInterrupted = false; - -static thread_local bool interruptThrown = false; -thread_local std::function interruptCheck; - -void setInterruptThrown() -{ - interruptThrown = true; -} - -void _interrupted() -{ - /* Block user interrupts while an exception is being handled. - Throwing an exception while another exception is being handled - kills the program! */ - if (!interruptThrown && !std::uncaught_exceptions()) { - interruptThrown = true; - throw Interrupted("interrupted by the user"); - } -} - ////////////////////////////////////////////////////////////////////// @@ -1438,32 +103,6 @@ std::string rewriteStrings(std::string s, const StringMap & rewrites) } -std::string statusToString(int status) -{ - if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { - if (WIFEXITED(status)) - return fmt("failed with exit code %1%", WEXITSTATUS(status)); - else if (WIFSIGNALED(status)) { - int sig = WTERMSIG(status); -#if HAVE_STRSIGNAL - const char * description = strsignal(sig); - return fmt("failed due to signal %1% (%2%)", sig, description); -#else - return fmt("failed due to signal %1%", sig); -#endif - } - else - return "died abnormally"; - } else return "succeeded"; -} - - -bool statusOk(int status) -{ - return WIFEXITED(status) && WEXITSTATUS(status) == 0; -} - - bool hasPrefix(std::string_view s, std::string_view prefix) { return s.compare(0, prefix.size(), prefix) == 0; @@ -1511,82 +150,6 @@ void ignoreException(Verbosity lvl) } catch (...) { } } -bool shouldANSI() -{ - return isatty(STDERR_FILENO) - && getEnv("TERM").value_or("dumb") != "dumb" - && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value()); -} - -std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) -{ - std::string t, e; - size_t w = 0; - auto i = s.begin(); - - while (w < (size_t) width && i != s.end()) { - - if (*i == '\e') { - std::string e; - e += *i++; - char last = 0; - - if (i != s.end() && *i == '[') { - e += *i++; - // eat parameter bytes - while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; - // eat intermediate bytes - while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; - // eat final byte - if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; - } else { - if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; - } - - if (!filterAll && last == 'm') - t += e; - } - - else if (*i == '\t') { - i++; t += ' '; w++; - while (w < (size_t) width && w % 8) { - t += ' '; w++; - } - } - - else if (*i == '\r' || *i == '\a') - // do nothing for now - i++; - - else { - w++; - // Copy one UTF-8 character. - if ((*i & 0xe0) == 0xc0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } else if ((*i & 0xf0) == 0xe0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } else if ((*i & 0xf8) == 0xf0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } - } else - t += *i++; - } - } - - return t; -} - constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; @@ -1703,386 +266,9 @@ std::pair getLine(std::string_view s) } -////////////////////////////////////////////////////////////////////// - -static Sync> windowSize{{0, 0}}; - - -static void updateWindowSize() -{ - struct winsize ws; - if (ioctl(2, TIOCGWINSZ, &ws) == 0) { - auto windowSize_(windowSize.lock()); - windowSize_->first = ws.ws_row; - windowSize_->second = ws.ws_col; - } -} - - -std::pair getWindowSize() -{ - return *windowSize.lock(); -} - - -/* We keep track of interrupt callbacks using integer tokens, so we can iterate - safely without having to lock the data structure while executing arbitrary - functions. - */ -struct InterruptCallbacks { - typedef int64_t Token; - - /* We use unique tokens so that we can't accidentally delete the wrong - handler because of an erroneous double delete. */ - Token nextToken = 0; - - /* Used as a list, see InterruptCallbacks comment. */ - std::map> callbacks; -}; - -static Sync _interruptCallbacks; - -static void signalHandlerThread(sigset_t set) -{ - while (true) { - int signal = 0; - sigwait(&set, &signal); - - if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) - triggerInterrupt(); - - else if (signal == SIGWINCH) { - updateWindowSize(); - } - } -} - -void triggerInterrupt() -{ - _isInterrupted = true; - - { - InterruptCallbacks::Token i = 0; - while (true) { - std::function callback; - { - auto interruptCallbacks(_interruptCallbacks.lock()); - auto lb = interruptCallbacks->callbacks.lower_bound(i); - if (lb == interruptCallbacks->callbacks.end()) - break; - - callback = lb->second; - i = lb->first + 1; - } - - try { - callback(); - } catch (...) { - ignoreException(); - } - } - } -} - -static sigset_t savedSignalMask; -static bool savedSignalMaskIsSet = false; - -void setChildSignalMask(sigset_t * sigs) -{ - assert(sigs); // C style function, but think of sigs as a reference - -#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE - sigemptyset(&savedSignalMask); - // There's no "assign" or "copy" function, so we rely on (math) idempotence - // of the or operator: a or a = a. - sigorset(&savedSignalMask, sigs, sigs); -#else - // Without sigorset, our best bet is to assume that sigset_t is a type that - // can be assigned directly, such as is the case for a sigset_t defined as - // an integer type. - savedSignalMask = *sigs; -#endif - - savedSignalMaskIsSet = true; -} - -void saveSignalMask() { - if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) - throw SysError("querying signal mask"); - - savedSignalMaskIsSet = true; -} - -void startSignalHandlerThread() -{ - updateWindowSize(); - - saveSignalMask(); - - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGINT); - sigaddset(&set, SIGTERM); - sigaddset(&set, SIGHUP); - sigaddset(&set, SIGPIPE); - sigaddset(&set, SIGWINCH); - if (pthread_sigmask(SIG_BLOCK, &set, nullptr)) - throw SysError("blocking signals"); - - std::thread(signalHandlerThread, set).detach(); -} - -static void restoreSignals() -{ - // If startSignalHandlerThread wasn't called, that means we're not running - // in a proper libmain process, but a process that presumably manages its - // own signal handlers. Such a process should call either - // - initNix(), to be a proper libmain process - // - startSignalHandlerThread(), to resemble libmain regarding signal - // handling only - // - saveSignalMask(), for processes that define their own signal handling - // thread - // TODO: Warn about this? Have a default signal mask? The latter depends on - // whether we should generally inherit signal masks from the caller. - // I don't know what the larger unix ecosystem expects from us here. - if (!savedSignalMaskIsSet) - return; - - if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) - throw SysError("restoring signals"); -} - -#if __linux__ -rlim_t savedStackSize = 0; -#endif - -void setStackSize(size_t stackSize) -{ - #if __linux__ - struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { - savedStackSize = limit.rlim_cur; - limit.rlim_cur = stackSize; - setrlimit(RLIMIT_STACK, &limit); - } - #endif -} - -#if __linux__ -static AutoCloseFD fdSavedMountNamespace; -static AutoCloseFD fdSavedRoot; -#endif - -void saveMountNamespace() -{ -#if __linux__ - static std::once_flag done; - std::call_once(done, []() { - fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); - if (!fdSavedMountNamespace) - throw SysError("saving parent mount namespace"); - - fdSavedRoot = open("/proc/self/root", O_RDONLY); - }); -#endif -} - -void restoreMountNamespace() -{ -#if __linux__ - try { - auto savedCwd = absPath("."); - - if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) - throw SysError("restoring parent mount namespace"); - - if (fdSavedRoot) { - if (fchdir(fdSavedRoot.get())) - throw SysError("chdir into saved root"); - if (chroot(".")) - throw SysError("chroot into saved root"); - } - - if (chdir(savedCwd.c_str()) == -1) - throw SysError("restoring cwd"); - } catch (Error & e) { - debug(e.msg()); - } -#endif -} - -void unshareFilesystem() -{ -#ifdef __linux__ - if (unshare(CLONE_FS) != 0 && errno != EPERM) - throw SysError("unsharing filesystem state in download thread"); -#endif -} - -void restoreProcessContext(bool restoreMounts) -{ - restoreSignals(); - if (restoreMounts) { - restoreMountNamespace(); - } - - #if __linux__ - if (savedStackSize) { - struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0) { - limit.rlim_cur = savedStackSize; - setrlimit(RLIMIT_STACK, &limit); - } - } - #endif -} - -/* RAII helper to automatically deregister a callback. */ -struct InterruptCallbackImpl : InterruptCallback -{ - InterruptCallbacks::Token token; - ~InterruptCallbackImpl() override - { - auto interruptCallbacks(_interruptCallbacks.lock()); - interruptCallbacks->callbacks.erase(token); - } -}; - -std::unique_ptr createInterruptCallback(std::function callback) -{ - auto interruptCallbacks(_interruptCallbacks.lock()); - auto token = interruptCallbacks->nextToken++; - interruptCallbacks->callbacks.emplace(token, callback); - - auto res = std::make_unique(); - res->token = token; - - return std::unique_ptr(res.release()); -} - - -AutoCloseFD createUnixDomainSocket() -{ - AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM - #ifdef SOCK_CLOEXEC - | SOCK_CLOEXEC - #endif - , 0); - if (!fdSocket) - throw SysError("cannot create Unix domain socket"); - closeOnExec(fdSocket.get()); - return fdSocket; -} - - -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) -{ - auto fdSocket = nix::createUnixDomainSocket(); - - bind(fdSocket.get(), path); - - if (chmod(path.c_str(), mode) == -1) - throw SysError("changing permissions on '%1%'", path); - - if (listen(fdSocket.get(), 100) == -1) - throw SysError("cannot listen on socket '%1%'", path); - - return fdSocket; -} - - -void bind(int fd, const std::string & path) -{ - unlink(path.c_str()); - - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - - if (path.size() + 1 >= sizeof(addr.sun_path)) { - Pid pid = startProcess([&]() { - Path dir = dirOf(path); - if (chdir(dir.c_str()) == -1) - throw SysError("chdir to '%s' failed", dir); - std::string base(baseNameOf(path)); - if (base.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%s' is too long", base); - memcpy(addr.sun_path, base.c_str(), base.size() + 1); - if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot bind to socket '%s'", path); - _exit(0); - }); - int status = pid.wait(); - if (status != 0) - throw Error("cannot bind to socket '%s'", path); - } else { - memcpy(addr.sun_path, path.c_str(), path.size() + 1); - if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot bind to socket '%s'", path); - } -} - - -void connect(int fd, const std::string & path) -{ - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - - if (path.size() + 1 >= sizeof(addr.sun_path)) { - Pid pid = startProcess([&]() { - Path dir = dirOf(path); - if (chdir(dir.c_str()) == -1) - throw SysError("chdir to '%s' failed", dir); - std::string base(baseNameOf(path)); - if (base.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%s' is too long", base); - memcpy(addr.sun_path, base.c_str(), base.size() + 1); - if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to socket at '%s'", path); - _exit(0); - }); - int status = pid.wait(); - if (status != 0) - throw Error("cannot connect to socket at '%s'", path); - } else { - memcpy(addr.sun_path, path.c_str(), path.size() + 1); - if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to socket at '%s'", path); - } -} - - std::string showBytes(uint64_t bytes) { return fmt("%.2f MiB", bytes / (1024.0 * 1024.0)); } - -// FIXME: move to libstore/build -void commonChildInit() -{ - logger = makeSimpleLogger(); - - const static std::string pathNullDevice = "/dev/null"; - restoreProcessContext(false); - - /* Put the child in a separate session (and thus a separate - process group) so that it has no controlling terminal (meaning - that e.g. ssh cannot open /dev/tty) and it doesn't receive - terminal signals. */ - if (setsid() == -1) - throw SysError("creating a new session"); - - /* Dup stderr to stdout. */ - if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) - throw SysError("cannot dup stderr into stdout"); - - /* Reroute stdin to /dev/null. */ - int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); - if (fdDevNull == -1) - throw SysError("cannot open '%1%'", pathNullDevice); - if (dup2(fdDevNull, STDIN_FILENO) == -1) - throw SysError("cannot dup null device into stdin"); - close(fdDevNull); -} - } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 75683f8fe..5f730eaf6 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -4,485 +4,18 @@ #include "types.hh" #include "error.hh" #include "logging.hh" -#include "ansicolor.hh" - -#include -#include -#include -#include -#include #include -#include #include #include #include #include -#ifndef HAVE_STRUCT_DIRENT_D_TYPE -#define DT_UNKNOWN 0 -#define DT_REG 1 -#define DT_LNK 2 -#define DT_DIR 3 -#endif - namespace nix { -struct Sink; -struct Source; - void initLibUtil(); -/** - * @return an environment variable. - */ -std::optional getEnv(const std::string & key); - -/** - * @return a non empty environment variable. Returns nullopt if the env - * variable is set to "" - */ -std::optional getEnvNonEmpty(const std::string & key); - -/** - * Get the entire environment. - */ -std::map getEnv(); - -/** - * Clear the environment. - */ -void clearEnv(); - -/** - * @return An absolutized path, resolving paths relative to the - * specified directory, or the current directory otherwise. The path - * is also canonicalised. - */ -Path absPath(Path path, - std::optional dir = {}, - bool resolveSymlinks = false); - -/** - * Canonicalise a path by removing all `.` or `..` components and - * double or trailing slashes. Optionally resolves all symlink - * components such that each component of the resulting path is *not* - * a symbolic link. - */ -Path canonPath(PathView path, bool resolveSymlinks = false); - -/** - * @return The directory part of the given canonical path, i.e., - * everything before the final `/`. If the path is the root or an - * immediate child thereof (e.g., `/foo`), this means `/` - * is returned. - */ -Path dirOf(const PathView path); - -/** - * @return the base name of the given canonical path, i.e., everything - * following the final `/` (trailing slashes are removed). - */ -std::string_view baseNameOf(std::string_view path); - -/** - * Perform tilde expansion on a path. - */ -std::string expandTilde(std::string_view path); - -/** - * Check whether 'path' is a descendant of 'dir'. Both paths must be - * canonicalized. - */ -bool isInDir(std::string_view path, std::string_view dir); - -/** - * Check whether 'path' is equal to 'dir' or a descendant of - * 'dir'. Both paths must be canonicalized. - */ -bool isDirOrInDir(std::string_view path, std::string_view dir); - -/** - * Get status of `path`. - */ -struct stat stat(const Path & path); -struct stat lstat(const Path & path); - -/** - * @return true iff the given path exists. - */ -bool pathExists(const Path & path); - -/** - * A version of pathExists that returns false on a permission error. - * Useful for inferring default paths across directories that might not - * be readable. - * @return true iff the given path can be accessed and exists - */ -bool pathAccessible(const Path & path); - -/** - * Read the contents (target) of a symbolic link. The result is not - * in any way canonicalised. - */ -Path readLink(const Path & path); - -bool isLink(const Path & path); - -/** - * Read the contents of a directory. The entries `.` and `..` are - * removed. - */ -struct DirEntry -{ - std::string name; - ino_t ino; - /** - * one of DT_* - */ - unsigned char type; - DirEntry(std::string name, ino_t ino, unsigned char type) - : name(std::move(name)), ino(ino), type(type) { } -}; - -typedef std::vector DirEntries; - -DirEntries readDirectory(const Path & path); - -unsigned char getFileType(const Path & path); - -/** - * Read the contents of a file into a string. - */ -std::string readFile(int fd); -std::string readFile(const Path & path); -void readFile(const Path & path, Sink & sink); - -/** - * Write a string to a file. - */ -void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); - -void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); - -/** - * Flush a file's parent directory to disk - */ -void syncParent(const Path & path); - -/** - * Read a line from a file descriptor. - */ -std::string readLine(int fd); - -/** - * Write a line to a file descriptor. - */ -void writeLine(int fd, std::string s); - -/** - * Delete a path; i.e., in the case of a directory, it is deleted - * recursively. It's not an error if the path does not exist. The - * second variant returns the number of bytes and blocks freed. - */ -void deletePath(const Path & path); - -void deletePath(const Path & path, uint64_t & bytesFreed); - -std::string getUserName(); - -/** - * @return the given user's home directory from /etc/passwd. - */ -Path getHomeOf(uid_t userId); - -/** - * @return $HOME or the user's home directory from /etc/passwd. - */ -Path getHome(); - -/** - * @return $XDG_CACHE_HOME or $HOME/.cache. - */ -Path getCacheDir(); - -/** - * @return $XDG_CONFIG_HOME or $HOME/.config. - */ -Path getConfigDir(); - -/** - * @return the directories to search for user configuration files - */ -std::vector getConfigDirs(); - -/** - * @return $XDG_DATA_HOME or $HOME/.local/share. - */ -Path getDataDir(); - -/** - * @return the path of the current executable. - */ -std::optional getSelfExe(); - -/** - * @return $XDG_STATE_HOME or $HOME/.local/state. - */ -Path getStateDir(); - -/** - * Create the Nix state directory and return the path to it. - */ -Path createNixStateDir(); - -/** - * Create a directory and all its parents, if necessary. Returns the - * list of created directories, in order of creation. - */ -Paths createDirs(const Path & path); -inline Paths createDirs(PathView path) -{ - return createDirs(Path(path)); -} - -/** - * Create a symlink. - */ -void createSymlink(const Path & target, const Path & link); - -/** - * Atomically create or replace a symlink. - */ -void replaceSymlink(const Path & target, const Path & link); - -void renameFile(const Path & src, const Path & dst); - -/** - * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` - * are on a different filesystem. - * - * Beware that this might not be atomic because of the copy that happens behind - * the scenes - */ -void moveFile(const Path & src, const Path & dst); - - -/** - * Wrappers arount read()/write() that read/write exactly the - * requested number of bytes. - */ -void readFull(int fd, char * buf, size_t count); -void writeFull(int fd, std::string_view s, bool allowInterrupts = true); - -MakeError(EndOfFile, Error); - - -/** - * Read a file descriptor until EOF occurs. - */ -std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); - -void drainFD(int fd, Sink & sink, bool block = true); - -/** - * If cgroups are active, attempt to calculate the number of CPUs available. - * If cgroups are unavailable or if cpu.max is set to "max", return 0. - */ -unsigned int getMaxCPU(); - -/** - * Automatic cleanup of resources. - */ - - -class AutoDelete -{ - Path path; - bool del; - bool recursive; -public: - AutoDelete(); - AutoDelete(const Path & p, bool recursive = true); - ~AutoDelete(); - void cancel(); - void reset(const Path & p, bool recursive = true); - operator Path() const { return path; } - operator PathView() const { return path; } -}; - - -class AutoCloseFD -{ - int fd; -public: - AutoCloseFD(); - AutoCloseFD(int fd); - AutoCloseFD(const AutoCloseFD & fd) = delete; - AutoCloseFD(AutoCloseFD&& fd); - ~AutoCloseFD(); - AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; - AutoCloseFD& operator =(AutoCloseFD&& fd); - int get() const; - explicit operator bool() const; - int release(); - void close(); - void fsync(); -}; - - -/** - * Create a temporary directory. - */ -Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); - -/** - * Create a temporary file, returning a file handle and its path. - */ -std::pair createTempFile(const Path & prefix = "nix"); - - -class Pipe -{ -public: - AutoCloseFD readSide, writeSide; - void create(); - void close(); -}; - - -struct DIRDeleter -{ - void operator()(DIR * dir) const { - closedir(dir); - } -}; - -typedef std::unique_ptr AutoCloseDir; - - -class Pid -{ - pid_t pid = -1; - bool separatePG = false; - int killSignal = SIGKILL; -public: - Pid(); - Pid(pid_t pid); - ~Pid(); - void operator =(pid_t pid); - operator pid_t(); - int kill(); - int wait(); - - void setSeparatePG(bool separatePG); - void setKillSignal(int signal); - pid_t release(); -}; - - -/** - * Kill all processes running under the specified uid by sending them - * a SIGKILL. - */ -void killUser(uid_t uid); - - -/** - * Fork a process that runs the given function, and return the child - * pid to the caller. - */ -struct ProcessOptions -{ - std::string errorPrefix = ""; - bool dieWithParent = true; - bool runExitHandlers = false; - bool allowVfork = false; - /** - * use clone() with the specified flags (Linux only) - */ - int cloneFlags = 0; -}; - -pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); - - -/** - * Run a program and return its stdout in a string (i.e., like the - * shell backtick operator). - */ -std::string runProgram(Path program, bool searchPath = false, - const Strings & args = Strings(), - const std::optional & input = {}, bool isInteractive = false); - -struct RunOptions -{ - Path program; - bool searchPath = true; - Strings args; - std::optional uid; - std::optional gid; - std::optional chdir; - std::optional> environment; - std::optional input; - Source * standardIn = nullptr; - Sink * standardOut = nullptr; - bool mergeStderrToStdout = false; - bool isInteractive = false; -}; - -std::pair runProgram(RunOptions && options); - -void runProgram2(const RunOptions & options); - - -/** - * Change the stack size. - */ -void setStackSize(size_t stackSize); - - -/** - * Restore the original inherited Unix process context (such as signal - * masks, stack size). - - * See startSignalHandlerThread(), saveSignalMask(). - */ -void restoreProcessContext(bool restoreMounts = true); - -/** - * Save the current mount namespace. Ignored if called more than - * once. - */ -void saveMountNamespace(); - -/** - * Restore the mount namespace saved by saveMountNamespace(). Ignored - * if saveMountNamespace() was never called. - */ -void restoreMountNamespace(); - -/** - * Cause this thread to not share any FS attributes with the main - * thread, because this causes setns() in restoreMountNamespace() to - * fail. - */ -void unshareFilesystem(); - - -class ExecError : public Error -{ -public: - int status; - - template - ExecError(int status, const Args & ... args) - : Error(args...), status(status) - { } -}; - /** * Convert a list of strings to a null-terminated vector of `char * *`s. The result must not be accessed beyond the lifetime of the @@ -490,36 +23,6 @@ public: */ std::vector stringsToCharPtrs(const Strings & ss); -/** - * Close all file descriptors except those listed in the given set. - * Good practice in child processes. - */ -void closeMostFDs(const std::set & exceptions); - -/** - * Set the close-on-exec flag for the given file descriptor. - */ -void closeOnExec(int fd); - - -/* User interruption. */ - -extern std::atomic _isInterrupted; - -extern thread_local std::function interruptCheck; - -void setInterruptThrown(); - -void _interrupted(); - -void inline checkInterrupt() -{ - if (_isInterrupted || (interruptCheck && interruptCheck())) - _interrupted(); -} - -MakeError(Interrupted, BaseError); - MakeError(FormatError, Error); @@ -595,15 +98,6 @@ std::string replaceStrings( std::string rewriteStrings(std::string s, const StringMap & rewrites); -/** - * Convert the exit status of a child as returned by wait() into an - * error string. - */ -std::string statusToString(int status); - -bool statusOk(int status); - - /** * Parse a string into an integer. */ @@ -711,23 +205,6 @@ constexpr char treeLast[] = "└───"; constexpr char treeLine[] = "│ "; constexpr char treeNull[] = " "; -/** - * Determine whether ANSI escape sequences are appropriate for the - * present output. - */ -bool shouldANSI(); - -/** - * Truncate a string to 'width' printable characters. If 'filterAll' - * is true, all ANSI escape sequences are filtered out. Otherwise, - * some escape sequences (such as colour setting) are copied but not - * included in the character count. Also, tabs are expanded to - * spaces. - */ -std::string filterANSIEscapes(std::string_view s, - bool filterAll = false, - unsigned int width = std::numeric_limits::max()); - /** * Base64 encoding/decoding. @@ -815,61 +292,6 @@ template class Callback; -/** - * Start a thread that handles various signals. Also block those signals - * on the current thread (and thus any threads created by it). - * Saves the signal mask before changing the mask to block those signals. - * See saveSignalMask(). - */ -void startSignalHandlerThread(); - -/** - * Saves the signal mask, which is the signal mask that nix will restore - * before creating child processes. - * See setChildSignalMask() to set an arbitrary signal mask instead of the - * current mask. - */ -void saveSignalMask(); - -/** - * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't - * necessarily match the current thread's mask. - * See saveSignalMask() to set the saved mask to the current mask. - */ -void setChildSignalMask(sigset_t *sigs); - -struct InterruptCallback -{ - virtual ~InterruptCallback() { }; -}; - -/** - * Register a function that gets called on SIGINT (in a non-signal - * context). - */ -std::unique_ptr createInterruptCallback( - std::function callback); - -void triggerInterrupt(); - -/** - * A RAII class that causes the current thread to receive SIGUSR1 when - * the signal handler thread receives SIGINT. That is, this allows - * SIGINT to be multiplexed to multiple threads. - */ -struct ReceiveInterrupts -{ - pthread_t target; - std::unique_ptr callback; - - ReceiveInterrupts() - : target(pthread_self()) - , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) - { } -}; - - - /** * A RAII helper that increments a counter on construction and * decrements it on destruction. @@ -884,45 +306,6 @@ struct MaintainCount }; -/** - * @return the number of rows and columns of the terminal. - */ -std::pair getWindowSize(); - - -/** - * Used in various places. - */ -typedef std::function PathFilter; - -extern PathFilter defaultPathFilter; - -/** - * Common initialisation performed in child processes. - */ -void commonChildInit(); - -/** - * Create a Unix domain socket. - */ -AutoCloseFD createUnixDomainSocket(); - -/** - * Create a Unix domain socket in listen mode. - */ -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); - -/** - * Bind a Unix domain socket to a path. - */ -void bind(int fd, const std::string & path); - -/** - * Connect to a Unix domain socket. - */ -void connect(int fd, const std::string & path); - - /** * A Rust/Python-like enumerate() iterator adapter. * diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 60bc08146..75ce12a8c 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -9,12 +9,12 @@ #include +#include "current-process.hh" #include "parsed-derivations.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" #include "derivations.hh" -#include "util.hh" #include "shared.hh" #include "path-with-outputs.hh" #include "eval.hh" diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 4504441fa..79db78236 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -5,7 +5,7 @@ #include "store-api.hh" #include "legacy.hh" #include "eval-settings.hh" // for defexpr -#include "util.hh" +#include "users.hh" #include "tarball.hh" #include diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 70af53b28..bb3f1bc6a 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -1,3 +1,5 @@ +#include "file-system.hh" +#include "signals.hh" #include "store-api.hh" #include "store-cast.hh" #include "gc-store.hh" diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 25068f801..213a20d93 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1,3 +1,4 @@ +#include "users.hh" #include "attr-path.hh" #include "common-eval-args.hh" #include "derivations.hh" @@ -11,7 +12,6 @@ #include "store-api.hh" #include "local-fs-store.hh" #include "user-env.hh" -#include "util.hh" #include "value-to-json.hh" #include "xml-writer.hh" #include "legacy.hh" diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index d12d70f33..250224e7d 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -1,5 +1,4 @@ #include "user-env.hh" -#include "util.hh" #include "derivations.hh" #include "store-api.hh" #include "path-with-outputs.hh" diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index d40196497..c67409e89 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -6,7 +6,6 @@ #include "attr-path.hh" #include "value-to-xml.hh" #include "value-to-json.hh" -#include "util.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "common-eval-args.hh" diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 577cadceb..2c530999b 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -1,5 +1,4 @@ #include "dotgraph.hh" -#include "util.hh" #include "store-api.hh" #include diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 439557658..3e789a2d8 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -1,5 +1,4 @@ #include "graphml.hh" -#include "util.hh" #include "store-api.hh" #include "derivations.hh" diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index e4dd94585..123283dfe 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -11,7 +11,6 @@ #include "serve-protocol.hh" #include "serve-protocol-impl.hh" #include "shared.hh" -#include "util.hh" #include "graphml.hh" #include "legacy.hh" #include "path-with-outputs.hh" diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index af428018a..373dedf7c 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -1,11 +1,12 @@ ///@file +#include "signals.hh" +#include "unix-domain-socket.hh" #include "command.hh" #include "shared.hh" #include "local-store.hh" #include "remote-store.hh" #include "remote-store-connection.hh" -#include "util.hh" #include "serialise.hh" #include "archive.hh" #include "globals.hh" diff --git a/src/nix/develop.cc b/src/nix/develop.cc index b080a3939..38482ed42 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -8,7 +8,6 @@ #include "derivations.hh" #include "progress-bar.hh" #include "run.hh" -#include "util.hh" #include #include diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 1aa6831d3..59f9e3e5d 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -6,7 +6,6 @@ #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "util.hh" #include "worker-protocol.hh" using namespace nix; diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 66629fab0..9cbab230b 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -1,3 +1,4 @@ +#include "current-process.hh" #include "command-installable-value.hh" #include "shared.hh" #include "eval.hh" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index e8906a252..38938f09e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -15,6 +15,7 @@ #include "registry.hh" #include "eval-cache.hh" #include "markdown.hh" +#include "users.hh" #include #include diff --git a/src/nix/main.cc b/src/nix/main.cc index d20bc1f8a..b582fc166 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -1,6 +1,8 @@ #include #include "args/root.hh" +#include "current-process.hh" +#include "namespaces.hh" #include "command.hh" #include "common-args.hh" #include "eval.hh" diff --git a/src/nix/run.cc b/src/nix/run.cc index 1465e8cde..ea0a17897 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -1,3 +1,4 @@ +#include "current-process.hh" #include "run.hh" #include "command-installable-value.hh" #include "common-args.hh" diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 45cd2e1a6..a68616355 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -1,3 +1,4 @@ +#include "signals.hh" #include "command.hh" #include "shared.hh" #include "store-api.hh" diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index d238456db..c529c2363 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -1,3 +1,4 @@ +#include "processes.hh" #include "command.hh" #include "common-args.hh" #include "store-api.hh" diff --git a/src/nix/verify.cc b/src/nix/verify.cc index adaa33c0c..78cb765ce 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -4,6 +4,7 @@ #include "sync.hh" #include "thread-pool.hh" #include "references.hh" +#include "signals.hh" #include From 6472c3bf0d4b529f28f9e50834e1fc3dd101c409 Mon Sep 17 00:00:00 2001 From: ThinkChaos Date: Sat, 4 Nov 2023 12:08:00 -0400 Subject: [PATCH 531/909] fix(ssh): extraneous master processes --- src/libstore/ssh.cc | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 03b2f0be9..300eb391c 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -132,7 +132,6 @@ Path SSHMaster::startMaster() if (state->sshMaster != -1) return state->socketPath; - state->socketPath = (Path) *state->tmpDir + "/ssh.sock"; Pipe out; @@ -144,7 +143,8 @@ Path SSHMaster::startMaster() logger->pause(); Finally cleanup = [&]() { logger->resume(); }; - bool wasMasterRunning = isMasterRunning(); + if (isMasterRunning()) + return state->socketPath; state->sshMaster = startProcess([&]() { restoreProcessContext(); @@ -165,14 +165,13 @@ Path SSHMaster::startMaster() out.writeSide = -1; - if (!wasMasterRunning) { - std::string reply; - try { - reply = readLine(out.readSide.get()); - } catch (EndOfFile & e) { } + std::string reply; + try { + reply = readLine(out.readSide.get()); + } catch (EndOfFile & e) { } - if (reply != "started") - throw Error("failed to start SSH master connection to '%s'", host); + if (reply != "started") { + throw Error("failed to start SSH master connection to '%s'", host); } return state->socketPath; From 2fb49759b8307838dd1208d8ce756a60d41e4ebf Mon Sep 17 00:00:00 2001 From: ThinkChaos Date: Sat, 4 Nov 2023 12:32:57 -0400 Subject: [PATCH 532/909] fix(ssh): log first line of stdout Spent a while debugging why `nix-copy-closure` wasn't working anymore and it was my shell RC printing something I added for debug. Hopefully this can save someone else some time. --- src/libstore/ssh.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 300eb391c..5c8d6a504 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -114,8 +114,10 @@ std::unique_ptr SSHMaster::startCommand(const std::string reply = readLine(out.readSide.get()); } catch (EndOfFile & e) { } - if (reply != "started") + if (reply != "started") { + printTalkative("SSH stdout first line: %s", reply); throw Error("failed to start SSH connection to '%s'", host); + } } conn->out = std::move(out.readSide); @@ -171,6 +173,7 @@ Path SSHMaster::startMaster() } catch (EndOfFile & e) { } if (reply != "started") { + printTalkative("SSH master stdout first line: %s", reply); throw Error("failed to start SSH master connection to '%s'", host); } From 0b0d1b521449e7a66e7fa33ca7afe292d88aa14b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 26 Oct 2023 19:39:21 -0400 Subject: [PATCH 533/909] Add comparison functions for `NarInfo` We will need these for tests. --- src/libstore/nar-info.cc | 9 +++++++++ src/libstore/nar-info.hh | 2 ++ 2 files changed, 11 insertions(+) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index ee2ddfd81..2b77c6ab7 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -4,6 +4,15 @@ namespace nix { +GENERATE_CMP_EXT( + , + NarInfo, + me->url, + me->compression, + me->fileHash, + me->fileSize, + static_cast(*me)); + NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack { diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 5dbdafac3..1b3551106 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -24,6 +24,8 @@ struct NarInfo : ValidPathInfo NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); + DECLARE_CMP(NarInfo); + std::string to_string(const Store & store) const; }; From 07ac53732b8989758c264d4e847c94a5d28072cf Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Sun, 5 Nov 2023 15:27:25 +0100 Subject: [PATCH 534/909] Fix moves in appendOrSet --- src/libutil/config-impl.hh | 10 +++++----- src/libutil/config.cc | 20 +++++++++----------- src/libutil/config.hh | 2 +- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index b9639e761..9f69e8444 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -45,13 +45,13 @@ bool BaseSetting::isAppendable() return trait::appendable; } -template<> void BaseSetting::appendOrSet(Strings && newValue, bool append); -template<> void BaseSetting::appendOrSet(StringSet && newValue, bool append); -template<> void BaseSetting::appendOrSet(StringMap && newValue, bool append); -template<> void BaseSetting>::appendOrSet(std::set && newValue, bool append); +template<> void BaseSetting::appendOrSet(Strings newValue, bool append); +template<> void BaseSetting::appendOrSet(StringSet newValue, bool append); +template<> void BaseSetting::appendOrSet(StringMap newValue, bool append); +template<> void BaseSetting>::appendOrSet(std::set newValue, bool append); template -void BaseSetting::appendOrSet(T && newValue, bool append) +void BaseSetting::appendOrSet(T newValue, bool append) { static_assert( !trait::appendable, diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 0bf36c987..5b510b69e 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -301,10 +301,11 @@ template<> Strings BaseSetting::parse(const std::string & str) const return tokenizeString(str); } -template<> void BaseSetting::appendOrSet(Strings && newValue, bool append) +template<> void BaseSetting::appendOrSet(Strings newValue, bool append) { if (!append) value.clear(); - for (auto && s : std::move(newValue)) value.push_back(std::move(s)); + value.insert(value.end(), std::make_move_iterator(newValue.begin()), + std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting::to_string() const @@ -317,11 +318,10 @@ template<> StringSet BaseSetting::parse(const std::string & str) cons return tokenizeString(str); } -template<> void BaseSetting::appendOrSet(StringSet && newValue, bool append) +template<> void BaseSetting::appendOrSet(StringSet newValue, bool append) { if (!append) value.clear(); - for (auto && s : std::move(newValue)) - value.insert(s); + value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting::to_string() const @@ -342,11 +342,10 @@ template<> std::set BaseSetting void BaseSetting>::appendOrSet(std::set && newValue, bool append) +template<> void BaseSetting>::appendOrSet(std::set newValue, bool append) { if (!append) value.clear(); - for (auto && s : std::move(newValue)) - value.insert(s); + value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting>::to_string() const @@ -369,11 +368,10 @@ template<> StringMap BaseSetting::parse(const std::string & str) cons return res; } -template<> void BaseSetting::appendOrSet(StringMap && newValue, bool append) +template<> void BaseSetting::appendOrSet(StringMap newValue, bool append) { if (!append) value.clear(); - for (auto && [k, v] : std::move(newValue)) - value.emplace(std::move(k), std::move(v)); + value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting::to_string() const diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 38c3ce0c4..d9441fb63 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -247,7 +247,7 @@ protected: * * @param append Whether to append or overwrite. */ - virtual void appendOrSet(T && newValue, bool append); + virtual void appendOrSet(T newValue, bool append); public: From ad385f9ec44f8d845e994764c45876042c715946 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Sun, 5 Nov 2023 15:27:48 +0100 Subject: [PATCH 535/909] Minor improvements --- src/libutil/config.hh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index d9441fb63..3f2522c38 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -150,7 +150,7 @@ public: AbstractSetting * setting; }; - typedef std::map Settings; + using Settings = std::map; private: @@ -316,7 +316,7 @@ std::ostream & operator <<(std::ostream & str, const BaseSetting & opt) } template -bool operator ==(const T & v1, const BaseSetting & v2) { return v1 == (const T &) v2; } +bool operator ==(const T & v1, const BaseSetting & v2) { return v1 == static_cast(v2); } template class Setting : public BaseSetting @@ -329,7 +329,7 @@ public: const std::set & aliases = {}, const bool documentDefault = true, std::optional experimentalFeature = std::nullopt) - : BaseSetting(def, documentDefault, name, description, aliases, experimentalFeature) + : BaseSetting(def, documentDefault, name, description, aliases, std::move(experimentalFeature)) { options->addSetting(this); } From f404e9b3b362a054219797df02bbe277de249f80 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Sun, 5 Nov 2023 16:12:20 +0100 Subject: [PATCH 536/909] Make toJSONObject const --- src/libutil/abstract-setting-to-json.hh | 2 +- src/libutil/config.cc | 2 +- src/libutil/config.hh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libutil/abstract-setting-to-json.hh b/src/libutil/abstract-setting-to-json.hh index d506dfb74..eea687d8a 100644 --- a/src/libutil/abstract-setting-to-json.hh +++ b/src/libutil/abstract-setting-to-json.hh @@ -7,7 +7,7 @@ namespace nix { template -std::map BaseSetting::toJSONObject() +std::map BaseSetting::toJSONObject() const { auto obj = AbstractSetting::toJSONObject(); obj.emplace("value", value); diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 5b510b69e..2a5cf6212 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -221,7 +221,7 @@ nlohmann::json AbstractSetting::toJSON() return nlohmann::json(toJSONObject()); } -std::map AbstractSetting::toJSONObject() +std::map AbstractSetting::toJSONObject() const { std::map obj; obj.emplace("description", description); diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 3f2522c38..5d7bd8e0c 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -213,7 +213,7 @@ protected: nlohmann::json toJSON(); - virtual std::map toJSONObject(); + virtual std::map toJSONObject() const; virtual void convertToArg(Args & args, const std::string & category); @@ -306,7 +306,7 @@ public: void convertToArg(Args & args, const std::string & category) override; - std::map toJSONObject() override; + std::map toJSONObject() const override; }; template From a4b7df7bfaee7d27a152be2445886c81881daf94 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Mon, 6 Nov 2023 15:47:25 +0100 Subject: [PATCH 537/909] More const, scope reductions, move fixes --- src/libutil/config.cc | 60 +++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 2a5cf6212..8e7901133 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -36,27 +36,25 @@ bool Config::set(const std::string & name, const std::string & value) void Config::addSetting(AbstractSetting * setting) { _settings.emplace(setting->name, Config::SettingData{false, setting}); - for (auto & alias : setting->aliases) + for (const auto & alias : setting->aliases) _settings.emplace(alias, Config::SettingData{true, setting}); bool set = false; - auto i = unknownSettings.find(setting->name); - if (i != unknownSettings.end()) { - setting->set(i->second); + if (auto i = unknownSettings.find(setting->name); i != unknownSettings.end()) { + setting->set(std::move(i->second)); setting->overridden = true; unknownSettings.erase(i); set = true; } for (auto & alias : setting->aliases) { - auto i = unknownSettings.find(alias); - if (i != unknownSettings.end()) { + if (auto i = unknownSettings.find(alias); i != unknownSettings.end()) { if (set) warn("setting '%s' is set, but it's an alias of '%s' which is also set", alias, setting->name); else { - setting->set(i->second); + setting->set(std::move(i->second)); setting->overridden = true; unknownSettings.erase(i); set = true; @@ -71,7 +69,7 @@ AbstractConfig::AbstractConfig(StringMap initials) void AbstractConfig::warnUnknownSettings() { - for (auto & s : unknownSettings) + for (const auto & s : unknownSettings) warn("unknown setting '%s'", s.first); } @@ -85,7 +83,7 @@ void AbstractConfig::reapplyUnknownSettings() void Config::getSettings(std::map & res, bool overriddenOnly) { - for (auto & opt : _settings) + for (const auto & opt : _settings) if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden)) res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } @@ -101,8 +99,7 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string line += contents[pos++]; pos++; - auto hash = line.find('#'); - if (hash != std::string::npos) + if (auto hash = line.find('#'); hash != line.npos) line = std::string(line, 0, hash); auto tokens = tokenizeString>(line); @@ -135,24 +132,24 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string if (tokens[1] != "=") throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); - std::string name = tokens[0]; + std::string name = std::move(tokens[0]); auto i = tokens.begin(); advance(i, 2); parsedContents.push_back({ - name, + std::move(name), concatStringsSep(" ", Strings(i, tokens.end())), }); }; // First apply experimental-feature related settings - for (auto & [name, value] : parsedContents) + for (const auto & [name, value] : parsedContents) if (name == "experimental-features" || name == "extra-experimental-features") set(name, value); // Then apply other settings - for (auto & [name, value] : parsedContents) + for (const auto & [name, value] : parsedContents) if (name != "experimental-features" && name != "extra-experimental-features") set(name, value); } @@ -174,7 +171,7 @@ void Config::resetOverridden() nlohmann::json Config::toJSON() { auto res = nlohmann::json::object(); - for (auto & s : _settings) + for (const auto & s : _settings) if (!s.second.isAlias) res.emplace(s.first, s.second.setting->toJSON()); return res; @@ -182,8 +179,8 @@ nlohmann::json Config::toJSON() std::string Config::toKeyValue() { - auto res = std::string(); - for (auto & s : _settings) + std::string res; + for (const auto & s : _settings) if (s.second.isAlias) res += fmt("%s = %s\n", s.first, s.second.setting->to_string()); return res; @@ -205,7 +202,7 @@ AbstractSetting::AbstractSetting( : name(name) , description(stripIndentation(description)) , aliases(aliases) - , experimentalFeature(experimentalFeature) + , experimentalFeature(std::move(experimentalFeature)) { } @@ -284,14 +281,14 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & .longName = name, .description = fmt("Enable the `%s` setting.", name), .category = category, - .handler = {[this]() { override(true); }}, + .handler = {[this] { override(true); }}, .experimentalFeature = experimentalFeature, }); args.addFlag({ .longName = "no-" + name, .description = fmt("Disable the `%s` setting.", name), .category = category, - .handler = {[this]() { override(false); }}, + .handler = {[this] { override(false); }}, .experimentalFeature = experimentalFeature, }); } @@ -333,8 +330,7 @@ template<> std::set BaseSetting res; for (auto & s : tokenizeString(str)) { - auto thisXpFeature = parseExperimentalFeature(s); - if (thisXpFeature) + if (auto thisXpFeature = parseExperimentalFeature(s); thisXpFeature) res.insert(thisXpFeature.value()); else warn("unknown experimental feature '%s'", s); @@ -351,7 +347,7 @@ template<> void BaseSetting>::appendOrSet(std::set template<> std::string BaseSetting>::to_string() const { StringSet stringifiedXpFeatures; - for (auto & feature : value) + for (const auto & feature : value) stringifiedXpFeatures.insert(std::string(showExperimentalFeature(feature))); return concatStringsSep(" ", stringifiedXpFeatures); } @@ -359,9 +355,8 @@ template<> std::string BaseSetting>::to_string() c template<> StringMap BaseSetting::parse(const std::string & str) const { StringMap res; - for (auto & s : tokenizeString(str)) { - auto eq = s.find_first_of('='); - if (std::string::npos != eq) + for (const auto & s : tokenizeString(str)) { + if (auto eq = s.find_first_of('='); s.npos != eq) res.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); // else ignored } @@ -376,10 +371,9 @@ template<> void BaseSetting::appendOrSet(StringMap newValue, bool app template<> std::string BaseSetting::to_string() const { - Strings kvstrs; - std::transform(value.begin(), value.end(), back_inserter(kvstrs), - [&](auto kvpair){ return kvpair.first + "=" + kvpair.second; }); - return concatStringsSep(" ", kvstrs); + return std::transform_reduce(value.cbegin(), value.cend(), std::string{}, + [](const auto & l, const auto &r) { return l + " " + r; }, + [](const auto & kvpair){ return kvpair.first + "=" + kvpair.second; }); } template class BaseSetting; @@ -468,7 +462,7 @@ void GlobalConfig::resetOverridden() nlohmann::json GlobalConfig::toJSON() { auto res = nlohmann::json::object(); - for (auto & config : *configRegistrations) + for (const auto & config : *configRegistrations) res.update(config->toJSON()); return res; } @@ -478,7 +472,7 @@ std::string GlobalConfig::toKeyValue() std::string res; std::map settings; globalConfig.getSettings(settings); - for (auto & s : settings) + for (const auto & s : settings) res += fmt("%s = %s\n", s.first, s.second.value); return res; } From 937e02e7b9538fd4500ade184eb4f0a888a9967d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 22 Oct 2023 21:12:54 -0400 Subject: [PATCH 538/909] Shuffle `ValidPathInfo` JSON rendering `Store::pathInfoToJSON` was a rather baroque functions, being full of parameters to support both parsed derivations and `nix path-info`. The common core of each, a simple `dValidPathInfo::toJSON` function, is factored out, but the rest of the logic is just duplicated and then specialized to its use-case (at which point it is no longer that duplicated). This keeps the human oriented CLI logic (which is currently unstable) and the core domain logic (export reference graphs with structured attrs, which is stable), separate, which I think is better. --- src/libstore/nar-info.cc | 46 +++++++++ src/libstore/nar-info.hh | 9 ++ src/libstore/parsed-derivations.cc | 34 ++++++- src/libstore/path-info.cc | 99 +++++++++++++++++++ src/libstore/path-info.hh | 12 +++ src/libstore/store-api.cc | 90 ----------------- src/libstore/store-api.hh | 23 ----- src/libstore/tests/nar-info.cc | 84 ++++++++++++++++ src/libstore/tests/path-info.cc | 79 +++++++++++++++ src/libutil/tests/characterization.hh | 1 + src/nix/path-info.cc | 80 ++++++++++++++- unit-test-data/libstore/nar-info/impure.json | 21 ++++ unit-test-data/libstore/nar-info/pure.json | 11 +++ unit-test-data/libstore/path-info/impure.json | 18 ++++ unit-test-data/libstore/path-info/pure.json | 11 +++ 15 files changed, 499 insertions(+), 119 deletions(-) create mode 100644 src/libstore/tests/nar-info.cc create mode 100644 src/libstore/tests/path-info.cc create mode 100644 unit-test-data/libstore/nar-info/impure.json create mode 100644 unit-test-data/libstore/nar-info/pure.json create mode 100644 unit-test-data/libstore/path-info/impure.json create mode 100644 unit-test-data/libstore/path-info/pure.json diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 2b77c6ab7..a90812ff9 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -134,4 +134,50 @@ std::string NarInfo::to_string(const Store & store) const return res; } +nlohmann::json NarInfo::toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const +{ + using nlohmann::json; + + auto jsonObject = ValidPathInfo::toJSON(store, includeImpureInfo, hashFormat); + + if (includeImpureInfo) { + if (!url.empty()) + jsonObject["url"] = url; + if (fileHash) + jsonObject["downloadHash"] = fileHash->to_string(hashFormat, true); + if (fileSize) + jsonObject["downloadSize"] = fileSize; + } + + return jsonObject; +} + +NarInfo NarInfo::fromJSON( + const Store & store, + const StorePath & path, + const nlohmann::json & json) +{ + using nlohmann::detail::value_t; + + NarInfo res { ValidPathInfo::fromJSON(store, json) }; + res.path = path; + + if (json.contains("url")) + res.url = ensureType(valueAt(json, "url"), value_t::string); + + if (json.contains("downloadHash")) + res.fileHash = Hash::parseAny( + static_cast( + ensureType(valueAt(json, "downloadHash"), value_t::string)), + std::nullopt); + + if (json.contains("downloadSize")) + res.fileSize = ensureType(valueAt(json, "downloadSize"), value_t::number_integer); + + return res; +} + } diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 1b3551106..cec65ff70 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -27,6 +27,15 @@ struct NarInfo : ValidPathInfo DECLARE_CMP(NarInfo); std::string to_string(const Store & store) const; + + nlohmann::json toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const override; + static NarInfo fromJSON( + const Store & store, + const StorePath & path, + const nlohmann::json & json); }; } diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 1d900c272..45629dc7f 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -132,6 +132,36 @@ bool ParsedDerivation::useUidRange() const static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); +/** + * Write a JSON representation of store object metadata, such as the + * hash and the references. + */ +static nlohmann::json pathInfoToJSON( + Store & store, + const StorePathSet & storePaths) +{ + nlohmann::json::array_t jsonList = nlohmann::json::array(); + + for (auto & storePath : storePaths) { + auto info = store.queryPathInfo(storePath); + + auto & jsonPath = jsonList.emplace_back( + info->toJSON(store, false, HashFormat::Base32)); + + jsonPath["closureSize"] = ({ + uint64_t totalNarSize = 0; + StorePathSet closure; + store.computeFSClosure(info->path, closure, false, false); + for (auto & p : closure) { + auto info = store.queryPathInfo(p); + totalNarSize += info->narSize; + } + totalNarSize; + }); + } + return jsonList; +} + std::optional ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths) { auto structuredAttrs = getStructuredAttrs(); @@ -152,8 +182,8 @@ std::optional ParsedDerivation::prepareStructuredAttrs(Store & s StorePathSet storePaths; for (auto & p : *i) storePaths.insert(store.parseStorePath(p.get())); - json[i.key()] = store.pathInfoToJSON( - store.exportReferences(storePaths, inputPaths), false, true); + json[i.key()] = pathInfoToJSON(store, + store.exportReferences(storePaths, inputPaths)); } } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index ab39e71f4..e5d5205f4 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -1,5 +1,8 @@ +#include + #include "path-info.hh" #include "store-api.hh" +#include "json-utils.hh" namespace nix { @@ -144,4 +147,100 @@ ValidPathInfo::ValidPathInfo( }, std::move(ca).raw); } + +nlohmann::json ValidPathInfo::toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const +{ + using nlohmann::json; + + auto jsonObject = json::object(); + + jsonObject["path"] = store.printStorePath(path); + jsonObject["valid"] = true; + jsonObject["narHash"] = narHash.to_string(hashFormat, true); + jsonObject["narSize"] = narSize; + + { + auto& jsonRefs = (jsonObject["references"] = json::array()); + for (auto & ref : references) + jsonRefs.emplace_back(store.printStorePath(ref)); + } + + if (ca) + jsonObject["ca"] = renderContentAddress(ca); + + if (includeImpureInfo) { + if (deriver) + jsonObject["deriver"] = store.printStorePath(*deriver); + + if (registrationTime) + jsonObject["registrationTime"] = registrationTime; + + if (ultimate) + jsonObject["ultimate"] = ultimate; + + if (!sigs.empty()) { + for (auto & sig : sigs) + jsonObject["signatures"].push_back(sig); + } + } + + return jsonObject; +} + +ValidPathInfo ValidPathInfo::fromJSON( + const Store & store, + const nlohmann::json & json) +{ + using nlohmann::detail::value_t; + + ValidPathInfo res { + StorePath(StorePath::dummy), + Hash(Hash::dummy), + }; + + ensureType(json, value_t::object); + res.path = store.parseStorePath( + static_cast( + ensureType(valueAt(json, "path"), value_t::string))); + res.narHash = Hash::parseAny( + static_cast( + ensureType(valueAt(json, "narHash"), value_t::string)), + std::nullopt); + res.narSize = ensureType(valueAt(json, "narSize"), value_t::number_integer); + + try { + auto & references = ensureType(valueAt(json, "references"), value_t::array); + for (auto & input : references) + res.references.insert(store.parseStorePath(static_cast +(input))); + } catch (Error & e) { + e.addTrace({}, "while reading key 'references'"); + throw; + } + + if (json.contains("ca")) + res.ca = ContentAddress::parse( + static_cast( + ensureType(valueAt(json, "ca"), value_t::string))); + + if (json.contains("deriver")) + res.deriver = store.parseStorePath( + static_cast( + ensureType(valueAt(json, "deriver"), value_t::string))); + + if (json.contains("registrationTime")) + res.registrationTime = ensureType(valueAt(json, "registrationTime"), value_t::number_integer); + + if (json.contains("ultimate")) + res.ultimate = ensureType(valueAt(json, "ultimate"), value_t::boolean); + + if (json.contains("signatures")) + res.sigs = valueAt(json, "signatures"); + + return res; +} + } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index c4c4a6366..feeda6c27 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -125,6 +125,18 @@ struct ValidPathInfo : UnkeyedValidPathInfo { Strings shortRefs() const; + /** + * @param includeImpureInfo If true, variable elements such as the + * registration time are included. + */ + virtual nlohmann::json toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const; + static ValidPathInfo fromJSON( + const Store & store, + const nlohmann::json & json); + ValidPathInfo(const ValidPathInfo & other) = default; ValidPathInfo(StorePath && path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(std::move(path)) { }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c9ebb6c14..0f88d9b92 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -951,96 +951,6 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor return paths; } -json Store::pathInfoToJSON(const StorePathSet & storePaths, - bool includeImpureInfo, bool showClosureSize, - HashFormat hashFormat, - AllowInvalidFlag allowInvalid) -{ - json::array_t jsonList = json::array(); - - for (auto & storePath : storePaths) { - auto& jsonPath = jsonList.emplace_back(json::object()); - - try { - auto info = queryPathInfo(storePath); - - jsonPath["path"] = printStorePath(info->path); - jsonPath["valid"] = true; - jsonPath["narHash"] = info->narHash.to_string(hashFormat, true); - jsonPath["narSize"] = info->narSize; - - { - auto& jsonRefs = (jsonPath["references"] = json::array()); - for (auto & ref : info->references) - jsonRefs.emplace_back(printStorePath(ref)); - } - - if (info->ca) - jsonPath["ca"] = renderContentAddress(info->ca); - - std::pair closureSizes; - - if (showClosureSize) { - closureSizes = getClosureSize(info->path); - jsonPath["closureSize"] = closureSizes.first; - } - - if (includeImpureInfo) { - - if (info->deriver) - jsonPath["deriver"] = printStorePath(*info->deriver); - - if (info->registrationTime) - jsonPath["registrationTime"] = info->registrationTime; - - if (info->ultimate) - jsonPath["ultimate"] = info->ultimate; - - if (!info->sigs.empty()) { - for (auto & sig : info->sigs) - jsonPath["signatures"].push_back(sig); - } - - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - - if (narInfo) { - if (!narInfo->url.empty()) - jsonPath["url"] = narInfo->url; - if (narInfo->fileHash) - jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashFormat, true); - if (narInfo->fileSize) - jsonPath["downloadSize"] = narInfo->fileSize; - if (showClosureSize) - jsonPath["closureDownloadSize"] = closureSizes.second; - } - } - - } catch (InvalidPath &) { - jsonPath["path"] = printStorePath(storePath); - jsonPath["valid"] = false; - } - } - return jsonList; -} - - -std::pair Store::getClosureSize(const StorePath & storePath) -{ - uint64_t totalNarSize = 0, totalDownloadSize = 0; - StorePathSet closure; - computeFSClosure(storePath, closure, false, false); - for (auto & p : closure) { - auto info = queryPathInfo(p); - totalNarSize += info->narSize; - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - if (narInfo) - totalDownloadSize += narInfo->fileSize; - } - return {totalNarSize, totalDownloadSize}; -} - const Store::Stats & Store::getStats() { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6aa317e3d..32ad2aa44 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -80,7 +80,6 @@ typedef std::map OutputPathMap; enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; -enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; /** * Magic header of exportPath() output (obsolete). @@ -665,28 +664,6 @@ public: std::string makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash); - /** - * Write a JSON representation of store path metadata, such as the - * hash and the references. - * - * @param includeImpureInfo If true, variable elements such as the - * registration time are included. - * - * @param showClosureSize If true, the closure size of each path is - * included. - */ - nlohmann::json pathInfoToJSON(const StorePathSet & storePaths, - bool includeImpureInfo, bool showClosureSize, - HashFormat hashFormat = HashFormat::Base32, - AllowInvalidFlag allowInvalid = DisallowInvalid); - - /** - * @return the size of the closure of the specified path, that is, - * the sum of the size of the NAR serialisation of each path in the - * closure. - */ - std::pair getClosureSize(const StorePath & storePath); - /** * Optimise the disk space usage of the Nix store by hard-linking files * with the same contents. diff --git a/src/libstore/tests/nar-info.cc b/src/libstore/tests/nar-info.cc new file mode 100644 index 000000000..cb92f3a28 --- /dev/null +++ b/src/libstore/tests/nar-info.cc @@ -0,0 +1,84 @@ +#include +#include + +#include "path-info.hh" + +#include "tests/characterization.hh" +#include "tests/libstore.hh" + +namespace nix { + +using nlohmann::json; + +class NarInfoTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/libstore/nar-info"; + + Path goldenMaster(PathView testStem) const override { + return unitTestData + "/" + testStem + ".json"; + } +}; + +static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) { + NarInfo info = ValidPathInfo { + store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.narSize = 34878; + if (includeImpureInfo) { + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.registrationTime = 23423; + info.ultimate = true; + info.sigs = { "asdf", "qwer" }; + + info.url = "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz"; + info.fileHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="); + info.fileSize = 4029176; + } + return info; +} + +#define JSON_TEST(STEM, PURE) \ + TEST_F(NarInfoTest, NarInfo_ ## STEM ## _from_json) { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + auto expected = makeNarInfo(*store, PURE); \ + NarInfo got = NarInfo::fromJSON( \ + *store, \ + expected.path, \ + encoded); \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(NarInfoTest, NarInfo_ ## STEM ## _to_json) { \ + writeTest(#STEM, [&]() -> json { \ + return makeNarInfo(*store, PURE) \ + .toJSON(*store, PURE, HashFormat::SRI); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ + } + +JSON_TEST(pure, false) +JSON_TEST(impure, true) + +} diff --git a/src/libstore/tests/path-info.cc b/src/libstore/tests/path-info.cc new file mode 100644 index 000000000..fbee751c6 --- /dev/null +++ b/src/libstore/tests/path-info.cc @@ -0,0 +1,79 @@ +#include +#include + +#include "path-info.hh" + +#include "tests/characterization.hh" +#include "tests/libstore.hh" + +namespace nix { + +using nlohmann::json; + +class PathInfoTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/libstore/path-info"; + + Path goldenMaster(PathView testStem) const override { + return unitTestData + "/" + testStem + ".json"; + } +}; + +static ValidPathInfo makePathInfo(const Store & store, bool includeImpureInfo) { + ValidPathInfo info { + store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.narSize = 34878; + if (includeImpureInfo) { + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.registrationTime = 23423; + info.ultimate = true; + info.sigs = { "asdf", "qwer" }; + } + return info; +} + +#define JSON_TEST(STEM, PURE) \ + TEST_F(PathInfoTest, PathInfo_ ## STEM ## _from_json) { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + ValidPathInfo got = ValidPathInfo::fromJSON( \ + *store, \ + encoded); \ + auto expected = makePathInfo(*store, PURE); \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(PathInfoTest, PathInfo_ ## STEM ## _to_json) { \ + writeTest(#STEM, [&]() -> json { \ + return makePathInfo(*store, PURE) \ + .toJSON(*store, PURE, HashFormat::SRI); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ + } + +JSON_TEST(pure, false) +JSON_TEST(impure, true) + +} diff --git a/src/libutil/tests/characterization.hh b/src/libutil/tests/characterization.hh index 6698c5239..6eb513d68 100644 --- a/src/libutil/tests/characterization.hh +++ b/src/libutil/tests/characterization.hh @@ -4,6 +4,7 @@ #include #include "types.hh" +#include "environment-variables.hh" namespace nix { diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index c16864d30..b4bdd15ba 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -9,6 +9,74 @@ #include using namespace nix; +using nlohmann::json; + +/** + * @return the total size of a set of store objects (specified by path), + * that is, the sum of the size of the NAR serialisation of each object + * in the set. + */ +static uint64_t getStoreObjectsTotalSize(Store & store, const StorePathSet & closure) +{ + uint64_t totalNarSize = 0; + for (auto & p : closure) { + totalNarSize += store.queryPathInfo(p)->narSize; + } + return totalNarSize; +} + + +/** + * Write a JSON representation of store object metadata, such as the + * hash and the references. + * + * @param showClosureSize If true, the closure size of each path is + * included. + */ +static json pathInfoToJSON( + Store & store, + const StorePathSet & storePaths, + bool showClosureSize) +{ + json::array_t jsonList = json::array(); + + for (auto & storePath : storePaths) { + try { + auto info = store.queryPathInfo(storePath); + + auto & jsonPath = jsonList.emplace_back( + info->toJSON(store, true, HashFormat::SRI)); + + if (showClosureSize) { + StorePathSet closure; + store.computeFSClosure(storePath, closure, false, false); + + jsonPath["closureSize"] = getStoreObjectsTotalSize(store, closure); + + if (auto * narInfo = dynamic_cast(&*info)) { + uint64_t totalDownloadSize = 0; + for (auto & p : closure) { + auto depInfo = store.queryPathInfo(p); + if (auto * depNarInfo = dynamic_cast(&*depInfo)) + totalDownloadSize += depNarInfo->fileSize; + else + throw Error("Missing .narinfo for dep %s of %s", + store.printStorePath(p), + store.printStorePath(storePath)); + } + jsonPath["closureDownloadSize"] = totalDownloadSize; + } + } + + } catch (InvalidPath &) { + auto & jsonPath = jsonList.emplace_back(json::object()); + jsonPath["path"] = store.printStorePath(storePath); + jsonPath["valid"] = false; + } + } + return jsonList; +} + struct CmdPathInfo : StorePathsCommand, MixJSON { @@ -87,10 +155,11 @@ struct CmdPathInfo : StorePathsCommand, MixJSON pathLen = std::max(pathLen, store->printStorePath(storePath).size()); if (json) { - std::cout << store->pathInfoToJSON( + std::cout << pathInfoToJSON( + *store, // FIXME: preserve order? StorePathSet(storePaths.begin(), storePaths.end()), - true, showClosureSize, HashFormat::SRI, AllowInvalid).dump(); + showClosureSize).dump(); } else { @@ -107,8 +176,11 @@ struct CmdPathInfo : StorePathsCommand, MixJSON if (showSize) printSize(info->narSize); - if (showClosureSize) - printSize(store->getClosureSize(info->path).first); + if (showClosureSize) { + StorePathSet closure; + store->computeFSClosure(storePath, closure, false, false); + printSize(getStoreObjectsTotalSize(*store, closure)); + } if (showSigs) { std::cout << '\t'; diff --git a/unit-test-data/libstore/nar-info/impure.json b/unit-test-data/libstore/nar-info/impure.json new file mode 100644 index 000000000..093f25025 --- /dev/null +++ b/unit-test-data/libstore/nar-info/impure.json @@ -0,0 +1,21 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "downloadHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "downloadSize": 4029176, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo", + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [ + "asdf", + "qwer" + ], + "ultimate": true, + "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz", + "valid": true +} diff --git a/unit-test-data/libstore/nar-info/pure.json b/unit-test-data/libstore/nar-info/pure.json new file mode 100644 index 000000000..62005d414 --- /dev/null +++ b/unit-test-data/libstore/nar-info/pure.json @@ -0,0 +1,11 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo", + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "valid": true +} diff --git a/unit-test-data/libstore/path-info/impure.json b/unit-test-data/libstore/path-info/impure.json new file mode 100644 index 000000000..c477c768c --- /dev/null +++ b/unit-test-data/libstore/path-info/impure.json @@ -0,0 +1,18 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo", + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [ + "asdf", + "qwer" + ], + "ultimate": true, + "valid": true +} diff --git a/unit-test-data/libstore/path-info/pure.json b/unit-test-data/libstore/path-info/pure.json new file mode 100644 index 000000000..62005d414 --- /dev/null +++ b/unit-test-data/libstore/path-info/pure.json @@ -0,0 +1,11 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo", + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "valid": true +} From a7212e169b7204f80ea67f60c855d05b72b5d4f7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 26 Oct 2023 20:01:36 -0400 Subject: [PATCH 539/909] Include `compression` in the `NarInfo` JSON format It was forgotten before. --- src/libstore/nar-info.cc | 5 +++++ src/libstore/tests/nar-info.cc | 1 + unit-test-data/libstore/nar-info/impure.json | 1 + 3 files changed, 7 insertions(+) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index a90812ff9..708cc7341 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -146,6 +146,8 @@ nlohmann::json NarInfo::toJSON( if (includeImpureInfo) { if (!url.empty()) jsonObject["url"] = url; + if (!compression.empty()) + jsonObject["compression"] = compression; if (fileHash) jsonObject["downloadHash"] = fileHash->to_string(hashFormat, true); if (fileSize) @@ -168,6 +170,9 @@ NarInfo NarInfo::fromJSON( if (json.contains("url")) res.url = ensureType(valueAt(json, "url"), value_t::string); + if (json.contains("compression")) + res.compression = ensureType(valueAt(json, "compression"), value_t::string); + if (json.contains("downloadHash")) res.fileHash = Hash::parseAny( static_cast( diff --git a/src/libstore/tests/nar-info.cc b/src/libstore/tests/nar-info.cc index cb92f3a28..c5b21d56b 100644 --- a/src/libstore/tests/nar-info.cc +++ b/src/libstore/tests/nar-info.cc @@ -48,6 +48,7 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) { info.sigs = { "asdf", "qwer" }; info.url = "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz"; + info.compression = "xz"; info.fileHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="); info.fileSize = 4029176; } diff --git a/unit-test-data/libstore/nar-info/impure.json b/unit-test-data/libstore/nar-info/impure.json index 093f25025..3f16667c9 100644 --- a/unit-test-data/libstore/nar-info/impure.json +++ b/unit-test-data/libstore/nar-info/impure.json @@ -1,5 +1,6 @@ { "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "compression": "xz", "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", "downloadHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "downloadSize": 4029176, From cc46ea163024254d0b74646e1b38b19896d40040 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 22 Oct 2023 21:12:54 -0400 Subject: [PATCH 540/909] Make `nix path-info --json` return an object not array Before it returned a list of JSON objects with store object information, including the path in each object. Now, it maps the paths to JSON objects with the metadata sans path. This matches how `nix derivation show` works. Quite hillariously, none of our existing functional tests caught this change to `path-info --json` though they did use it. So just new functional tests need to be added. --- doc/manual/src/release-notes/rl-next.md | 36 ++++++++++++++++++- src/libstore/nar-info.cc | 8 +++-- src/libstore/parsed-derivations.cc | 5 +++ src/libstore/path-info.cc | 12 ++----- src/libstore/path-info.hh | 24 ++++++------- src/libstore/tests/path-info.cc | 6 ++-- src/nix/path-info.cc | 19 +++++----- src/nix/path-info.md | 12 +++---- tests/functional/local.mk | 1 + tests/functional/path-info.sh | 23 ++++++++++++ unit-test-data/libstore/nar-info/impure.json | 4 +-- unit-test-data/libstore/nar-info/pure.json | 4 +-- unit-test-data/libstore/path-info/impure.json | 4 +-- unit-test-data/libstore/path-info/pure.json | 4 +-- 14 files changed, 108 insertions(+), 54 deletions(-) create mode 100644 tests/functional/path-info.sh diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 8cd69f8fd..85e180e37 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -28,5 +28,39 @@ - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. They are superceded by `nix flake update`. - + - Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + +- [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) + (experimental) now returns a JSON map rather than JSON list. + The `path` field of each object has instead become the key in th outer map, since it is unique. + The `valid` field also goes away because we just use null instead. + + - Old way: + + ```json5 + [ + { + "path": "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15", + "valid": true, + // ... + }, + { + "path": "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path", + "valid": false + } + ] + ``` + + - New way + + ```json5 + { + "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15": { + // ... + }, + "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path": null, + } + ``` + + This makes it match `nix derivation show`, which also maps store paths to information. diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 708cc7341..ae2223fb0 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -164,8 +164,12 @@ NarInfo NarInfo::fromJSON( { using nlohmann::detail::value_t; - NarInfo res { ValidPathInfo::fromJSON(store, json) }; - res.path = path; + NarInfo res { + ValidPathInfo { + path, + UnkeyedValidPathInfo::fromJSON(store, json), + } + }; if (json.contains("url")) res.url = ensureType(valueAt(json, "url"), value_t::string); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 45629dc7f..73e55a96c 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -148,6 +148,11 @@ static nlohmann::json pathInfoToJSON( auto & jsonPath = jsonList.emplace_back( info->toJSON(store, false, HashFormat::Base32)); + // Add the path to the object whose metadata we are including. + jsonPath["path"] = store.printStorePath(storePath); + + jsonPath["valid"] = true; + jsonPath["closureSize"] = ({ uint64_t totalNarSize = 0; StorePathSet closure; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index e5d5205f4..2d7dc972f 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -148,7 +148,7 @@ ValidPathInfo::ValidPathInfo( } -nlohmann::json ValidPathInfo::toJSON( +nlohmann::json UnkeyedValidPathInfo::toJSON( const Store & store, bool includeImpureInfo, HashFormat hashFormat) const @@ -157,8 +157,6 @@ nlohmann::json ValidPathInfo::toJSON( auto jsonObject = json::object(); - jsonObject["path"] = store.printStorePath(path); - jsonObject["valid"] = true; jsonObject["narHash"] = narHash.to_string(hashFormat, true); jsonObject["narSize"] = narSize; @@ -190,21 +188,17 @@ nlohmann::json ValidPathInfo::toJSON( return jsonObject; } -ValidPathInfo ValidPathInfo::fromJSON( +UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON( const Store & store, const nlohmann::json & json) { using nlohmann::detail::value_t; - ValidPathInfo res { - StorePath(StorePath::dummy), + UnkeyedValidPathInfo res { Hash(Hash::dummy), }; ensureType(json, value_t::object); - res.path = store.parseStorePath( - static_cast( - ensureType(valueAt(json, "path"), value_t::string))); res.narHash = Hash::parseAny( static_cast( ensureType(valueAt(json, "narHash"), value_t::string)), diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index feeda6c27..077abc7e1 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -78,6 +78,18 @@ struct UnkeyedValidPathInfo DECLARE_CMP(UnkeyedValidPathInfo); virtual ~UnkeyedValidPathInfo() { } + + /** + * @param includeImpureInfo If true, variable elements such as the + * registration time are included. + */ + virtual nlohmann::json toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const; + static UnkeyedValidPathInfo fromJSON( + const Store & store, + const nlohmann::json & json); }; struct ValidPathInfo : UnkeyedValidPathInfo { @@ -125,18 +137,6 @@ struct ValidPathInfo : UnkeyedValidPathInfo { Strings shortRefs() const; - /** - * @param includeImpureInfo If true, variable elements such as the - * registration time are included. - */ - virtual nlohmann::json toJSON( - const Store & store, - bool includeImpureInfo, - HashFormat hashFormat) const; - static ValidPathInfo fromJSON( - const Store & store, - const nlohmann::json & json); - ValidPathInfo(const ValidPathInfo & other) = default; ValidPathInfo(StorePath && path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(std::move(path)) { }; diff --git a/src/libstore/tests/path-info.cc b/src/libstore/tests/path-info.cc index fbee751c6..49bf623bd 100644 --- a/src/libstore/tests/path-info.cc +++ b/src/libstore/tests/path-info.cc @@ -19,8 +19,8 @@ class PathInfoTest : public CharacterizationTest, public LibStoreTest } }; -static ValidPathInfo makePathInfo(const Store & store, bool includeImpureInfo) { - ValidPathInfo info { +static UnkeyedValidPathInfo makePathInfo(const Store & store, bool includeImpureInfo) { + UnkeyedValidPathInfo info = ValidPathInfo { store, "foo", FixedOutputInfo { @@ -54,7 +54,7 @@ static ValidPathInfo makePathInfo(const Store & store, bool includeImpureInfo) { TEST_F(PathInfoTest, PathInfo_ ## STEM ## _from_json) { \ readTest(#STEM, [&](const auto & encoded_) { \ auto encoded = json::parse(encoded_); \ - ValidPathInfo got = ValidPathInfo::fromJSON( \ + UnkeyedValidPathInfo got = UnkeyedValidPathInfo::fromJSON( \ *store, \ encoded); \ auto expected = makePathInfo(*store, PURE); \ diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index b4bdd15ba..23198a120 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -38,20 +38,21 @@ static json pathInfoToJSON( const StorePathSet & storePaths, bool showClosureSize) { - json::array_t jsonList = json::array(); + json::object_t jsonAllObjects = json::object(); for (auto & storePath : storePaths) { + json jsonObject; + try { auto info = store.queryPathInfo(storePath); - auto & jsonPath = jsonList.emplace_back( - info->toJSON(store, true, HashFormat::SRI)); + jsonObject = info->toJSON(store, true, HashFormat::SRI); if (showClosureSize) { StorePathSet closure; store.computeFSClosure(storePath, closure, false, false); - jsonPath["closureSize"] = getStoreObjectsTotalSize(store, closure); + jsonObject["closureSize"] = getStoreObjectsTotalSize(store, closure); if (auto * narInfo = dynamic_cast(&*info)) { uint64_t totalDownloadSize = 0; @@ -64,17 +65,17 @@ static json pathInfoToJSON( store.printStorePath(p), store.printStorePath(storePath)); } - jsonPath["closureDownloadSize"] = totalDownloadSize; + jsonObject["closureDownloadSize"] = totalDownloadSize; } } } catch (InvalidPath &) { - auto & jsonPath = jsonList.emplace_back(json::object()); - jsonPath["path"] = store.printStorePath(storePath); - jsonPath["valid"] = false; + jsonObject = nullptr; } + + jsonAllObjects[store.printStorePath(storePath)] = std::move(jsonObject); } - return jsonList; + return jsonAllObjects; } diff --git a/src/nix/path-info.md b/src/nix/path-info.md index 2dda866d0..4594854eb 100644 --- a/src/nix/path-info.md +++ b/src/nix/path-info.md @@ -43,7 +43,7 @@ R""( command): ```console - # nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path' + # nix path-info --json --all | jq -r 'to_entries | sort_by(.value.registrationTime) | .[-11:-1][] | .key' ``` * Show the size of the entire Nix store: @@ -58,13 +58,13 @@ R""( ```console # nix path-info --json --all --closure-size \ - | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])' + | jq 'map_values(.closureSize | select(. < 1e9)) | to_entries | sort_by(.value)' [ …, - [ - "/nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65", - 5887562256 - ] + { + .key = "/nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65", + .value = 5887562256, + } ] ``` diff --git a/tests/functional/local.mk b/tests/functional/local.mk index fe0d0c4ed..21dabca88 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -120,6 +120,7 @@ nix_tests = \ flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ + path-info.sh \ toString-path.sh \ read-only-store.sh \ nested-sandboxing.sh \ diff --git a/tests/functional/path-info.sh b/tests/functional/path-info.sh new file mode 100644 index 000000000..763935eb7 --- /dev/null +++ b/tests/functional/path-info.sh @@ -0,0 +1,23 @@ +source common.sh + +echo foo > $TEST_ROOT/foo +foo=$(nix store add-file $TEST_ROOT/foo) + +echo bar > $TEST_ROOT/bar +bar=$(nix store add-file $TEST_ROOT/bar) + +echo baz > $TEST_ROOT/baz +baz=$(nix store add-file $TEST_ROOT/baz) +nix-store --delete "$baz" + +diff --unified --color=always \ + <(nix path-info --json "$foo" "$bar" "$baz" | + jq --sort-keys 'map_values(.narHash)') \ + <(jq --sort-keys <<-EOF + { + "$foo": "sha256-QvtAMbUl/uvi+LCObmqOhvNOapHdA2raiI4xG5zI5pA=", + "$bar": "sha256-9fhYGu9fqxcQC2Kc81qh2RMo1QcLBUBo8U+pPn+jthQ=", + "$baz": null + } +EOF + ) diff --git a/unit-test-data/libstore/nar-info/impure.json b/unit-test-data/libstore/nar-info/impure.json index 3f16667c9..bb9791a6a 100644 --- a/unit-test-data/libstore/nar-info/impure.json +++ b/unit-test-data/libstore/nar-info/impure.json @@ -6,7 +6,6 @@ "downloadSize": 4029176, "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, - "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo", "references": [ "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" @@ -17,6 +16,5 @@ "qwer" ], "ultimate": true, - "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz", - "valid": true + "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz" } diff --git a/unit-test-data/libstore/nar-info/pure.json b/unit-test-data/libstore/nar-info/pure.json index 62005d414..955baec31 100644 --- a/unit-test-data/libstore/nar-info/pure.json +++ b/unit-test-data/libstore/nar-info/pure.json @@ -2,10 +2,8 @@ "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, - "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo", "references": [ "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" - ], - "valid": true + ] } diff --git a/unit-test-data/libstore/path-info/impure.json b/unit-test-data/libstore/path-info/impure.json index c477c768c..0c452cc49 100644 --- a/unit-test-data/libstore/path-info/impure.json +++ b/unit-test-data/libstore/path-info/impure.json @@ -3,7 +3,6 @@ "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, - "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo", "references": [ "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" @@ -13,6 +12,5 @@ "asdf", "qwer" ], - "ultimate": true, - "valid": true + "ultimate": true } diff --git a/unit-test-data/libstore/path-info/pure.json b/unit-test-data/libstore/path-info/pure.json index 62005d414..955baec31 100644 --- a/unit-test-data/libstore/path-info/pure.json +++ b/unit-test-data/libstore/path-info/pure.json @@ -2,10 +2,8 @@ "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", "narSize": 34878, - "path": "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo", "references": [ "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" - ], - "valid": true + ] } From 61d6fe059e959455e156c1d57bb91155d363e983 Mon Sep 17 00:00:00 2001 From: Mel Zuser Date: Mon, 6 Nov 2023 14:13:40 -0500 Subject: [PATCH 541/909] Fix `boost::bad_format_string` exception in `builtins.addErrorContext` (#9291) * Fix boost::bad_format_string exception in builtins.addErrorContext The message passed to addTrace was incorrectly being used as a format string and this this would cause an exception when the string contained a '%', which can be hit in places where arbitrary file paths are interpolated. * add test --- src/libexpr/primops.cc | 2 +- tests/functional/lang.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 36340d0f9..8d3a18526 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -825,7 +825,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * * auto message = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtins.addErrorContext", false, false).toOwned(); - e.addTrace(nullptr, message, true); + e.addTrace(nullptr, hintfmt(message), true); throw; } } diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh index c3acef5ee..12df32c87 100755 --- a/tests/functional/lang.sh +++ b/tests/functional/lang.sh @@ -23,6 +23,7 @@ nix-instantiate --trace-verbose --eval -E 'builtins.traceVerbose "Hello" 123' 2> nix-instantiate --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grepQuietInverse Hello nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grepQuietInverse Hello expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' | grepQuiet Hello +expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello %" (throw "Foo")' | grepQuiet 'Hello %' nix-instantiate --eval -E 'let x = builtins.trace { x = x; } true; in x' \ 2>&1 | grepQuiet -E 'trace: { x = «potential infinite recursion»; }' From 867f894289437a96630579592a46a4253151f079 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Fri, 15 Sep 2023 10:49:30 -0700 Subject: [PATCH 542/909] Populate $XDG_DATA_DIRS with appropriate folder from Nix profile On non-NixOS systems, the default `nix` install does not populate the `$XDG_DATA_DIRS`. This populates it and enables things like bash-completion and `.desktop` file detection for `nix` profile installed packages. Signed-off-by: Ana Hobden --- scripts/nix-profile-daemon.fish.in | 8 ++++++++ scripts/nix-profile-daemon.sh.in | 8 ++++++++ scripts/nix-profile.fish.in | 8 ++++++++ scripts/nix-profile.sh.in | 8 ++++++++ 4 files changed, 32 insertions(+) diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index 400696812..e7b394d56 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -19,6 +19,14 @@ set __ETC_PROFILE_NIX_SOURCED 1 set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" +# Populate bash completions, .desktop files, etc +if test -n "$NIX_SSH_CERT_FILE" + # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default + set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" +else + set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share" +end + # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if test -n "$NIX_SSH_CERT_FILE" : # Allow users to override the NIX_SSL_CERT_FILE diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 8cfd3149e..3089cec66 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -30,6 +30,14 @@ fi export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" +# Populate bash completions, .desktop files, etc +if [ -n "${XDG_DATA_DIRS-}" ]; then + # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default + export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" +else + export XDG_DATA_DIRS="$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share" +fi + # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ -n "${NIX_SSL_CERT_FILE:-}" ]; then : # Allow users to override the NIX_SSL_CERT_FILE diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in index 731498c76..fc8fe4e97 100644 --- a/scripts/nix-profile.fish.in +++ b/scripts/nix-profile.fish.in @@ -20,6 +20,14 @@ if test -n "$HOME" && test -n "$USER" # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" + # Populate bash completions, .desktop files, etc + if test -n "$NIX_SSH_CERT_FILE" + # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default + set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" + else + set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share" + end + # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if test -n "$NIX_SSH_CERT_FILE" : # Allow users to override the NIX_SSL_CERT_FILE diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index c4d60cf37..a0d098588 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -32,6 +32,14 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" + # Populate bash completions, .desktop files, etc + if [ -n "${XDG_DATA_DIRS-}" ]; then + # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default + export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" + else + export XDG_DATA_DIRS="$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share" + fi + # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt From 896013ec0c0d4633349ff0373bdae626667adc77 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Thu, 21 Sep 2023 09:27:35 -0700 Subject: [PATCH 543/909] Fix bad copy-paste --- scripts/nix-profile-daemon.fish.in | 2 +- scripts/nix-profile.fish.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index e7b394d56..5f5a53141 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -20,7 +20,7 @@ set __ETC_PROFILE_NIX_SOURCED 1 set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" # Populate bash completions, .desktop files, etc -if test -n "$NIX_SSH_CERT_FILE" +if test -n "$XDG_DATA_DIRS" # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" else diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in index fc8fe4e97..2523594f2 100644 --- a/scripts/nix-profile.fish.in +++ b/scripts/nix-profile.fish.in @@ -21,7 +21,7 @@ if test -n "$HOME" && test -n "$USER" set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" # Populate bash completions, .desktop files, etc - if test -n "$NIX_SSH_CERT_FILE" + if test -n "$XDG_DATA_DIRS" # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" else From 150b5aba509d169a50c6ad62100c3ad7bf00242b Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Mon, 6 Nov 2023 10:07:53 -0800 Subject: [PATCH 544/909] Update scripts/nix-profile-daemon.fish.in Co-authored-by: Valentin Gagarin --- scripts/nix-profile-daemon.fish.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index 5f5a53141..3fe9e782a 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -22,9 +22,9 @@ set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profi # Populate bash completions, .desktop files, etc if test -n "$XDG_DATA_DIRS" # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default - set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" + set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:/nix/var/nix/profiles/default/share" else - set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share" + set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:/nix/var/nix/profiles/default/share" end # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. From 257b768436a0e8ab7887f9b790c5b92a7fe51ef5 Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Sun, 9 Jul 2023 22:16:21 +0200 Subject: [PATCH 545/909] Enable using human-readable name in nix profile --- doc/manual/src/release-notes/rl-next.md | 2 + src/libutil/tests/url-name.cc | 64 ++++++++++ src/libutil/url-name.cc | 46 +++++++ src/libutil/url-name.hh | 20 +++ src/nix/profile-list.md | 10 +- src/nix/profile-remove.md | 15 +-- src/nix/profile-upgrade.md | 10 +- src/nix/profile.cc | 160 ++++++++++++++++-------- tests/functional/nix-profile.sh | 21 ++-- 9 files changed, 274 insertions(+), 74 deletions(-) create mode 100644 src/libutil/tests/url-name.cc create mode 100644 src/libutil/url-name.cc create mode 100644 src/libutil/url-name.hh diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 85e180e37..d50da32cd 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -64,3 +64,5 @@ ``` This makes it match `nix derivation show`, which also maps store paths to information. + +- [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) now uses names to refer to installed packages when running [`list`](@docroot@/command-ref/new-cli/nix3-profile-list.md), [`remove`](@docroot@/command-ref/new-cli/nix3-profile-remove.md) or [`upgrade`](@docroot@/command-ref/new-cli/nix3-profile-upgrade.md) as opposed to indices. Indices are deprecated and will be removed in a future version. diff --git a/src/libutil/tests/url-name.cc b/src/libutil/tests/url-name.cc new file mode 100644 index 000000000..6ee66e826 --- /dev/null +++ b/src/libutil/tests/url-name.cc @@ -0,0 +1,64 @@ +#include "url-name.hh" +#include + +namespace nix { + +/* ----------- tests for url-name.hh --------------------------------------------------*/ + + TEST(getNameFromURL, getsNameFromURL) { + ASSERT_EQ(getNameFromURL(parseURL("path:/home/user/project")), "project"); + ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("path:.#nonStandardAttr.mylaptop")), "nonStandardAttr.mylaptop"); + ASSERT_EQ(getNameFromURL(parseURL("path:./repos/myflake#nonStandardAttr.mylaptop")), "nonStandardAttr.mylaptop"); + + ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nixpkgs#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nixpkgs#hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nix#packages.x86_64-linux.default")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nix#")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nix")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("github:cachix/devenv/main#packages.x86_64-linux.default")), "devenv"); + ASSERT_EQ(getNameFromURL(parseURL("github:edolstra/nix-warez?rev=1234&dir=blender&ref=master")), "blender"); + + ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nixpkgs#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nixpkgs#hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nix#packages.x86_64-linux.default")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nix#")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nix")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("gitlab:cachix/devenv/main#packages.x86_64-linux.default")), "devenv"); + + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nixpkgs#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nixpkgs#hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nix#packages.x86_64-linux.default")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nix#")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nix")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:cachix/devenv/main#packages.x86_64-linux.default")), "devenv"); + + ASSERT_EQ(getNameFromURL(parseURL("git://github.com/edolstra/dwarffs")), "dwarffs"); + ASSERT_EQ(getNameFromURL(parseURL("git://github.com/edolstra/nix-warez?dir=blender")), "blender"); + ASSERT_EQ(getNameFromURL(parseURL("git+file:///home/user/project")), "project"); + ASSERT_EQ(getNameFromURL(parseURL("git+file:///home/user/project?ref=fa1e2d23a22")), "project"); + ASSERT_EQ(getNameFromURL(parseURL("git+ssh://git@github.com/someuser/my-repo#")), "my-repo"); + ASSERT_EQ(getNameFromURL(parseURL("git+git://github.com/someuser/my-repo?rev=v1.2.3")), "my-repo"); + ASSERT_EQ(getNameFromURL(parseURL("git+ssh:///home/user/project?dir=subproject&rev=v2.4")), "subproject"); + ASSERT_EQ(getNameFromURL(parseURL("git+http://not-even-real#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("git+https://not-even-real#packages.aarch64-darwin.hello")), "hello"); + + ASSERT_EQ(getNameFromURL(parseURL("tarball+http://github.com/NixOS/nix/archive/refs/tags/2.18.1#packages.x86_64-linux.jq")), "jq"); + ASSERT_EQ(getNameFromURL(parseURL("tarball+https://github.com/NixOS/nix/archive/refs/tags/2.18.1#packages.x86_64-linux.hg")), "hg"); + ASSERT_EQ(getNameFromURL(parseURL("tarball+file:///home/user/Downloads/nixpkgs-2.18.1#packages.aarch64-darwin.ripgrep")), "ripgrep"); + + ASSERT_EQ(getNameFromURL(parseURL("https://github.com/NixOS/nix/archive/refs/tags/2.18.1.tar.gz#packages.x86_64-linux.pv")), "pv"); + ASSERT_EQ(getNameFromURL(parseURL("http://github.com/NixOS/nix/archive/refs/tags/2.18.1.tar.gz#packages.x86_64-linux.pv")), "pv"); + + ASSERT_EQ(getNameFromURL(parseURL("file:///home/user/project?ref=fa1e2d23a22")), "project"); + ASSERT_EQ(getNameFromURL(parseURL("file+file:///home/user/project?ref=fa1e2d23a22")), "project"); + ASSERT_EQ(getNameFromURL(parseURL("file+http://not-even-real#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("file+http://gitfantasy.com/org/user/notaflake")), "notaflake"); + ASSERT_EQ(getNameFromURL(parseURL("file+https://not-even-real#packages.aarch64-darwin.hello")), "hello"); + + ASSERT_EQ(getNameFromURL(parseURL("https://www.github.com/")), std::nullopt); + ASSERT_EQ(getNameFromURL(parseURL("path:.")), std::nullopt); + ASSERT_EQ(getNameFromURL(parseURL("file:.#")), std::nullopt); + ASSERT_EQ(getNameFromURL(parseURL("path:.#packages.x86_64-linux.default")), std::nullopt); + } +} diff --git a/src/libutil/url-name.cc b/src/libutil/url-name.cc new file mode 100644 index 000000000..ab65e78df --- /dev/null +++ b/src/libutil/url-name.cc @@ -0,0 +1,46 @@ +#include "url-name.hh" +#include +#include + +namespace nix { + +static const std::string attributeNamePattern("[a-z0-9_-]+"); +static const std::regex lastAttributeRegex("(?:" + attributeNamePattern + "\\.)*(?!default)(" + attributeNamePattern +")"); +static const std::string pathSegmentPattern("[a-zA-Z0-9_-]+"); +static const std::regex lastPathSegmentRegex(".*/(" + pathSegmentPattern +")"); +static const std::regex secondPathSegmentRegex("(?:" + pathSegmentPattern + ")/(" + pathSegmentPattern +")(?:/.*)?"); +static const std::regex gitProviderRegex("github|gitlab|sourcehut"); +static const std::regex gitSchemeRegex("git($|\\+.*)"); + +std::optional getNameFromURL(ParsedURL url) { + std::smatch match; + + /* If there is a dir= argument, use its value */ + if (url.query.count("dir") > 0) + return url.query.at("dir"); + + /* If the fragment isn't a "default" and contains two attribute elements, use the last one */ + if (std::regex_match(url.fragment, match, lastAttributeRegex)) + return match.str(1); + + /* If this is a github/gitlab/sourcehut flake, use the repo name */ + if (std::regex_match(url.scheme, gitProviderRegex) && std::regex_match(url.path, match, secondPathSegmentRegex)) + return match.str(1); + + /* If it is a regular git flake, use the directory name */ + if (std::regex_match(url.scheme, gitSchemeRegex) && std::regex_match(url.path, match, lastPathSegmentRegex)) + return match.str(1); + + /* If everything failed but there is a non-default fragment, use it in full */ + if (!url.fragment.empty() && !hasSuffix(url.fragment, "default")) + return url.fragment; + + /* If there is no fragment, take the last element of the path */ + if (std::regex_match(url.path, match, lastPathSegmentRegex)) + return match.str(1); + + /* If even that didn't work, the URL does not contain enough info to determine a useful name */ + return {}; +} + +} diff --git a/src/libutil/url-name.hh b/src/libutil/url-name.hh new file mode 100644 index 000000000..188b951e5 --- /dev/null +++ b/src/libutil/url-name.hh @@ -0,0 +1,20 @@ +#include "url.hh" +#include "url-parts.hh" +#include "util.hh" +#include "split.hh" + +namespace nix { + +/** + * Try to extract a reasonably unique and meaningful, human-readable + * name of a flake output from a parsed URL. + * When nullopt is returned, the callsite should use information available + * to it outside of the URL to determine a useful name. + * This is a heuristic approach intended for user interfaces. + * @return nullopt if the extracted name is not useful to identify a + * flake output, for example because it is empty or "default". + * Otherwise returns the extracted name. + */ +std::optional getNameFromURL(ParsedURL url); + +} diff --git a/src/nix/profile-list.md b/src/nix/profile-list.md index 5d7fcc0ec..facfdf0d6 100644 --- a/src/nix/profile-list.md +++ b/src/nix/profile-list.md @@ -6,12 +6,14 @@ R""( ```console # nix profile list + Name: gdb Index: 0 Flake attribute: legacyPackages.x86_64-linux.gdb Original flake URL: flake:nixpkgs Locked flake URL: github:NixOS/nixpkgs/7b38b03d76ab71bdc8dc325e3f6338d984cc35ca Store paths: /nix/store/indzcw5wvlhx6vwk7k4iq29q15chvr3d-gdb-11.1 + Name: blender-bin Index: 1 Flake attribute: packages.x86_64-linux.default Original flake URL: flake:blender-bin @@ -26,7 +28,7 @@ R""( # nix build github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender#packages.x86_64-linux.default ``` - will build the package with index 1 shown above. + will build the package `blender-bin` shown above. # Description @@ -34,10 +36,14 @@ This command shows what packages are currently installed in a profile. For each installed package, it shows the following information: -* `Index`: An integer that can be used to unambiguously identify the +* `Name`: A unique name used to unambiguously identify the package in invocations of `nix profile remove` and `nix profile upgrade`. +* `Index`: An integer that can be used to unambiguously identify the + package in invocations of `nix profile remove` and `nix profile upgrade`. + (*Deprecated, will be removed in a future version in favor of `Name`.*) + * `Flake attribute`: The flake output attribute path that provides the package (e.g. `packages.x86_64-linux.hello`). diff --git a/src/nix/profile-remove.md b/src/nix/profile-remove.md index ba85441d8..c994b79bd 100644 --- a/src/nix/profile-remove.md +++ b/src/nix/profile-remove.md @@ -2,18 +2,19 @@ R""( # Examples -* Remove a package by position: +* Remove a package by name: + + ```console + # nix profile remove hello + ``` + +* Remove a package by index + *(deprecated, will be removed in a future version)*: ```console # nix profile remove 3 ``` -* Remove a package by attribute path: - - ```console - # nix profile remove packages.x86_64-linux.hello - ``` - * Remove all packages: ```console diff --git a/src/nix/profile-upgrade.md b/src/nix/profile-upgrade.md index 39cca428b..47103edfc 100644 --- a/src/nix/profile-upgrade.md +++ b/src/nix/profile-upgrade.md @@ -9,18 +9,16 @@ R""( # nix profile upgrade '.*' ``` -* Upgrade a specific package: +* Upgrade a specific package by name: ```console - # nix profile upgrade packages.x86_64-linux.hello + # nix profile upgrade hello ``` -* Upgrade a specific profile element by number: +* Upgrade a specific package by index + *(deprecated, will be removed in a future version)*: ```console - # nix profile list - 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify … - # nix profile upgrade 0 ``` diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 476ddcd60..48a481858 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -10,6 +10,8 @@ #include "../nix-env/user-env.hh" #include "profiles.hh" #include "names.hh" +#include "url.hh" +#include "url-name.hh" #include #include @@ -43,6 +45,7 @@ const int defaultPriority = 5; struct ProfileElement { StorePathSet storePaths; + std::string name; std::optional source; bool active = true; int priority = defaultPriority; @@ -116,6 +119,8 @@ struct ProfileManifest if (pathExists(manifestPath)) { auto json = nlohmann::json::parse(readFile(manifestPath)); + /* Keep track of alreay found names to allow preventing duplicates */ + std::set foundNames; auto version = json.value("version", 0); std::string sUrl; @@ -149,6 +154,25 @@ struct ProfileManifest e["outputs"].get() }; } + + std::string nameCandidate = element.identifier(); + if (e.contains("name")) { + nameCandidate = e["name"]; + } + else if (element.source) { + auto url = parseURL(element.source->to_string()); + auto name = getNameFromURL(url); + if (name) + nameCandidate = *name; + } + + auto finalName = nameCandidate; + for (int i = 1; foundNames.contains(finalName); ++i) { + finalName = nameCandidate + std::to_string(i); + } + element.name = finalName; + foundNames.insert(element.name); + elements.emplace_back(std::move(element)); } } @@ -163,6 +187,7 @@ struct ProfileManifest for (auto & drvInfo : drvInfos) { ProfileElement element; element.storePaths = {drvInfo.queryOutPath()}; + element.name = element.identifier(); elements.emplace_back(std::move(element)); } } @@ -451,15 +476,25 @@ public: { std::vector res; + auto anyIndexMatchers = false; + for (auto & s : _matchers) { - if (auto n = string2Int(s)) + if (auto n = string2Int(s)) { res.push_back(*n); + anyIndexMatchers = true; + } else if (store->isStorePath(s)) res.push_back(s); else res.push_back(RegexPattern{s,std::regex(s, std::regex::extended | std::regex::icase)}); } + if (anyIndexMatchers) { + warn("Indices are deprecated and will be removed in a future version!\n" + " Refer to packages by their `Name` as printed by `nix profile list`.\n" + " See https://github.com/NixOS/nix/issues/9171 for more information."); + } + return res; } @@ -471,8 +506,7 @@ public: } else if (auto path = std::get_if(&matcher)) { if (element.storePaths.count(store.parseStorePath(*path))) return true; } else if (auto regex = std::get_if(&matcher)) { - if (element.source - && std::regex_match(element.source->attrPath, regex->reg)) + if (std::regex_match(element.name, regex->reg)) return true; } } @@ -556,62 +590,83 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf Installables installables; std::vector indices; + auto matchedCount = 0; auto upgradedCount = 0; for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); - if (element.source - && !element.source->originalRef.input.isLocked() - && matches(*store, element, i, matchers)) - { - upgradedCount++; - - Activity act(*logger, lvlChatty, actUnknown, - fmt("checking '%s' for updates", element.source->attrPath)); - - auto installable = make_ref( - this, - getEvalState(), - FlakeRef(element.source->originalRef), - "", - element.source->outputs, - Strings{element.source->attrPath}, - Strings{}, - lockFlags); - - auto derivedPaths = installable->toDerivedPaths(); - if (derivedPaths.empty()) continue; - auto * infop = dynamic_cast(&*derivedPaths[0].info); - // `InstallableFlake` should use `ExtraPathInfoFlake`. - assert(infop); - auto & info = *infop; - - if (element.source->lockedRef == info.flake.lockedRef) continue; - - printInfo("upgrading '%s' from flake '%s' to '%s'", - element.source->attrPath, element.source->lockedRef, info.flake.lockedRef); - - element.source = ProfileElementSource { - .originalRef = installable->flakeRef, - .lockedRef = info.flake.lockedRef, - .attrPath = info.value.attrPath, - .outputs = installable->extendedOutputsSpec, - }; - - installables.push_back(installable); - indices.push_back(i); + if (!matches(*store, element, i, matchers)) { + continue; } + + matchedCount++; + + if (!element.source) { + warn( + "Found package '%s', but it was not installed from a flake, so it can't be checked for upgrades!", + element.identifier() + ); + continue; + } + if (element.source->originalRef.input.isLocked()) { + warn( + "Found package '%s', but it was installed from a locked flake reference so it can't be upgraded!", + element.identifier() + ); + continue; + } + + upgradedCount++; + + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking '%s' for updates", element.source->attrPath)); + + auto installable = make_ref( + this, + getEvalState(), + FlakeRef(element.source->originalRef), + "", + element.source->outputs, + Strings{element.source->attrPath}, + Strings{}, + lockFlags); + + auto derivedPaths = installable->toDerivedPaths(); + if (derivedPaths.empty()) continue; + auto * infop = dynamic_cast(&*derivedPaths[0].info); + // `InstallableFlake` should use `ExtraPathInfoFlake`. + assert(infop); + auto & info = *infop; + + if (element.source->lockedRef == info.flake.lockedRef) continue; + + printInfo("upgrading '%s' from flake '%s' to '%s'", + element.source->attrPath, element.source->lockedRef, info.flake.lockedRef); + + element.source = ProfileElementSource { + .originalRef = installable->flakeRef, + .lockedRef = info.flake.lockedRef, + .attrPath = info.value.attrPath, + .outputs = installable->extendedOutputsSpec, + }; + + installables.push_back(installable); + indices.push_back(i); } if (upgradedCount == 0) { - for (auto & matcher : matchers) { - if (const size_t * index = std::get_if(&matcher)){ - warn("'%d' is not a valid index", *index); - } else if (const Path * path = std::get_if(&matcher)){ - warn("'%s' does not match any paths", *path); - } else if (const RegexPattern * regex = std::get_if(&matcher)){ - warn("'%s' does not match any packages", regex->pattern); + if (matchedCount == 0) { + for (auto & matcher : matchers) { + if (const size_t * index = std::get_if(&matcher)){ + warn("'%d' is not a valid index", *index); + } else if (const Path * path = std::get_if(&matcher)){ + warn("'%s' does not match any paths", *path); + } else if (const RegexPattern * regex = std::get_if(&matcher)){ + warn("'%s' does not match any packages", regex->pattern); + } } + } else { + warn("Found some packages but none of them could be upgraded."); } warn ("Use 'nix profile list' to see the current profile."); } @@ -657,9 +712,10 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); if (i) logger->cout(""); - logger->cout("Index: " ANSI_BOLD "%s" ANSI_NORMAL "%s", - i, + logger->cout("Name: " ANSI_BOLD "%s" ANSI_NORMAL "%s", + element.name, element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL); + logger->cout("Index: %s", i); if (element.source) { logger->cout("Flake attribute: %s%s", element.source->attrPath, element.source->outputs.to_string()); logger->cout("Original flake URL: %s", element.source->originalRef.to_string()); diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh index 7c478a0cd..1fdbfb644 100644 --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -47,7 +47,7 @@ cp ./config.nix $flake1Dir/ # Test upgrading from nix-env. nix-env -f ./user-envs.nix -i foo-1.0 -nix profile list | grep -A2 'Index:.*0' | grep 'Store paths:.*foo-1.0' +nix profile list | grep -A2 'Name:.*foo' | grep 'Store paths:.*foo-1.0' nix profile install $flake1Dir -L nix profile list | grep -A4 'Index:.*1' | grep 'Locked flake URL:.*narHash' [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] @@ -81,7 +81,7 @@ nix profile rollback # Test uninstall. [ -e $TEST_HOME/.nix-profile/bin/foo ] -nix profile remove 0 +nix profile remove foo (! [ -e $TEST_HOME/.nix-profile/bin/foo ]) nix profile history | grep 'foo: 1.0 -> ∅' nix profile diff-closures | grep 'Version 3 -> 4' @@ -93,6 +93,13 @@ nix profile remove 1 nix profile install $(nix-build --no-out-link ./simple.nix) [[ $(cat $TEST_HOME/.nix-profile/hello) = "Hello World!" ]] +# Test packages with same name from different sources +mkdir $TEST_ROOT/simple-too +cp ./simple.nix ./config.nix simple.builder.sh $TEST_ROOT/simple-too +nix profile install --file $TEST_ROOT/simple-too/simple.nix '' +nix profile list | grep -A4 'Name:.*simple' | grep 'Name:.*simple1' +nix profile remove simple1 + # Test wipe-history. nix profile wipe-history [[ $(nix profile history | grep Version | wc -l) -eq 1 ]] @@ -104,7 +111,7 @@ nix profile upgrade 0 nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 3.0, 3.0-man" # Test new install of CA package. -nix profile remove 0 +nix profile remove flake1 printf 4.0 > $flake1Dir/version printf Utrecht > $flake1Dir/who nix profile install $flake1Dir @@ -112,26 +119,26 @@ nix profile install $flake1Dir [[ $(nix path-info --json $(realpath $TEST_HOME/.nix-profile/bin/hello) | jq -r .[].ca) =~ fixed:r:sha256: ]] # Override the outputs. -nix profile remove 0 1 +nix profile remove simple flake1 nix profile install "$flake1Dir^*" [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Utrecht" ]] [ -e $TEST_HOME/.nix-profile/share/man ] [ -e $TEST_HOME/.nix-profile/include ] printf Nix > $flake1Dir/who -nix profile upgrade 0 +nix profile upgrade flake1 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Nix" ]] [ -e $TEST_HOME/.nix-profile/share/man ] [ -e $TEST_HOME/.nix-profile/include ] -nix profile remove 0 +nix profile remove flake1 nix profile install "$flake1Dir^man" (! [ -e $TEST_HOME/.nix-profile/bin/hello ]) [ -e $TEST_HOME/.nix-profile/share/man ] (! [ -e $TEST_HOME/.nix-profile/include ]) # test priority -nix profile remove 0 +nix profile remove flake1 # Make another flake. flake2Dir=$TEST_ROOT/flake2 From 9c0a09f09fbb930483b26f60f8552fbe5236b777 Mon Sep 17 00:00:00 2001 From: Bob van der Linden Date: Sun, 1 Oct 2023 22:09:55 +0200 Subject: [PATCH 546/909] allow ^ in URLs Users may select specific outputs using the ^output syntax or selecting any output using ^*. URL parsing currently doesn't support these kinds of output references: parsing will fail. Currently `queryRegex` was reused for URL fragments, which didn't include support for ^. Now queryRegex has been split from fragmentRegex, where only the fragmentRegex supports ^. --- src/libexpr/flake/flakeref.cc | 2 +- src/libutil/tests/url-name.cc | 3 +++ src/libutil/url-name.cc | 5 +++-- src/libutil/url-parts.hh | 1 + src/libutil/url.cc | 2 +- tests/functional/nix-profile.sh | 1 + 6 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 16f45ace7..49d6940b1 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -190,7 +190,7 @@ std::optional> parseFlakeIdRef( static std::regex flakeRegex( "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)" - + "(?:#(" + queryRegex + "))?", + + "(?:#(" + fragmentRegex + "))?", std::regex::ECMAScript); if (std::regex_match(url, match, flakeRegex)) { diff --git a/src/libutil/tests/url-name.cc b/src/libutil/tests/url-name.cc index 6ee66e826..f637efa89 100644 --- a/src/libutil/tests/url-name.cc +++ b/src/libutil/tests/url-name.cc @@ -10,6 +10,8 @@ namespace nix { ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#packages.x86_64-linux.hello")), "hello"); ASSERT_EQ(getNameFromURL(parseURL("path:.#nonStandardAttr.mylaptop")), "nonStandardAttr.mylaptop"); ASSERT_EQ(getNameFromURL(parseURL("path:./repos/myflake#nonStandardAttr.mylaptop")), "nonStandardAttr.mylaptop"); + ASSERT_EQ(getNameFromURL(parseURL("path:./nixpkgs#packages.x86_64-linux.complex^bin,man")), "complex"); + ASSERT_EQ(getNameFromURL(parseURL("path:./myproj#packages.x86_64-linux.default^*")), "myproj"); ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nixpkgs#packages.x86_64-linux.hello")), "hello"); ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nixpkgs#hello")), "hello"); @@ -60,5 +62,6 @@ namespace nix { ASSERT_EQ(getNameFromURL(parseURL("path:.")), std::nullopt); ASSERT_EQ(getNameFromURL(parseURL("file:.#")), std::nullopt); ASSERT_EQ(getNameFromURL(parseURL("path:.#packages.x86_64-linux.default")), std::nullopt); + ASSERT_EQ(getNameFromURL(parseURL("path:.#packages.x86_64-linux.default^*")), std::nullopt); } } diff --git a/src/libutil/url-name.cc b/src/libutil/url-name.cc index ab65e78df..f94383e32 100644 --- a/src/libutil/url-name.cc +++ b/src/libutil/url-name.cc @@ -5,12 +5,13 @@ namespace nix { static const std::string attributeNamePattern("[a-z0-9_-]+"); -static const std::regex lastAttributeRegex("(?:" + attributeNamePattern + "\\.)*(?!default)(" + attributeNamePattern +")"); +static const std::regex lastAttributeRegex("(?:" + attributeNamePattern + "\\.)*(?!default)(" + attributeNamePattern +")(\\^.*)?"); static const std::string pathSegmentPattern("[a-zA-Z0-9_-]+"); static const std::regex lastPathSegmentRegex(".*/(" + pathSegmentPattern +")"); static const std::regex secondPathSegmentRegex("(?:" + pathSegmentPattern + ")/(" + pathSegmentPattern +")(?:/.*)?"); static const std::regex gitProviderRegex("github|gitlab|sourcehut"); static const std::regex gitSchemeRegex("git($|\\+.*)"); +static const std::regex defaultOutputRegex(".*\\.default($|\\^.*)"); std::optional getNameFromURL(ParsedURL url) { std::smatch match; @@ -32,7 +33,7 @@ std::optional getNameFromURL(ParsedURL url) { return match.str(1); /* If everything failed but there is a non-default fragment, use it in full */ - if (!url.fragment.empty() && !hasSuffix(url.fragment, "default")) + if (!url.fragment.empty() && !std::regex_match(url.fragment, defaultOutputRegex)) return url.fragment; /* If there is no fragment, take the last element of the path */ diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 5c5a30dc2..59c17df34 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -19,6 +19,7 @@ const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncod const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; +const static std::string fragmentRegex = "(?:" + pcharRegex + "|[/? \"^])*"; const static std::string segmentRegex = "(?:" + pcharRegex + "*)"; const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 9b438e6cd..2a0a5c839 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -15,7 +15,7 @@ ParsedURL parseURL(const std::string & url) "((" + schemeRegex + "):" + "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))" + "(?:\\?(" + queryRegex + "))?" - + "(?:#(" + queryRegex + "))?", + + "(?:#(" + fragmentRegex + "))?", std::regex::ECMAScript); std::smatch match; diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh index 1fdbfb644..eced4d3f1 100644 --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -126,6 +126,7 @@ nix profile install "$flake1Dir^*" [ -e $TEST_HOME/.nix-profile/include ] printf Nix > $flake1Dir/who +nix profile list nix profile upgrade flake1 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Nix" ]] [ -e $TEST_HOME/.nix-profile/share/man ] From c60eba3276d7417a7f51ef606e5b9ca580cf5e5b Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Mon, 6 Nov 2023 21:43:18 +0100 Subject: [PATCH 547/909] Add release note on XDG_DATA_DIRS change Follow-up to https://github.com/NixOS/nix/pull/8985 --- doc/manual/src/release-notes/rl-next.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 85e180e37..73ba03fc4 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -64,3 +64,8 @@ ``` This makes it match `nix derivation show`, which also maps store paths to information. + +- When Nix is installed using the [binary installer](@docroot@/installation/installing-binary.md), in supported shells (Bash, Zsh, Fish) + [`XDG_DATA_DIRS`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables) is now populated with the path to the `/share` subdirectory of the current profile. + This means that command completion scripts, `.desktop` files, and similar artifacts installed via [`nix-env`](@docroot@/command-ref/nix-env.md) or [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) + (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). From 9fec62a10044629ad4758ec95f9b1e67d7aefff5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 22:21:07 +0000 Subject: [PATCH 548/909] build(deps): bump zeebe-io/backport-action from 2.0.0 to 2.1.0 Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 2.0.0 to 2.1.0. - [Release notes](https://github.com/zeebe-io/backport-action/releases) - [Commits](https://github.com/zeebe-io/backport-action/compare/v2.0.0...v2.1.0) --- updated-dependencies: - dependency-name: zeebe-io/backport-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 312c211dd..893f4a56f 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v2.0.0 + uses: zeebe-io/backport-action@v2.1.0 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} From b733f4ab29cec07cf17e1fe6580c9d2f8a4362a0 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 7 Nov 2023 01:12:39 +0100 Subject: [PATCH 549/909] maintainers: refine the mission statement phrasing setting a direction falls short of what we're already doing: guide contributors. the direction aspect is still important, as that is the authoritative part. guidance is the supportive part. --- maintainers/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintainers/README.md b/maintainers/README.md index 5be4f9d04..ee97c1195 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -2,7 +2,7 @@ ## Motivation -The team's main responsibility is to set a direction for the development of Nix and ensure that the code is in good shape. +The team's main responsibility is to guide and direct the development of Nix and ensure that the code is in good shape. We aim to achieve this by improving the contributor experience and attracting more maintainers – that is, by helping other people contributing to Nix and eventually taking responsibility – in order to scale the development process to match users' needs. From 1362a0a55aaddccef5a525e3b1179239d650bb07 Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Mon, 6 Nov 2023 23:16:05 +0100 Subject: [PATCH 550/909] Fix logic for default XDG_DATA_DIRS value The [POSIX test manpage](https://pubs.opengroup.org/onlinepubs/9699919799/utilities/test.html) as well as the [fish test manpage](https://fishshell.com/docs/current/cmds/test.html#operators-for-text-strings) specify that `-z` will be "True if the length of string string is zero; otherwise, false." The `-n` was likely a mixup and not caught during testing of https://github.com/NixOS/nix/pull/8985 due to a lack of missing conflicting entries in `XDG_DATA_DIRS`. --- scripts/nix-profile-daemon.fish.in | 2 +- scripts/nix-profile-daemon.sh.in | 2 +- scripts/nix-profile.fish.in | 2 +- scripts/nix-profile.sh.in | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index 3fe9e782a..c23aa64f0 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -20,7 +20,7 @@ set __ETC_PROFILE_NIX_SOURCED 1 set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" # Populate bash completions, .desktop files, etc -if test -n "$XDG_DATA_DIRS" +if test -z "$XDG_DATA_DIRS" # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:/nix/var/nix/profiles/default/share" else diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 3089cec66..c63db4648 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -31,7 +31,7 @@ fi export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" # Populate bash completions, .desktop files, etc -if [ -n "${XDG_DATA_DIRS-}" ]; then +if [ -z "$XDG_DATA_DIRS" ]; then # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" else diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in index 2523594f2..619df52b8 100644 --- a/scripts/nix-profile.fish.in +++ b/scripts/nix-profile.fish.in @@ -21,7 +21,7 @@ if test -n "$HOME" && test -n "$USER" set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" # Populate bash completions, .desktop files, etc - if test -n "$XDG_DATA_DIRS" + if test -z "$XDG_DATA_DIRS" # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" else diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index a0d098588..56e070ae1 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -33,7 +33,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" # Populate bash completions, .desktop files, etc - if [ -n "${XDG_DATA_DIRS-}" ]; then + if [ -z "$XDG_DATA_DIRS" ]; then # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" else From 74210c12feccc6c6b717c5f39c28d7ce86614e60 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Sat, 28 Aug 2021 16:26:53 -0400 Subject: [PATCH 551/909] Shellbang support with flakes Enables shebang usage of nix shell. All arguments with `#! nix` get added to the nix invocation. This implementation does NOT set any additional arguments other than placing the script path itself as the first argument such that the interpreter can utilize it. Example below: ``` #!/usr/bin/env nix #! nix shell --quiet #! nix nixpkgs#bash #! nix nixpkgs#shellcheck #! nix nixpkgs#hello #! nix --ignore-environment --command bash # shellcheck shell=bash set -eu shellcheck "$0" || exit 1 function main { hello echo 0:"$0" 1:"$1" 2:"$2" } "$@" ``` fix: include programName usage EDIT: For posterity I've changed shellwords to shellwords2 in order not to interfere with other changes during a rebase. shellwords2 is removed in a later commit. -- roberth --- src/libutil/args.cc | 37 +++++++++++++++++++++++++ src/libutil/args.hh | 8 +++++- src/libutil/util.cc | 45 +++++++++++++++++++++++++++++++ src/libutil/util.hh | 11 +++++--- src/nix/main.cc | 2 +- tests/functional/flakes/common.sh | 5 ++-- tests/functional/flakes/flakes.sh | 15 ++++++++++- 7 files changed, 114 insertions(+), 9 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 0b65519a3..7106491fd 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -6,6 +6,7 @@ #include "users.hh" #include "json-utils.hh" +#include #include namespace nix { @@ -78,6 +79,12 @@ std::optional RootArgs::needsCompletion(std::string_view s) } void RootArgs::parseCmdline(const Strings & _cmdline) +{ + // Default via 5.1.2.2.1 in C standard + Args::parseCmdline("", _cmdline); +} + +void Args::parseCmdline(const std::string & programName, const Strings & _cmdline) { Strings pendingArgs; bool dashDash = false; @@ -93,6 +100,36 @@ void RootArgs::parseCmdline(const Strings & _cmdline) } bool argsSeen = false; + + // Heuristic to see if we're invoked as a shebang script, namely, + // if we have at least one argument, it's the name of an + // executable file, and it starts with "#!". + Strings savedArgs; + auto isNixCommand = std::regex_search(programName, std::regex("nix$")); + if (isNixCommand && cmdline.size() > 0) { + auto script = *cmdline.begin(); + try { + auto lines = tokenizeString(readFile(script), "\n"); + if (std::regex_search(lines.front(), std::regex("^#!"))) { + lines.pop_front(); + for (auto pos = std::next(cmdline.begin()); pos != cmdline.end();pos++) + savedArgs.push_back(*pos); + cmdline.clear(); + + for (auto line : lines) { + line = chomp(line); + + std::smatch match; + if (std::regex_match(line, match, std::regex("^#!\\s*nix\\s(.*)$"))) + for (const auto & word : shellwords(match[1].str())) + cmdline.push_back(word); + } + cmdline.push_back(script); + for (auto pos = savedArgs.begin(); pos != savedArgs.end();pos++) + cmdline.push_back(*pos); + } + } catch (SysError &) { } + } for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { auto arg = *pos; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 45fd678e7..1d056678d 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -27,8 +27,14 @@ class Args public: /** - * Return a short one-line description of the command. + * Parse the command line with argv0, throwing a UsageError if something + goes wrong. */ + void parseCmdline(const std::string & argv0, const Strings & cmdline); + + /** + * Return a short one-line description of the command. + */ virtual std::string description() { return ""; } virtual bool forceImpureByDefault() { return false; } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index ee7a22849..6ca1dbd7a 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -5,6 +5,8 @@ #include #include #include +#include + namespace nix { @@ -136,6 +138,49 @@ std::string shellEscape(const std::string_view s) return r; } +/* Recreate the effect of the perl shellwords function, breaking up a + * string into arguments like a shell word, including escapes + */ +std::vector shellwords2(const std::string & s) +{ + std::regex whitespace("^(\\s+).*"); + auto begin = s.cbegin(); + std::vector res; + std::string cur; + enum state { + sBegin, + sQuote + }; + state st = sBegin; + auto it = begin; + for (; it != s.cend(); ++it) { + if (st == sBegin) { + std::smatch match; + if (regex_search(it, s.cend(), match, whitespace)) { + cur.append(begin, it); + res.push_back(cur); + cur.clear(); + it = match[1].second; + begin = it; + } + } + switch (*it) { + case '"': + cur.append(begin, it); + begin = it + 1; + st = st == sBegin ? sQuote : sBegin; + break; + case '\\': + /* perl shellwords mostly just treats the next char as part of the string with no special processing */ + cur.append(begin, it); + begin = ++it; + break; + } + } + cur.append(begin, it); + if (!cur.empty()) res.push_back(cur); + return res; +} void ignoreException(Verbosity lvl) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 5f730eaf6..bcd0c1769 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -189,10 +189,13 @@ std::string toLower(const std::string & s); std::string shellEscape(const std::string_view s); -/** - * Exception handling in destructors: print an error message, then - * ignore the exception. - */ +/* Recreate the effect of the perl shellwords function, breaking up a + * string into arguments like a shell word, including escapes */ +std::vector shellwords2(const std::string & s); + + +/* Exception handling in destructors: print an error message, then + ignore the exception. */ void ignoreException(Verbosity lvl = lvlError); diff --git a/src/nix/main.cc b/src/nix/main.cc index b582fc166..16fb50806 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -428,7 +428,7 @@ void mainWrapped(int argc, char * * argv) }); try { - args.parseCmdline(argvToStrings(argc, argv)); + args.parseCmdline(programName, argvToStrings(argc, argv)); } catch (UsageError &) { if (!args.helpRequested && !args.completions) throw; } diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh index 8aed296e6..fc45cf7bf 100644 --- a/tests/functional/flakes/common.sh +++ b/tests/functional/flakes/common.sh @@ -11,6 +11,7 @@ writeSimpleFlake() { outputs = inputs: rec { packages.$system = rec { foo = import ./simple.nix; + fooScript = (import ./shell.nix {}).foo; default = foo; }; packages.someOtherSystem = rec { @@ -24,13 +25,13 @@ writeSimpleFlake() { } EOF - cp ../simple.nix ../simple.builder.sh ../config.nix $flakeDir/ + cp ../simple.nix ../shell.nix ../simple.builder.sh ../config.nix $flakeDir/ } createSimpleGitFlake() { local flakeDir="$1" writeSimpleFlake $flakeDir - git -C $flakeDir add flake.nix simple.nix simple.builder.sh config.nix + git -C $flakeDir add flake.nix simple.nix shell.nix simple.builder.sh config.nix git -C $flakeDir commit -m 'Initial' } diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index b0038935c..c4b18a21b 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -66,7 +66,17 @@ cat > "$nonFlakeDir/README.md" < "$nonFlakeDir/shebang.sh" < Date: Mon, 14 Nov 2022 17:04:19 +0100 Subject: [PATCH 552/909] src/libutil/util.hh: Formatting --- src/libutil/util.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index bcd0c1769..b7d3ac504 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -190,7 +190,7 @@ std::string shellEscape(const std::string_view s); /* Recreate the effect of the perl shellwords function, breaking up a - * string into arguments like a shell word, including escapes */ + string into arguments like a shell word, including escapes. */ std::vector shellwords2(const std::string & s); From eea5a003d99094d8488fd0d1ecd97f98d3573133 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Mon, 14 Nov 2022 19:40:01 -0500 Subject: [PATCH 553/909] fix: test to ensure arguments are passed --- tests/functional/flakes/flakes.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index c4b18a21b..e7dffde07 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -71,8 +71,9 @@ cat > "$nonFlakeDir/shebang.sh" < Date: Mon, 14 Nov 2022 23:58:58 -0500 Subject: [PATCH 554/909] doc: shebang release notes, docs, tests fix: release notes --- doc/manual/src/release-notes/rl-next.md | 44 +++++++++ src/nix/shell.md | 117 ++++++++++++++++++++++++ tests/functional/flakes/flakes.sh | 13 +++ 3 files changed, 174 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 73ba03fc4..93d4f432b 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,5 +1,49 @@ # Release X.Y (202?-??-??) +- The experimental nix command is now a `#!-interpreter` by appending the + contents of any `#! nix` lines and the script's location to a single call. + Some examples: + ``` + #!/usr/bin/env nix + #! nix shell --file "" hello --command bash + + hello | cowsay + ``` + or with flakes: + ``` + #!/usr/bin/env nix + #! nix shell nixpkgs#bash nixpkgs#hello nixpkgs#cowsay --command bash + + hello | cowsay + ``` + or + ```bash + #! /usr/bin/env nix + #! nix shell --impure --expr + #! nix "with (import (builtins.getFlake ''nixpkgs'') {}); terraform.withPlugins (plugins: [ plugins.openstack ])" + #! nix --command bash + + terraform "$@" + ``` + or + ``` + #!/usr/bin/env nix + //! ```cargo + //! [dependencies] + //! time = "0.1.25" + //! ``` + /* + #!nix shell nixpkgs#rustc nixpkgs#rust-script nixpkgs#cargo --command rust-script + */ + fn main() { + for argument in std::env::args().skip(1) { + println!("{}", argument); + }; + println!("{}", std::env::var("HOME").expect("")); + println!("{}", time::now().rfc822z()); + } + // vim: ft=rust + ``` - [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. - [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). diff --git a/src/nix/shell.md b/src/nix/shell.md index f36919575..b0bfa1609 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -51,4 +51,121 @@ R""( provides the specified [*installables*](./nix.md#installable). If no command is specified, it starts the default shell of your user account specified by `$SHELL`. +# Use as a `#!`-interpreter + +You can use `nix` as a script interpreter to allow scripts written +in arbitrary languages to obtain their own dependencies via Nix. This is +done by starting the script with the following lines: + +```bash +#! /usr/bin/env nix +#! nix shell installables --command real-interpreter +``` + +where *real-interpreter* is the “real” script interpreter that will be +invoked by `nix shell` after it has obtained the dependencies and +initialised the environment, and *installables* are the attribute names of +the dependencies in Nixpkgs. + +The lines starting with `#! nix` specify options (see above). Note that you +cannot write `#! /usr/bin/env nix shell -i ...` because many operating systems +only allow one argument in `#!` lines. + +For example, here is a Python script that depends on Python and the +`prettytable` package: + +```python +#! /usr/bin/env nix +#! nix shell github:tomberek/-#python3With.prettytable --command python + +import prettytable + +# Print a simple table. +t = prettytable.PrettyTable(["N", "N^2"]) +for n in range(1, 10): t.add_row([n, n * n]) +print t +``` + +Similarly, the following is a Perl script that specifies that it +requires Perl and the `HTML::TokeParser::Simple` and `LWP` packages: + +```perl +#! /usr/bin/env nix +#! nix shell github:tomberek/-#perlWith.HTMLTokeParserSimple.LWP --command perl -x + +use HTML::TokeParser::Simple; + +# Fetch nixos.org and print all hrefs. +my $p = HTML::TokeParser::Simple->new(url => 'http://nixos.org/'); + +while (my $token = $p->get_tag("a")) { + my $href = $token->get_attr("href"); + print "$href\n" if $href; +} +``` + +Sometimes you need to pass a simple Nix expression to customize a +package like Terraform: + +```bash +#! /usr/bin/env nix +#! nix shell --impure --expr +#! nix "with (import (builtins.getFlake ''nixpkgs'') {}); terraform.withPlugins (plugins: [ plugins.openstack ])" +#! nix --command bash + +terraform "$@" +``` + +> **Note** +> +> You must use double quotes (`"`) when passing a simple Nix expression +> in a nix shell shebang. + +Finally, using the merging of multiple nix shell shebangs the following +Haskell script uses a specific branch of Nixpkgs/NixOS (the 21.11 stable +branch): + +```haskell +#!/usr/bin/env nix +#!nix shell --override-input nixpkgs github:NixOS/nixpkgs/nixos-21.11 +#!nix github:tomberek/-#haskellWith.download-curl.tagsoup --command runghc + +import Network.Curl.Download +import Text.HTML.TagSoup +import Data.Either +import Data.ByteString.Char8 (unpack) + +-- Fetch nixos.org and print all hrefs. +main = do + resp <- openURI "https://nixos.org/" + let tags = filter (isTagOpenName "a") $ parseTags $ unpack $ fromRight undefined resp + let tags' = map (fromAttrib "href") tags + mapM_ putStrLn $ filter (/= "") tags' +``` + +If you want to be even more precise, you can specify a specific revision +of Nixpkgs: + + #!nix shell --override-input nixpkgs github:NixOS/nixpkgs/eabc38219184cc3e04a974fe31857d8e0eac098d + +The examples above all used `-p` to get dependencies from Nixpkgs. You +can also use a Nix expression to build your own dependencies. For +example, the Python example could have been written as: + +```python +#! /usr/bin/env nix +#! nix shell --impure --file deps.nix -i python +``` + +where the file `deps.nix` in the same directory as the `#!`-script +contains: + +```nix +with import {}; +python3.withPackages (ps: with ps; [ prettytable ]) +``` + + + + )"" diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index e7dffde07..f27925493 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -80,6 +80,18 @@ chmod +x "$nonFlakeDir/shebang.sh" git -C "$nonFlakeDir" add README.md shebang.sh git -C "$nonFlakeDir" commit -m 'Initial' +cat > $nonFlakeDir/shebang-perl.sh < Date: Wed, 1 Sep 2021 02:19:51 -0400 Subject: [PATCH 555/909] Read file incrementally --- src/libutil/args.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 7106491fd..80216e7ad 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -6,6 +6,8 @@ #include "users.hh" #include "json-utils.hh" +#include +#include #include #include @@ -109,14 +111,17 @@ void Args::parseCmdline(const std::string & programName, const Strings & _cmdlin if (isNixCommand && cmdline.size() > 0) { auto script = *cmdline.begin(); try { - auto lines = tokenizeString(readFile(script), "\n"); - if (std::regex_search(lines.front(), std::regex("^#!"))) { - lines.pop_front(); + std::ifstream stream(script); + char shebang[3]={0,0,0}; + stream.get(shebang,3); + if (strncmp(shebang,"#!",2) == 0){ for (auto pos = std::next(cmdline.begin()); pos != cmdline.end();pos++) savedArgs.push_back(*pos); cmdline.clear(); - for (auto line : lines) { + std::string line; + std::getline(stream,line); + while (std::getline(stream,line) && !line.empty()){ line = chomp(line); std::smatch match; From 06f3583b1c860b24f2f704f216f4db8fd1dcae9c Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Sat, 26 Nov 2022 09:06:39 -0500 Subject: [PATCH 556/909] feat: break out of shebang processing for non-comments --- src/libutil/args.cc | 3 ++- tests/functional/flakes/flakes.sh | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 80216e7ad..d90374adc 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -121,7 +121,8 @@ void Args::parseCmdline(const std::string & programName, const Strings & _cmdlin std::string line; std::getline(stream,line); - while (std::getline(stream,line) && !line.empty()){ + std::string commentChars("#/\\%@*-"); + while (std::getline(stream,line) && !line.empty() && commentChars.find(line[0]) != std::string::npos){ line = chomp(line); std::smatch match; diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index f27925493..28b5e4e0f 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -80,17 +80,17 @@ chmod +x "$nonFlakeDir/shebang.sh" git -C "$nonFlakeDir" add README.md shebang.sh git -C "$nonFlakeDir" commit -m 'Initial' -cat > $nonFlakeDir/shebang-perl.sh < $nonFlakeDir/shebang-comments.sh < Date: Tue, 3 Jan 2023 05:55:06 -0500 Subject: [PATCH 557/909] doc: remove reference to nix-shell --- src/nix/shell.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/nix/shell.md b/src/nix/shell.md index b0bfa1609..7e0e5f213 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -148,9 +148,8 @@ of Nixpkgs: #!nix shell --override-input nixpkgs github:NixOS/nixpkgs/eabc38219184cc3e04a974fe31857d8e0eac098d -The examples above all used `-p` to get dependencies from Nixpkgs. You -can also use a Nix expression to build your own dependencies. For -example, the Python example could have been written as: +You can also use a Nix expression to build your own dependencies. For example, +the Python example could have been written as: ```python #! /usr/bin/env nix @@ -166,6 +165,4 @@ python3.withPackages (ps: with ps; [ prettytable ]) ``` - - )"" From bbeddf06027424dc08742c1d54bf2fdc85ff6e8e Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 12 May 2023 07:44:25 -0400 Subject: [PATCH 558/909] fix: refactor parseCmdline interface --- src/libutil/args.cc | 9 ++++----- src/libutil/args.hh | 2 +- src/nix/main.cc | 5 ++++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index d90374adc..481ed33ff 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -83,10 +83,10 @@ std::optional RootArgs::needsCompletion(std::string_view s) void RootArgs::parseCmdline(const Strings & _cmdline) { // Default via 5.1.2.2.1 in C standard - Args::parseCmdline("", _cmdline); + Args::parseCmdline(_cmdline, false); } -void Args::parseCmdline(const std::string & programName, const Strings & _cmdline) +void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) { Strings pendingArgs; bool dashDash = false; @@ -107,8 +107,7 @@ void Args::parseCmdline(const std::string & programName, const Strings & _cmdlin // if we have at least one argument, it's the name of an // executable file, and it starts with "#!". Strings savedArgs; - auto isNixCommand = std::regex_search(programName, std::regex("nix$")); - if (isNixCommand && cmdline.size() > 0) { + if (allowShebang){ auto script = *cmdline.begin(); try { std::ifstream stream(script); @@ -121,7 +120,7 @@ void Args::parseCmdline(const std::string & programName, const Strings & _cmdlin std::string line; std::getline(stream,line); - std::string commentChars("#/\\%@*-"); + static const std::string commentChars("#/\\%@*-"); while (std::getline(stream,line) && !line.empty() && commentChars.find(line[0]) != std::string::npos){ line = chomp(line); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 1d056678d..e753dcaf6 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -30,7 +30,7 @@ public: * Parse the command line with argv0, throwing a UsageError if something goes wrong. */ - void parseCmdline(const std::string & argv0, const Strings & cmdline); + void parseCmdline(const Strings & _cmdline, bool allowShebang); /** * Return a short one-line description of the command. diff --git a/src/nix/main.cc b/src/nix/main.cc index 16fb50806..73641f6d2 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include @@ -428,7 +429,9 @@ void mainWrapped(int argc, char * * argv) }); try { - args.parseCmdline(programName, argvToStrings(argc, argv)); + auto isNixCommand = std::regex_search(programName, std::regex("nix$")); + auto allowShebang = isNixCommand && argc > 1; + args.parseCmdline(argvToStrings(argc, argv),allowShebang); } catch (UsageError &) { if (!args.helpRequested && !args.completions) throw; } From cc68ed8ff7b9e3898308a39dfdad2660bacc153f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 May 2023 19:42:49 +0200 Subject: [PATCH 559/909] libcmd: lookupFileArg(): add baseDir This will allow a different base directory to be used, matching a shebang script location instead of the working directory. --- src/libcmd/common-eval-args.cc | 4 ++-- src/libcmd/common-eval-args.hh | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 91fa881b1..401acc38e 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -164,7 +164,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) return res.finish(); } -SourcePath lookupFileArg(EvalState & state, std::string_view s) +SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir) { if (EvalSettings::isPseudoUrl(s)) { auto storePath = fetchers::downloadTarball( @@ -185,7 +185,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s) } else - return state.rootPath(CanonPath::fromCwd(s)); + return state.rootPath(CanonPath(s, baseDir)); } } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 6359b2579..4b403d936 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -2,6 +2,7 @@ ///@file #include "args.hh" +#include "canon-path.hh" #include "common-args.hh" #include "search-path.hh" @@ -28,6 +29,6 @@ private: std::map autoArgs; }; -SourcePath lookupFileArg(EvalState & state, std::string_view s); +SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir = CanonPath::fromCwd()); } From 20ff61ab252fc1d2bd69987f51a000739b24c670 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 May 2023 19:46:37 +0200 Subject: [PATCH 560/909] nix: Reserve shebang line syntax and only parse double backtick quotes Being restrictive about syntax leaves opportunity to improve the syntax and functionality later. --- doc/manual/src/release-notes/rl-next.md | 11 +- src/libutil/args.cc | 152 +++++++++++++++++++++++- src/libutil/util.cc | 43 ------- src/libutil/util.hh | 5 - src/nix/shell.md | 8 +- tests/functional/flakes/flakes.sh | 16 ++- 6 files changed, 177 insertions(+), 58 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 93d4f432b..4bff3c685 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -2,10 +2,13 @@ - The experimental nix command is now a `#!-interpreter` by appending the contents of any `#! nix` lines and the script's location to a single call. + + Verbatim strings may be passed in double backtick (```` `` ````) quotes. + Some examples: ``` #!/usr/bin/env nix - #! nix shell --file "" hello --command bash + #! nix shell --file ```` hello --command bash hello | cowsay ``` @@ -19,8 +22,10 @@ or ```bash #! /usr/bin/env nix - #! nix shell --impure --expr - #! nix "with (import (builtins.getFlake ''nixpkgs'') {}); terraform.withPlugins (plugins: [ plugins.openstack ])" + #! nix shell --impure --expr `` + #! nix with (import (builtins.getFlake "nixpkgs") {}); + #! nix terraform.withPlugins (plugins: [ plugins.openstack ]) + #! nix `` #! nix --command bash terraform "$@" diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 481ed33ff..ab6e0e266 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -86,6 +86,147 @@ void RootArgs::parseCmdline(const Strings & _cmdline) Args::parseCmdline(_cmdline, false); } +/** + * Basically this is `typedef std::optional Parser(std::string_view s, Strings & r);` + * + * Except we can't recursively reference the Parser typedef, so we have to write a class. + */ +struct Parser { + std::string_view remaining; + + /** + * @brief Parse the next character(s) + * + * @param r + * @return std::shared_ptr + */ + virtual void operator()(std::shared_ptr & state, Strings & r) = 0; + + Parser(std::string_view s) : remaining(s) {}; +}; + +struct ParseQuoted : public Parser { + /** + * @brief Accumulated string + * + * Parsed argument up to this point. + */ + std::string acc; + + ParseQuoted(std::string_view s) : Parser(s) {}; + + virtual void operator()(std::shared_ptr & state, Strings & r) override; +}; + + +struct ParseUnquoted : public Parser { + /** + * @brief Accumulated string + * + * Parsed argument up to this point. Empty string is not representable in + * unquoted syntax, so we use it for the initial state. + */ + std::string acc; + + ParseUnquoted(std::string_view s) : Parser(s) {}; + + virtual void operator()(std::shared_ptr & state, Strings & r) override { + if (remaining.empty()) { + if (!acc.empty()) + r.push_back(acc); + state = nullptr; // done + return; + } + switch (remaining[0]) { + case ' ': case '\t': case '\n': case '\r': + if (!acc.empty()) + r.push_back(acc); + state = std::make_shared(ParseUnquoted(remaining.substr(1))); + return; + case '`': + if (remaining.size() > 1 && remaining[1] == '`') { + state = std::make_shared(ParseQuoted(remaining.substr(2))); + return; + } + else + throw Error("single backtick is not a supported syntax in the nix shebang."); + + // reserved characters + // meaning to be determined, or may be reserved indefinitely so that + // #!nix syntax looks unambiguous + case '$': + case '*': + case '~': + case '<': + case '>': + case '|': + case ';': + case '(': + case ')': + case '[': + case ']': + case '{': + case '}': + case '\'': + case '"': + case '\\': + throw Error("unsupported unquoted character in nix shebang: " + std::string(1, remaining[0]) + ". Use double backticks to escape?"); + + case '#': + if (acc.empty()) { + throw Error ("unquoted nix shebang argument cannot start with #. Use double backticks to escape?"); + } else { + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + + default: + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + assert(false); + } +}; + +void ParseQuoted::operator()(std::shared_ptr &state, Strings & r) { + if (remaining.empty()) { + throw Error("unterminated quoted string in nix shebang"); + } + switch (remaining[0]) { + case '`': + if (remaining.size() > 1 && remaining[1] == '`') { + state = std::make_shared(ParseUnquoted(remaining.substr(2))); + r.push_back(acc); + return; + } + else { + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + default: + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + assert(false); +} + +static Strings parseShebangContent(std::string_view s) { + Strings result; + std::shared_ptr parserState(std::make_shared(ParseUnquoted(s))); + + // trampoline == iterated strategy pattern + while (parserState) { + auto currentState = parserState; + (*currentState)(parserState, result); + } + + return result; +} + void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) { Strings pendingArgs; @@ -121,13 +262,18 @@ void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) std::string line; std::getline(stream,line); static const std::string commentChars("#/\\%@*-"); + std::string shebangContent; while (std::getline(stream,line) && !line.empty() && commentChars.find(line[0]) != std::string::npos){ line = chomp(line); std::smatch match; - if (std::regex_match(line, match, std::regex("^#!\\s*nix\\s(.*)$"))) - for (const auto & word : shellwords(match[1].str())) - cmdline.push_back(word); + // We match one space after `nix` so that we preserve indentation. + // No space is necessary for an empty line. An empty line has basically no effect. + if (std::regex_match(line, match, std::regex("^#!\\s*nix(:? |$)(.*)$"))) + shebangContent += match[2].str() + "\n"; + } + for (const auto & word : parseShebangContent(shebangContent)) { + cmdline.push_back(word); } cmdline.push_back(script); for (auto pos = savedArgs.begin(); pos != savedArgs.end();pos++) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 6ca1dbd7a..5bb3f374b 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -138,49 +138,6 @@ std::string shellEscape(const std::string_view s) return r; } -/* Recreate the effect of the perl shellwords function, breaking up a - * string into arguments like a shell word, including escapes - */ -std::vector shellwords2(const std::string & s) -{ - std::regex whitespace("^(\\s+).*"); - auto begin = s.cbegin(); - std::vector res; - std::string cur; - enum state { - sBegin, - sQuote - }; - state st = sBegin; - auto it = begin; - for (; it != s.cend(); ++it) { - if (st == sBegin) { - std::smatch match; - if (regex_search(it, s.cend(), match, whitespace)) { - cur.append(begin, it); - res.push_back(cur); - cur.clear(); - it = match[1].second; - begin = it; - } - } - switch (*it) { - case '"': - cur.append(begin, it); - begin = it + 1; - st = st == sBegin ? sQuote : sBegin; - break; - case '\\': - /* perl shellwords mostly just treats the next char as part of the string with no special processing */ - cur.append(begin, it); - begin = ++it; - break; - } - } - cur.append(begin, it); - if (!cur.empty()) res.push_back(cur); - return res; -} void ignoreException(Verbosity lvl) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index b7d3ac504..27faa4d6d 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -189,11 +189,6 @@ std::string toLower(const std::string & s); std::string shellEscape(const std::string_view s); -/* Recreate the effect of the perl shellwords function, breaking up a - string into arguments like a shell word, including escapes. */ -std::vector shellwords2(const std::string & s); - - /* Exception handling in destructors: print an error message, then ignore the exception. */ void ignoreException(Verbosity lvl = lvlError); diff --git a/src/nix/shell.md b/src/nix/shell.md index 7e0e5f213..7c315fb3f 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -109,8 +109,10 @@ package like Terraform: ```bash #! /usr/bin/env nix -#! nix shell --impure --expr -#! nix "with (import (builtins.getFlake ''nixpkgs'') {}); terraform.withPlugins (plugins: [ plugins.openstack ])" +#! nix shell --impure --expr `` +#! nix with (import (builtins.getFlake ''nixpkgs'') {}); +#! nix terraform.withPlugins (plugins: [ plugins.openstack ]) +#! nix `` #! nix --command bash terraform "$@" @@ -118,7 +120,7 @@ terraform "$@" > **Note** > -> You must use double quotes (`"`) when passing a simple Nix expression +> You must use double backticks (```` `` ````) when passing a simple Nix expression > in a nix shell shebang. Finally, using the merging of multiple nix shell shebangs the following diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 28b5e4e0f..a0a34ffa9 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -80,6 +80,7 @@ chmod +x "$nonFlakeDir/shebang.sh" git -C "$nonFlakeDir" add README.md shebang.sh git -C "$nonFlakeDir" commit -m 'Initial' +# this also tests a fairly trivial double backtick quoted string, ``--command`` cat > $nonFlakeDir/shebang-comments.sh < $nonFlakeDir/shebang-comments.sh < $nonFlakeDir/shebang-reject.sh <&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' From 198bc22e3b856bf2a86225c2ce5b3a7394e3ac0c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 May 2023 19:54:54 +0200 Subject: [PATCH 561/909] nix: Add command baseDir to parse --expr relative to shebang script --- doc/manual/src/release-notes/rl-next.md | 1 + src/libcmd/installables.cc | 3 ++- src/libutil/args.cc | 9 +++++++++ src/libutil/args.hh | 20 ++++++++++++++++++++ tests/functional/flakes/flakes.sh | 19 +++++++++++++++++++ 5 files changed, 51 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 4bff3c685..28b6d75f5 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -4,6 +4,7 @@ contents of any `#! nix` lines and the script's location to a single call. Verbatim strings may be passed in double backtick (```` `` ````) quotes. + `--expr` resolves relative paths based on the shebang script location. Some examples: ``` diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index e7f58556f..528643dc5 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -445,7 +445,8 @@ Installables SourceExprCommand::parseInstallables( else if (file) state->evalFile(lookupFileArg(*state, *file), *vFile); else { - auto e = state->parseExprFromString(*expr, state->rootPath(CanonPath::fromCwd())); + CanonPath dir(CanonPath::fromCwd(getCommandBaseDir())); + auto e = state->parseExprFromString(*expr, state->rootPath(dir)); state->eval(e, *vFile); } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index ab6e0e266..0012b3f47 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -276,6 +276,7 @@ void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) cmdline.push_back(word); } cmdline.push_back(script); + commandBaseDir = dirOf(script); for (auto pos = savedArgs.begin(); pos != savedArgs.end();pos++) cmdline.push_back(*pos); } @@ -336,6 +337,14 @@ void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) d.completer(*completions, d.n, d.prefix); } +Path Args::getCommandBaseDir() const +{ + if (parent) + return parent->getCommandBaseDir(); + else + return commandBaseDir; +} + bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) { assert(pos != end); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index e753dcaf6..9c942606e 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -24,6 +24,16 @@ class AddCompletions; class Args { + /** + * @brief The command's "working directory", but only set when top level. + * + * Use getCommandBaseDir() to get the directory regardless of whether this + * is a top-level command or subcommand. + * + * @see getCommandBaseDir() + */ + Path commandBaseDir = "."; + public: /** @@ -44,6 +54,16 @@ public: */ virtual std::string doc() { return ""; } + /** + * @brief Get the base directory for the command. + * + * @return Generally the working directory, but in case of a shebang + * interpreter, returns the directory of the script. + * + * This only returns the correct value after parseCmdline() has run. + */ + Path getCommandBaseDir() const; + protected: /** diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index a0a34ffa9..76f3495dd 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -105,6 +105,24 @@ foo EOF chmod +x $nonFlakeDir/shebang-reject.sh +cat > $nonFlakeDir/shebang-inline-expr.sh <> $nonFlakeDir/shebang-inline-expr.sh <<"EOF" +#! nix --offline shell +#! nix --impure --expr `` +#! nix let flake = (builtins.getFlake (toString ../flake1)).packages; +#! nix fooScript = flake.${builtins.currentSystem}.fooScript; +#! nix /* just a comment !@#$%^&*()__+ # */ +#! nix in fooScript +#! nix `` +#! nix --no-write-lock-file --command bash +set -ex +foo +echo "$@" +EOF +chmod +x $nonFlakeDir/shebang-inline-expr.sh + # Construct a custom registry, additionally test the --registry flag nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" nix registry add --registry "$registry" flake2 "git+file://$percentEncodedFlake2Dir" @@ -552,4 +570,5 @@ expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock [[ $($nonFlakeDir/shebang.sh) = "foo" ]] [[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]] [[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]] +[[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' From 466271568be7d3bcf0151dc7e09899775ac31f13 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 May 2023 19:56:04 +0200 Subject: [PATCH 562/909] nix: Parse --file relative to shebang script --- doc/manual/src/release-notes/rl-next.md | 2 +- src/libcmd/installables.cc | 5 +++-- tests/functional/flakes/flakes.sh | 20 ++++++++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 28b6d75f5..7eae5d96e 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -4,7 +4,7 @@ contents of any `#! nix` lines and the script's location to a single call. Verbatim strings may be passed in double backtick (```` `` ````) quotes. - `--expr` resolves relative paths based on the shebang script location. + `--file` and `--expr` resolve relative paths based on the script location. Some examples: ``` diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 528643dc5..d897a01c4 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -442,8 +442,9 @@ Installables SourceExprCommand::parseInstallables( auto e = state->parseStdin(); state->eval(e, *vFile); } - else if (file) - state->evalFile(lookupFileArg(*state, *file), *vFile); + else if (file) { + state->evalFile(lookupFileArg(*state, *file, CanonPath::fromCwd(getCommandBaseDir())), *vFile); + } else { CanonPath dir(CanonPath::fromCwd(getCommandBaseDir())); auto e = state->parseExprFromString(*expr, state->rootPath(dir)); diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 76f3495dd..ccf1699f9 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -123,6 +123,25 @@ echo "$@" EOF chmod +x $nonFlakeDir/shebang-inline-expr.sh +cat > $nonFlakeDir/fooScript.nix <<"EOF" +let flake = (builtins.getFlake (toString ../flake1)).packages; + fooScript = flake.${builtins.currentSystem}.fooScript; + in fooScript +EOF + +cat > $nonFlakeDir/shebang-file.sh <> $nonFlakeDir/shebang-file.sh <<"EOF" +#! nix --offline shell +#! nix --impure --file ./fooScript.nix +#! nix --no-write-lock-file --command bash +set -ex +foo +echo "$@" +EOF +chmod +x $nonFlakeDir/shebang-file.sh + # Construct a custom registry, additionally test the --registry flag nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" nix registry add --registry "$registry" flake2 "git+file://$percentEncodedFlake2Dir" @@ -571,4 +590,5 @@ expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock [[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]] [[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]] [[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] +[[ $($nonFlakeDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]] expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' From 51bb69535b76060582f91e5c044d5752d8e3998b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 May 2023 19:57:36 +0200 Subject: [PATCH 563/909] nix/installables.cc: Use getCommandBaseDir() where possible These usages of the working directory are perhaps unlikely to interact with shebangs, but the code is more consistent this way, and we're less likely to miss usages that do interact. --- src/libcmd/installables.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index d897a01c4..f840865d2 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -88,7 +88,7 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), - parseFlakeRef(flakeRef, absPath("."), true)); + parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true)); }}, .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { if (n == 0) { @@ -130,7 +130,7 @@ MixFlakeOptions::MixFlakeOptions() auto evalState = getEvalState(); auto flake = flake::lockFlake( *evalState, - parseFlakeRef(flakeRef, absPath(".")), + parseFlakeRef(flakeRef, absPath(getCommandBaseDir())), { .writeLockFile = false }); for (auto & [inputName, input] : flake.lockFile.root->inputs) { auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes @@ -294,7 +294,7 @@ void completeFlakeRefWithFragment( prefixRoot = "."; } auto flakeRefS = std::string(prefix.substr(0, hash)); - auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); + auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(getCommandBaseDir())); auto evalCache = openEvalCache(*evalState, std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); @@ -482,7 +482,7 @@ Installables SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(".")); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir())); result.push_back(make_ref( this, getEvalState(), @@ -756,7 +756,7 @@ std::vector RawInstallablesCommand::getFlakeRefsForCompletion() for (auto i : rawInstallables) res.push_back(parseFlakeRefWithFragment( expandTilde(i), - absPath(".")).first); + absPath(getCommandBaseDir())).first); return res; } @@ -778,7 +778,7 @@ std::vector InstallableCommand::getFlakeRefsForCompletion() return { parseFlakeRefWithFragment( expandTilde(_installable), - absPath(".")).first + absPath(getCommandBaseDir())).first }; } From e91fd837ee997cc1879cc9035158260f3dc7cf67 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 23 Oct 2023 16:16:51 +0200 Subject: [PATCH 564/909] Move shebang docs from rl-next to nix.md --- doc/manual/src/release-notes/rl-next.md | 48 ------------------- src/nix/nix.md | 61 +++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 48 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 7eae5d96e..608699270 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -2,54 +2,6 @@ - The experimental nix command is now a `#!-interpreter` by appending the contents of any `#! nix` lines and the script's location to a single call. - - Verbatim strings may be passed in double backtick (```` `` ````) quotes. - `--file` and `--expr` resolve relative paths based on the script location. - - Some examples: - ``` - #!/usr/bin/env nix - #! nix shell --file ```` hello --command bash - - hello | cowsay - ``` - or with flakes: - ``` - #!/usr/bin/env nix - #! nix shell nixpkgs#bash nixpkgs#hello nixpkgs#cowsay --command bash - - hello | cowsay - ``` - or - ```bash - #! /usr/bin/env nix - #! nix shell --impure --expr `` - #! nix with (import (builtins.getFlake "nixpkgs") {}); - #! nix terraform.withPlugins (plugins: [ plugins.openstack ]) - #! nix `` - #! nix --command bash - - terraform "$@" - ``` - or - ``` - #!/usr/bin/env nix - //! ```cargo - //! [dependencies] - //! time = "0.1.25" - //! ``` - /* - #!nix shell nixpkgs#rustc nixpkgs#rust-script nixpkgs#cargo --command rust-script - */ - fn main() { - for argument in std::env::args().skip(1) { - println!("{}", argument); - }; - println!("{}", std::env::var("HOME").expect("")); - println!("{}", time::now().rfc822z()); - } - // vim: ft=rust - ``` - [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. - [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). diff --git a/src/nix/nix.md b/src/nix/nix.md index 6e7e8a649..5bf82a8bf 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -238,4 +238,65 @@ operate are determined as follows: Most `nix` subcommands operate on a *Nix store*. These are documented in [`nix help-stores`](./nix3-help-stores.md). +# Shebang interpreter + +The `nix` command can be used as a `#!` interpreter. +Arguments to Nix can be passed on subsequent lines in the script. + +Verbatim strings may be passed in double backtick (```` `` ````) quotes. + +`--file` and `--expr` resolve relative paths based on the script location. + +Examples: + +``` +#!/usr/bin/env nix +#! nix shell --file ```` hello cowsay --command bash + +hello | cowsay +``` + +or with **flakes**: + +``` +#!/usr/bin/env nix +#! nix shell nixpkgs#bash nixpkgs#hello nixpkgs#cowsay --command bash + +hello | cowsay +``` + +or with an **expression**: + +```bash +#! /usr/bin/env nix +#! nix shell --impure --expr `` +#! nix with (import (builtins.getFlake "nixpkgs") {}); +#! nix terraform.withPlugins (plugins: [ plugins.openstack ]) +#! nix `` +#! nix --command bash + +terraform "$@" +``` + +or with cascading interpreters. Note that the `#! nix` lines don't need to follow after the first line, to accomodate other interpreters. + +``` +#!/usr/bin/env nix +//! ```cargo +//! [dependencies] +//! time = "0.1.25" +//! ``` +/* +#!nix shell nixpkgs#rustc nixpkgs#rust-script nixpkgs#cargo --command rust-script +*/ +fn main() { + for argument in std::env::args().skip(1) { + println!("{}", argument); + }; + println!("{}", std::env::var("HOME").expect("")); + println!("{}", time::now().rfc822z()); +} +// vim: ft=rust +``` + )"" From ffd414eb756dcb3c64348551d5dbaf674c0d4900 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 23 Oct 2023 18:38:54 +0200 Subject: [PATCH 565/909] Fix nix shebang interaction with #8131 overhaul completions --- src/libcmd/installables.cc | 4 +++- src/libutil/args.cc | 19 ++++++++----------- src/libutil/args.hh | 17 +---------------- src/libutil/args/root.hh | 14 +++++++++++++- 4 files changed, 25 insertions(+), 29 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index f840865d2..1c6103020 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -294,7 +294,9 @@ void completeFlakeRefWithFragment( prefixRoot = "."; } auto flakeRefS = std::string(prefix.substr(0, hash)); - auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(getCommandBaseDir())); + + // TODO: ideally this would use the command base directory instead of assuming ".". + auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 0012b3f47..5ba1e5c55 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -80,12 +80,6 @@ std::optional RootArgs::needsCompletion(std::string_view s) return {}; } -void RootArgs::parseCmdline(const Strings & _cmdline) -{ - // Default via 5.1.2.2.1 in C standard - Args::parseCmdline(_cmdline, false); -} - /** * Basically this is `typedef std::optional Parser(std::string_view s, Strings & r);` * @@ -227,7 +221,7 @@ static Strings parseShebangContent(std::string_view s) { return result; } -void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) +void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) { Strings pendingArgs; bool dashDash = false; @@ -339,10 +333,13 @@ void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) Path Args::getCommandBaseDir() const { - if (parent) - return parent->getCommandBaseDir(); - else - return commandBaseDir; + assert(parent); + return parent->getCommandBaseDir(); +} + +Path RootArgs::getCommandBaseDir() const +{ + return commandBaseDir; } bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 9c942606e..30a44cd10 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -24,24 +24,9 @@ class AddCompletions; class Args { - /** - * @brief The command's "working directory", but only set when top level. - * - * Use getCommandBaseDir() to get the directory regardless of whether this - * is a top-level command or subcommand. - * - * @see getCommandBaseDir() - */ - Path commandBaseDir = "."; public: - /** - * Parse the command line with argv0, throwing a UsageError if something - goes wrong. - */ - void parseCmdline(const Strings & _cmdline, bool allowShebang); - /** * Return a short one-line description of the command. */ @@ -62,7 +47,7 @@ public: * * This only returns the correct value after parseCmdline() has run. */ - Path getCommandBaseDir() const; + virtual Path getCommandBaseDir() const; protected: diff --git a/src/libutil/args/root.hh b/src/libutil/args/root.hh index bb98732a1..5c55c37a5 100644 --- a/src/libutil/args/root.hh +++ b/src/libutil/args/root.hh @@ -29,14 +29,26 @@ struct Completions final : AddCompletions */ class RootArgs : virtual public Args { + /** + * @brief The command's "working directory", but only set when top level. + * + * Use getCommandBaseDir() to get the directory regardless of whether this + * is a top-level command or subcommand. + * + * @see getCommandBaseDir() + */ + Path commandBaseDir = "."; + public: /** Parse the command line, throwing a UsageError if something goes * wrong. */ - void parseCmdline(const Strings & cmdline); + void parseCmdline(const Strings & cmdline, bool allowShebang = false); std::shared_ptr completions; + Path getCommandBaseDir() const override; + protected: friend class Args; From 589d3387769b18de9c8d42035eea7ac1e21c6fde Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 6 Nov 2023 18:19:14 +0100 Subject: [PATCH 566/909] parseShebangs: Make strings with backtick sequences representable --- src/libutil/args.cc | 32 ++++++++++++- src/libutil/args.hh | 2 + src/libutil/tests/args.cc | 94 +++++++++++++++++++++++++++++++++++++++ src/nix/nix.md | 4 +- 4 files changed, 129 insertions(+), 3 deletions(-) create mode 100644 src/libutil/tests/args.cc diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 5ba1e5c55..4359c5e8e 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -189,12 +189,40 @@ void ParseQuoted::operator()(std::shared_ptr &state, Strings & r) { throw Error("unterminated quoted string in nix shebang"); } switch (remaining[0]) { + case ' ': + if ((remaining.size() == 3 && remaining[1] == '`' && remaining[2] == '`') + || (remaining.size() > 3 && remaining[1] == '`' && remaining[2] == '`' && remaining[3] != '`')) { + // exactly two backticks mark the end of a quoted string, but a preceding space is ignored if present. + state = std::make_shared(ParseUnquoted(remaining.substr(3))); + r.push_back(acc); + return; + } + else { + // just a normal space + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } case '`': - if (remaining.size() > 1 && remaining[1] == '`') { + // exactly two backticks mark the end of a quoted string + if ((remaining.size() == 2 && remaining[1] == '`') + || (remaining.size() > 2 && remaining[1] == '`' && remaining[2] != '`')) { state = std::make_shared(ParseUnquoted(remaining.substr(2))); r.push_back(acc); return; } + + // a sequence of at least 3 backticks is one escape-backtick which is ignored, followed by any number of backticks, which are verbatim + else if (remaining.size() >= 3 && remaining[1] == '`' && remaining[2] == '`') { + // ignore "escape" backtick + remaining = remaining.substr(1); + // add the rest + while (remaining.size() > 0 && remaining[0] == '`') { + acc += '`'; + remaining = remaining.substr(1); + } + return; + } else { acc += remaining[0]; remaining = remaining.substr(1); @@ -208,7 +236,7 @@ void ParseQuoted::operator()(std::shared_ptr &state, Strings & r) { assert(false); } -static Strings parseShebangContent(std::string_view s) { +Strings parseShebangContent(std::string_view s) { Strings result; std::shared_ptr parserState(std::make_shared(ParseUnquoted(s))); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 30a44cd10..7af82b178 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -409,4 +409,6 @@ public: virtual void add(std::string completion, std::string description = "") = 0; }; +Strings parseShebangContent(std::string_view s); + } diff --git a/src/libutil/tests/args.cc b/src/libutil/tests/args.cc new file mode 100644 index 000000000..e7a16b0be --- /dev/null +++ b/src/libutil/tests/args.cc @@ -0,0 +1,94 @@ +#include "../args.hh" +#include + +#include + +namespace nix { + + TEST(parseShebangContent, basic) { + std::list r = parseShebangContent("hi there"); + ASSERT_EQ(r.size(), 2); + auto i = r.begin(); + ASSERT_EQ(*i++, "hi"); + ASSERT_EQ(*i++, "there"); + } + + TEST(parseShebangContent, empty) { + std::list r = parseShebangContent(""); + ASSERT_EQ(r.size(), 0); + } + + TEST(parseShebangContent, doubleBacktick) { + std::list r = parseShebangContent("``\"ain't that nice\"``"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, "\"ain't that nice\""); + } + + TEST(parseShebangContent, doubleBacktickEmpty) { + std::list r = parseShebangContent("````"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, ""); + } + + TEST(parseShebangContent, doubleBacktickMarkdownInlineCode) { + std::list r = parseShebangContent("``# I'm markdown section about `coolFunction` ``"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, "# I'm markdown section about `coolFunction`"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlockNaive) { + std::list r = parseShebangContent("``Example 1\n```nix\na: a\n``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n``nix\na: a\n``"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlockCorrect) { + std::list r = parseShebangContent("``Example 1\n````nix\na: a\n```` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n```nix\na: a\n```"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlock2) { + std::list r = parseShebangContent("``Example 1\n````nix\na: a\n````\nExample 2\n````nix\na: a\n```` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n```nix\na: a\n```\nExample 2\n```nix\na: a\n```"); + } + + TEST(parseShebangContent, singleBacktickInDoubleBacktickQuotes) { + std::list r = parseShebangContent("``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "`"); + } + + TEST(parseShebangContent, singleBacktickAndSpaceInDoubleBacktickQuotes) { + std::list r = parseShebangContent("``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "` "); + } + + TEST(parseShebangContent, doubleBacktickInDoubleBacktickQuotes) { + std::list r = parseShebangContent("````` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "``"); + } + + TEST(parseShebangContent, increasingQuotes) { + std::list r = parseShebangContent("```` ``` `` ````` `` `````` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 4); + ASSERT_EQ(*i++, ""); + ASSERT_EQ(*i++, "`"); + ASSERT_EQ(*i++, "``"); + ASSERT_EQ(*i++, "```"); + } + +} \ No newline at end of file diff --git a/src/nix/nix.md b/src/nix/nix.md index 5bf82a8bf..eb150f03b 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -243,7 +243,9 @@ in [`nix help-stores`](./nix3-help-stores.md). The `nix` command can be used as a `#!` interpreter. Arguments to Nix can be passed on subsequent lines in the script. -Verbatim strings may be passed in double backtick (```` `` ````) quotes. +Verbatim strings may be passed in double backtick (```` `` ````) quotes. +Sequences of _n_ backticks of 3 or longer are parsed as _n-1_ literal backticks. +A single space before the closing ```` `` ```` is ignored if present. `--file` and `--expr` resolve relative paths based on the script location. From ab69dc4da3ce5dc270e11b460c5b99f549bcf5d3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 6 Nov 2023 19:15:36 +0100 Subject: [PATCH 567/909] Test parseShebangContent round trip --- src/libutil/tests/args.cc | 74 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/src/libutil/tests/args.cc b/src/libutil/tests/args.cc index e7a16b0be..bea74a8c8 100644 --- a/src/libutil/tests/args.cc +++ b/src/libutil/tests/args.cc @@ -1,7 +1,9 @@ #include "../args.hh" +#include "libutil/fs-sink.hh" #include #include +#include namespace nix { @@ -91,4 +93,76 @@ namespace nix { ASSERT_EQ(*i++, "```"); } + +#ifndef COVERAGE + +// quick and dirty +static inline std::string escape(std::string_view s_) { + + std::string_view s = s_; + std::string r = "``"; + + // make a guess to allocate ahead of time + r.reserve( + // plain chars + s.size() + // quotes + + 5 + // some "escape" backticks + + s.size() / 8); + + while (!s.empty()) { + if (s[0] == '`' && s.size() >= 2 && s[1] == '`') { + // escape it + r += "`"; + while (!s.empty() && s[0] == '`') { + r += "`"; + s = s.substr(1); + } + } else { + r += s[0]; + s = s.substr(1); + } + } + + if (!r.empty() + && ( + r[r.size() - 1] == '`' + || r[r.size() - 1] == ' ' + )) { + r += " "; + } + + r += "``"; + + return r; +}; + +RC_GTEST_PROP( + parseShebangContent, + prop_round_trip_single, + (const std::string & orig)) +{ + auto escaped = escape(orig); + // RC_LOG() << "escaped: <[[" << escaped << "]]>" << std::endl; + auto ss = parseShebangContent(escaped); + RC_ASSERT(ss.size() == 1); + RC_ASSERT(*ss.begin() == orig); +} + +RC_GTEST_PROP( + parseShebangContent, + prop_round_trip_two, + (const std::string & one, const std::string & two)) +{ + auto ss = parseShebangContent(escape(one) + " " + escape(two)); + RC_ASSERT(ss.size() == 2); + auto i = ss.begin(); + RC_ASSERT(*i++ == one); + RC_ASSERT(*i++ == two); +} + + +#endif + } \ No newline at end of file From c0c7c4b6cd1aefaa65fc11fcdc8df7e608960825 Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Tue, 7 Nov 2023 21:16:18 +0100 Subject: [PATCH 568/909] Link to shebang interpreter docs from release notes --- doc/manual/src/release-notes/rl-next.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 608699270..da81ed83b 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,7 +1,8 @@ # Release X.Y (202?-??-??) -- The experimental nix command is now a `#!-interpreter` by appending the - contents of any `#! nix` lines and the script's location to a single call. +- The experimental nix command can now act as a [shebang interpreter](@docroot@/command-ref/new-cli/nix.md#shebang-interpreter) + by appending the contents of any `#! nix` lines and the script's location to a single call. + - [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. - [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). From 6a47629530469b84d33444119e43c61effa88aa4 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 7 Nov 2023 13:38:52 +0100 Subject: [PATCH 569/909] Fix initialization of struct members (wrong order) --- src/libstore/nar-accessor.cc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 02993680f..58740b685 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -60,12 +60,22 @@ struct NarAccessor : public SourceAccessor void createDirectory(const Path & path) override { - createMember(path, {Type::tDirectory, false, 0, 0}); + createMember(path, NarMember{ .stat = { + .type = Type::tDirectory, + .fileSize = 0, + .isExecutable = false, + .narOffset = 0 + } }); } void createRegularFile(const Path & path) override { - createMember(path, {Type::tRegular, false, 0, 0}); + createMember(path, NarMember{ .stat = { + .type = Type::tRegular, + .fileSize = 0, + .isExecutable = false, + .narOffset = 0 + } }); } void closeRegularFile() override From 77dceb2844276217bff321d80f601297f3581530 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 7 Nov 2023 13:39:10 +0100 Subject: [PATCH 570/909] Drop obsolete assert and cast --- src/libstore/nar-accessor.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 58740b685..cfbbbd80b 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -88,9 +88,8 @@ struct NarAccessor : public SourceAccessor void preallocateContents(uint64_t size) override { - assert(size <= std::numeric_limits::max()); auto & st = parents.top()->stat; - st.fileSize = (uint64_t) size; + st.fileSize = size; st.narOffset = pos; } From c581143e0c6721fba455e6616e7c6f07e47000b1 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 7 Nov 2023 13:39:30 +0100 Subject: [PATCH 571/909] Use structured binding for json iteration --- src/libstore/nar-accessor.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index cfbbbd80b..1a4936736 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -137,9 +137,8 @@ struct NarAccessor : public SourceAccessor if (type == "directory") { member.stat = {.type = Type::tDirectory}; - for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) { - std::string name = i.key(); - recurse(member.children[name], i.value()); + for (const auto &[name, function] : v["entries"].items()) { + recurse(member.children[name], function); } } else if (type == "regular") { member.stat = { From df8bfe84cca62c89417d676af2c6fbe3bcf23412 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 7 Nov 2023 13:40:21 +0100 Subject: [PATCH 572/909] Fix consts and casts --- src/libstore/nar-accessor.cc | 6 +++--- src/libstore/nar-accessor.hh | 2 +- src/libstore/nar-info.cc | 4 ++-- src/libstore/nar-info.hh | 4 ++-- src/libstore/worker-protocol.cc | 8 ++++---- src/libstore/worker-protocol.hh | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 1a4936736..15b05fe25 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -161,7 +161,7 @@ struct NarAccessor : public SourceAccessor { NarMember * current = &root; - for (auto & i : path) { + for (const auto & i : path) { if (current->stat.type != Type::tDirectory) return nullptr; auto child = current->children.find(std::string(i)); if (child == current->children.end()) return nullptr; @@ -194,7 +194,7 @@ struct NarAccessor : public SourceAccessor throw Error("path '%1%' inside NAR file is not a directory", path); DirEntries res; - for (auto & child : i.children) + for (const auto & child : i.children) res.insert_or_assign(child.first, std::nullopt); return res; @@ -259,7 +259,7 @@ json listNar(ref accessor, const CanonPath & path, bool recurse) { obj["entries"] = json::object(); json &res2 = obj["entries"]; - for (auto & [name, type] : accessor->readDirectory(path)) { + for (const auto & [name, type] : accessor->readDirectory(path)) { if (recurse) { res2[name] = listNar(accessor, path + name, true); } else diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 433774524..0043897c6 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -25,7 +25,7 @@ ref makeNarAccessor(Source & source); * readFile() method of the accessor to get the contents of files * inside the NAR. */ -typedef std::function GetNarBytes; +using GetNarBytes = std::function; ref makeLazyNarAccessor( const std::string & listing, diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index ae2223fb0..1060a6c8b 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -38,12 +38,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & while (pos < s.size()) { size_t colon = s.find(':', pos); - if (colon == std::string::npos) throw corrupt("expecting ':'"); + if (colon == s.npos) throw corrupt("expecting ':'"); std::string name(s, pos, colon - pos); size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) throw corrupt("expecting '\\n'"); + if (eol == s.npos) throw corrupt("expecting '\\n'"); std::string value(s, colon + 2, eol - colon - 2); diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index cec65ff70..fd538a7cd 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -17,10 +17,10 @@ struct NarInfo : ValidPathInfo uint64_t fileSize = 0; NarInfo() = delete; - NarInfo(const Store & store, std::string && name, ContentAddressWithReferences && ca, Hash narHash) + NarInfo(const Store & store, std::string name, ContentAddressWithReferences ca, Hash narHash) : ValidPathInfo(store, std::move(name), std::move(ca), narHash) { } - NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } + NarInfo(StorePath path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 1d202f8d1..7118558b1 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -31,14 +31,14 @@ std::optional WorkerProto::Serialise>::r void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & optTrusted) { if (!optTrusted) - conn.to << (uint8_t)0; + conn.to << uint8_t{0}; else { switch (*optTrusted) { case Trusted: - conn.to << (uint8_t)1; + conn.to << uint8_t{1}; break; case NotTrusted: - conn.to << (uint8_t)2; + conn.to << uint8_t{2}; break; default: assert(false); @@ -101,7 +101,7 @@ void WorkerProto::Serialise::write(const Store & store, Worker BuildResult WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { BuildResult res; - res.status = (BuildResult::Status) readInt(conn.from); + res.status = static_cast(readInt(conn.from)); conn.from >> res.errorMsg; if (GET_PROTOCOL_MINOR(conn.version) >= 29) { conn.from diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index dcd54ad16..25d544ba7 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -171,7 +171,7 @@ enum struct WorkerProto::Op : uint64_t */ inline Sink & operator << (Sink & sink, WorkerProto::Op op) { - return sink << (uint64_t) op; + return sink << static_cast(op); } /** @@ -181,7 +181,7 @@ inline Sink & operator << (Sink & sink, WorkerProto::Op op) */ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op) { - return s << (uint64_t) op; + return s << static_cast(op); } /** From d854e8696b549de15ac9960736a39302d7846ece Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 4 Nov 2023 15:57:43 -0400 Subject: [PATCH 573/909] Specify the size of the experimental feature array in a more robust way See doc comment for details. --- src/libutil/experimental-features.cc | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 47edca3a5..6b9427423 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,19 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +/** + * If two different PRs both add an experimental feature, and we just + * used a number for this, we *woudln't* get merge conflict and the + * counter will be incremented once instead of twice, causing a build + * failure. + * + * By instead defining this instead as 1 + the bottom experimental + * feature, we either have no issue at all if few features are not added + * at the end of the list, or a proper merge conflict if they are. + */ +constexpr size_t numXpFeatures = 1 + static_cast(Xp::VerifiedFetches); + +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", From f0adb72c238aa6f21c2f07fe2e434a3adcea975d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 8 Nov 2023 23:08:05 -0500 Subject: [PATCH 574/909] Mark `fetchTree` as unstable again As discussed in our last meeting, we need a bit more time, but we are "time boxing" the work left to do to ensure there is not unbounded delay. Rather than putting it back underneath `flakes`, though, put it underneath its own `fetch-tree` experimental feature (which `flakes` includes/implies). This signals our commitment to the plan to stabilize it first without waiting to go through the rest of Flakes, and also will give users a "release candidate" when we get closer to stabilization. This reverts commit 4112dd1fc93c9ff03a5a4e8be773c45ebefbbd1f. --- doc/manual/src/release-notes/rl-next.md | 3 ++- src/libexpr/primops/fetchTree.cc | 1 + src/libutil/config.cc | 8 ++++++-- src/libutil/experimental-features.cc | 15 +++++++++++++++ src/libutil/experimental-features.hh | 1 + tests/functional/config.sh | 3 ++- 6 files changed, 27 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index da81ed83b..addb7de71 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -17,7 +17,8 @@ - `nix-shell` shebang lines now support single-quoted arguments. -- `builtins.fetchTree` is now marked as stable. +- `builtins.fetchTree` is now unstable under its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). + As described in the document for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of Flakes. - The interface for creating and updating lock files has been overhauled: diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 3717b9022..8031bf809 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -228,6 +228,7 @@ static RegisterPrimOp primop_fetchTree({ ``` )", .fun = prim_fetchTree, + .experimentalFeature = Xp::FetchTree, }); static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v, diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8e7901133..eddc4a588 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -330,9 +330,13 @@ template<> std::set BaseSetting res; for (auto & s : tokenizeString(str)) { - if (auto thisXpFeature = parseExperimentalFeature(s); thisXpFeature) + if (auto thisXpFeature = parseExperimentalFeature(s); thisXpFeature) { res.insert(thisXpFeature.value()); - else + // FIXME: Replace this hack with a proper notion of + // experimental feature implications/dependencies. + if (thisXpFeature.value() == Xp::Flakes) + res.insert(Xp::FetchTree); + } else warn("unknown experimental feature '%s'", s); } return res; diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 6b9427423..b0edbe185 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -74,6 +74,21 @@ constexpr std::array xpFeatureDetails flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details. )", }, + { + .tag = Xp::FetchTree, + .name = "fetch-tree", + .description = R"( + Enable the use of the [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) built-in function in the Nix language. + + `fetchTree` exposes a larger suite of fetching functionality in a more systematic way. + The same fetching functionality is always used for for + [`flakes`](#xp-feature-flakes). + + This built-in was previously guarded by the `flakes` experimental feature because of that overlap, + but since the plan is to work on stabilizing this first (due 2024 Q1), we are putting it underneath a separate feature. + Once we've made the changes we want to make, enabling just this feature will serve as a "release candidate" --- allowing users to try out the functionality we want to stabilize and not any other functionality we don't yet want to, in isolation. + )", + }, { .tag = Xp::NixCommand, .name = "nix-command", diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index f005cc9ee..f580fd030 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -20,6 +20,7 @@ enum struct ExperimentalFeature CaDerivations, ImpureDerivations, Flakes, + FetchTree, NixCommand, RecursiveNix, NoUrlLiterals, diff --git a/tests/functional/config.sh b/tests/functional/config.sh index 723f575ed..0780c55d0 100644 --- a/tests/functional/config.sh +++ b/tests/functional/config.sh @@ -50,7 +50,8 @@ exp_cores=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) exp_features=$(nix show-config | grep '^experimental-features' | cut -d '=' -f 2 | xargs) [[ $prev != $exp_cores ]] [[ $exp_cores == "4242" ]] -[[ $exp_features == "flakes nix-command" ]] +# flakes implies fetch-tree +[[ $exp_features == "fetch-tree flakes nix-command" ]] # Test that it's possible to retrieve a single setting's value val=$(nix show-config | grep '^warn-dirty' | cut -d '=' -f 2 | xargs) From 12953b942c7752568070e0b703b448dd8f16f21b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 9 Nov 2023 07:08:56 +0100 Subject: [PATCH 575/909] Fixup docs --- doc/manual/src/release-notes/rl-next.md | 2 +- src/libutil/config.cc | 2 -- src/libutil/experimental-features.cc | 5 ++--- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index addb7de71..1e6ad6922 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -17,7 +17,7 @@ - `nix-shell` shebang lines now support single-quoted arguments. -- `builtins.fetchTree` is now unstable under its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). As described in the document for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of Flakes. - The interface for creating and updating lock files has been overhauled: diff --git a/src/libutil/config.cc b/src/libutil/config.cc index eddc4a588..96a0a4df8 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -332,8 +332,6 @@ template<> std::set BaseSetting(str)) { if (auto thisXpFeature = parseExperimentalFeature(s); thisXpFeature) { res.insert(thisXpFeature.value()); - // FIXME: Replace this hack with a proper notion of - // experimental feature implications/dependencies. if (thisXpFeature.value() == Xp::Flakes) res.insert(Xp::FetchTree); } else diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index b0edbe185..88fb55713 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -80,9 +80,8 @@ constexpr std::array xpFeatureDetails .description = R"( Enable the use of the [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) built-in function in the Nix language. - `fetchTree` exposes a larger suite of fetching functionality in a more systematic way. - The same fetching functionality is always used for for - [`flakes`](#xp-feature-flakes). + `fetchTree` exposes a large suite of fetching functionality in a more systematic way. + The [`flakes`](#xp-feature-flakes) feature flag always enables `fetch-tree`. This built-in was previously guarded by the `flakes` experimental feature because of that overlap, but since the plan is to work on stabilizing this first (due 2024 Q1), we are putting it underneath a separate feature. From a903f85f84b78a28490f3aa9615ba87d070d01d1 Mon Sep 17 00:00:00 2001 From: Artturin Date: Tue, 31 Oct 2023 01:36:13 +0200 Subject: [PATCH 576/909] `nix-env --query`: fix `--json` ignoring `--drv-path` ```json { "AMB-plugins": { "drvPath": "/nix/store/l99cb7h2hy8dg005arsjbd9kx0w05d3h-AMB-plugins-0.8.1.drv", "name": "AMB-plugins-0.8.1", "outputName": "out", "outputs": { "out": null }, "pname": "AMB-plugins", "system": "x86_64-linux", "version": "0.8.1" }, "ArchiSteamFarm": { "drvPath": "/nix/store/nhplgyjj34fz6hjmnyih25gxscfh8s7b-ArchiSteamFarm-5.4.12.5.drv", "name": "ArchiSteamFarm-5.4.12.5", "outputName": "out", "outputs": { "out": null }, "pname": "ArchiSteamFarm", "system": "x86_64-linux", "version": "5.4.12.5" }, ... ``` --- doc/manual/src/release-notes/rl-next.md | 2 ++ src/nix-env/nix-env.cc | 9 +++++++-- tests/functional/user-envs.sh | 1 + 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 276252c37..2163a5392 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -12,6 +12,8 @@ - Introduce a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash). +- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set. + - `nix-shell` shebang lines now support single-quoted arguments. - `builtins.fetchTree` is now marked as stable. diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 01742daa8..558d0d6cd 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -922,7 +922,7 @@ static VersionDiff compareVersionAgainstSet( } -static void queryJSON(Globals & globals, std::vector & elems, bool printOutPath, bool printMeta) +static void queryJSON(Globals & globals, std::vector & elems, bool printOutPath, bool printDrvPath, bool printMeta) { using nlohmann::json; json topObj = json::object(); @@ -953,6 +953,11 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin } } + if (printDrvPath) { + auto drvPath = i.queryDrvPath(); + if (drvPath) pkgObj["drvPath"] = globals.state->store->printStorePath(*drvPath); + } + if (printMeta) { json &metaObj = pkgObj["meta"]; metaObj = json::object(); @@ -1079,7 +1084,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) /* Print the desired columns, or XML output. */ if (jsonOutput) { - queryJSON(globals, elems, printOutPath, printMeta); + queryJSON(globals, elems, printOutPath, printDrvPath, printMeta); cout << '\n'; return; } diff --git a/tests/functional/user-envs.sh b/tests/functional/user-envs.sh index d1260ba04..dcd6b1b97 100644 --- a/tests/functional/user-envs.sh +++ b/tests/functional/user-envs.sh @@ -26,6 +26,7 @@ nix-env -f ./user-envs.nix -qa --json --out-path | jq -e '.[] | select(.name == .outputName == "out", (.outputs.out | test("'$NIX_STORE_DIR'.*-0\\.1")) ] | all' +nix-env -f ./user-envs.nix -qa --json --drv-path | jq -e '.[] | select(.name == "bar-0.1") | (.drvPath | test("'$NIX_STORE_DIR'.*-0\\.1\\.drv"))' # Query descriptions. nix-env -f ./user-envs.nix -qa '*' --description | grepQuiet silly From 1d5a48240cd3c5b81939b0562141772323550d99 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 9 Nov 2023 23:10:42 -0500 Subject: [PATCH 577/909] `.editorconfig`: Also affect Perl FFI `xs` file This way `perl/lib/Nix/Store.xs` is affected. --- .editorconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.editorconfig b/.editorconfig index 887ecadba..86360e658 100644 --- a/.editorconfig +++ b/.editorconfig @@ -17,7 +17,7 @@ indent_style = space indent_size = 2 # Match c++/shell/perl, set indent to spaces with width of four -[*.{hpp,cc,hh,sh,pl}] +[*.{hpp,cc,hh,sh,pl,xs}] indent_style = space indent_size = 4 From cf59ea83ec98522113bf2fd81678537a871d0339 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 Nov 2023 13:58:59 +0100 Subject: [PATCH 578/909] configure: Check for libgit2 --- Makefile.config.in | 7 ++++--- configure.ac | 6 ++++++ src/libfetchers/local.mk | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index 19992fa20..aadece0e1 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -8,7 +8,9 @@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ CXXLTO = @CXXLTO@ EDITLINE_LIBS = @EDITLINE_LIBS@ +ENABLE_BUILD = @ENABLE_BUILD@ ENABLE_S3 = @ENABLE_S3@ +ENABLE_TESTS = @ENABLE_TESTS@ GTEST_LIBS = @GTEST_LIBS@ HAVE_LIBCPUID = @HAVE_LIBCPUID@ HAVE_SECCOMP = @HAVE_SECCOMP@ @@ -17,6 +19,7 @@ LDFLAGS = @LDFLAGS@ LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ LIBCURL_LIBS = @LIBCURL_LIBS@ +LIBGIT2_LIBS = @LIBGIT2_LIBS@ LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@ LOWDOWN_LIBS = @LOWDOWN_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ @@ -35,6 +38,7 @@ docdir = @docdir@ embedded_sandbox_shell = @embedded_sandbox_shell@ exec_prefix = @exec_prefix@ includedir = @includedir@ +internal_api_docs = @internal_api_docs@ libdir = @libdir@ libexecdir = @libexecdir@ localstatedir = @localstatedir@ @@ -46,6 +50,3 @@ sandbox_shell = @sandbox_shell@ storedir = @storedir@ sysconfdir = @sysconfdir@ system = @system@ -ENABLE_BUILD = @ENABLE_BUILD@ -ENABLE_TESTS = @ENABLE_TESTS@ -internal_api_docs = @internal_api_docs@ diff --git a/configure.ac b/configure.ac index 75ce7d01d..1cda0852a 100644 --- a/configure.ac +++ b/configure.ac @@ -335,9 +335,15 @@ AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation doc_generate=$enableval, doc_generate=yes) AC_SUBST(doc_generate) + # Look for lowdown library. PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"]) + +# Look for libgit2. +PKG_CHECK_MODULES([LIBGIT2], [libgit2]) + + # Setuid installations. AC_CHECK_FUNCS([setresuid setreuid lchown]) diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk index f21651d77..266e7a211 100644 --- a/src/libfetchers/local.mk +++ b/src/libfetchers/local.mk @@ -8,6 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc) libfetchers_CXXFLAGS += -I src/libutil -I src/libstore -libfetchers_LDFLAGS += -pthread -lgit2 -larchive +libfetchers_LDFLAGS += -pthread $(LIBGIT2_LIBS) -larchive libfetchers_LIBS = libutil libstore From 3d9d5dc18977d21a04299f4a37b366f9a1d32051 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 3 Nov 2023 00:57:19 -0400 Subject: [PATCH 579/909] Create `MemorySink` This is for writing to a `MemorySourceAccessor`. --- src/libutil/memory-source-accessor.cc | 56 +++++++++++++++++++++++++++ src/libutil/memory-source-accessor.hh | 25 ++++++++++++ 2 files changed, 81 insertions(+) diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index f34f6c091..78a4dd298 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -121,4 +121,60 @@ CanonPath MemorySourceAccessor::addFile(CanonPath path, std::string && contents) return path; } + +using File = MemorySourceAccessor::File; + +void MemorySink::createDirectory(const Path & path) +{ + auto * f = dst.open(CanonPath{path}, File { File::Directory { } }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + + if (!std::holds_alternative(f->raw)) + throw Error("file '%s' is not a directory", path); +}; + +void MemorySink::createRegularFile(const Path & path) +{ + auto * f = dst.open(CanonPath{path}, File { File::Regular {} }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + if (!(r = std::get_if(&f->raw))) + throw Error("file '%s' is not a regular file", path); +} + +void MemorySink::closeRegularFile() +{ + r = nullptr; +} + +void MemorySink::isExecutable() +{ + assert(r); + r->executable = true; +} + +void MemorySink::preallocateContents(uint64_t len) +{ + assert(r); + r->contents.reserve(len); +} + +void MemorySink::receiveContents(std::string_view data) +{ + assert(r); + r->contents += data; +} + +void MemorySink::createSymlink(const Path & path, const std::string & target) +{ + auto * f = dst.open(CanonPath{path}, File { File::Symlink { } }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + if (auto * s = std::get_if(&f->raw)) + s->target = target; + else + throw Error("file '%s' is not a symbolic link", path); +} + } diff --git a/src/libutil/memory-source-accessor.hh b/src/libutil/memory-source-accessor.hh index 014fa8098..b908f3713 100644 --- a/src/libutil/memory-source-accessor.hh +++ b/src/libutil/memory-source-accessor.hh @@ -1,4 +1,5 @@ #include "source-accessor.hh" +#include "fs-sink.hh" #include "variant-wrapper.hh" namespace nix { @@ -71,4 +72,28 @@ struct MemorySourceAccessor : virtual SourceAccessor CanonPath addFile(CanonPath path, std::string && contents); }; +/** + * Write to a `MemorySourceAccessor` at the given path + */ +struct MemorySink : ParseSink +{ + MemorySourceAccessor & dst; + + MemorySink(MemorySourceAccessor & dst) : dst(dst) { } + + void createDirectory(const Path & path) override; + + void createRegularFile(const Path & path) override; + void receiveContents(std::string_view data) override; + void isExecutable() override; + void closeRegularFile() override; + + void createSymlink(const Path & path, const std::string & target) override; + + void preallocateContents(uint64_t size) override; + +private: + MemorySourceAccessor::File::Regular * r; +}; + } From 9afa697ab61ea6bbbb0d88e629b62606681cc744 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 8 Nov 2023 00:30:55 -0500 Subject: [PATCH 580/909] Refactor bash test build system a bit The basic idea here is to separate a few intertwined notions: 1. Not all "run bash tests" are "install tests" 2. Not all "run bash tests" use `tests/functional/init.sh`, or any pre-test initialization at all. This will used in the next commit when we have a test that check unit test golden master data. Also, move our custom `PS4` from the test to the test runner, as it is part of how we want to display the tests, not the test themselves. Co-authored-by: Robert Hensing --- doc/manual/src/contributing/testing.md | 10 ++++---- mk/common-test.sh | 24 ++++++++++++++----- mk/debug-test.sh | 5 +++- mk/lib.mk | 7 +++--- mk/run-test.sh | 5 +++- mk/tests.mk | 21 +++++++++------- .../common/vars-and-functions.sh.in | 2 +- 7 files changed, 48 insertions(+), 26 deletions(-) diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index 3d75ebe7b..0b45b88a3 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -133,17 +133,17 @@ ran test tests/functional/${testName}.sh... [PASS] or without `make`: ```shell-session -$ ./mk/run-test.sh tests/functional/${testName}.sh +$ ./mk/run-test.sh tests/functional/${testName}.sh tests/functional/init.sh ran test tests/functional/${testName}.sh... [PASS] ``` To see the complete output, one can also run: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh -+ foo +$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh ++(${testName}.sh:1) foo output from foo -+ bar ++(${testName}.sh:2) bar output from bar ... ``` @@ -175,7 +175,7 @@ edit it like so: Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh ... + gdb blash blub GNU gdb (GDB) 12.1 diff --git a/mk/common-test.sh b/mk/common-test.sh index 7ab25febf..00ccd1584 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -1,15 +1,27 @@ -test_dir=tests/functional +# Remove overall test dir (at most one of the two should match) and +# remove file extension. +test_name=$(echo -n "$test" | sed \ + -e "s|^unit-test-data/||" \ + -e "s|^tests/functional/||" \ + -e "s|\.sh$||" \ + ) -test=$(echo -n "$test" | sed -e "s|^$test_dir/||") - -TESTS_ENVIRONMENT=("TEST_NAME=${test%.*}" 'NIX_REMOTE=') +TESTS_ENVIRONMENT=( + "TEST_NAME=$test_name" + 'NIX_REMOTE=' + 'PS4=+(${BASH_SOURCE[0]-$0}:$LINENO) ' +) : ${BASH:=/usr/bin/env bash} +run () { + cd "$(dirname $1)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -x -e -u -o pipefail $(basename $1) +} + init_test () { - cd "$test_dir" && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null + run "$init" 2>/dev/null > /dev/null } run_test_proper () { - cd "$test_dir/$(dirname $test)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test) + run "$test" } diff --git a/mk/debug-test.sh b/mk/debug-test.sh index b5b628ecd..52482c01e 100755 --- a/mk/debug-test.sh +++ b/mk/debug-test.sh @@ -3,9 +3,12 @@ set -eu -o pipefail test=$1 +init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" -(init_test) +if [ -n "$init" ]; then + (init_test) +fi run_test_proper diff --git a/mk/lib.mk b/mk/lib.mk index e86a7f1a4..49abe9862 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -122,14 +122,15 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) +install_test_init=tests/functional/init.sh $(foreach test, $(install-tests), \ - $(eval $(call run-install-test,$(test))) \ + $(eval $(call run-test,$(test),$(install_test_init))) \ $(eval installcheck: $(test).test)) $(foreach test-group, $(install-tests-groups), \ - $(eval $(call run-install-test-group,$(test-group))) \ + $(eval $(call run-test-group,$(test-group),$(install_test_init))) \ $(eval installcheck: $(test-group).test-group) \ $(foreach test, $($(test-group)-tests), \ - $(eval $(call run-install-test,$(test))) \ + $(eval $(call run-test,$(test),$(install_test_init))) \ $(eval $(test-group).test-group: $(test).test))) $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file)))))) diff --git a/mk/run-test.sh b/mk/run-test.sh index 1a1d65930..da9c5a473 100755 --- a/mk/run-test.sh +++ b/mk/run-test.sh @@ -8,6 +8,7 @@ yellow="" normal="" test=$1 +init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" @@ -21,7 +22,9 @@ if [ -t 1 ]; then fi run_test () { - (init_test 2>/dev/null > /dev/null) + if [ -n "$init" ]; then + (init_test 2>/dev/null > /dev/null) + fi log="$(run_test_proper 2>&1)" && status=0 || status=$? } diff --git a/mk/tests.mk b/mk/tests.mk index ec8128bdf..bac9b704a 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -2,19 +2,22 @@ test-deps = -define run-install-test +define run-bash - .PHONY: $1.test - $1.test: $1 $(test-deps) - @env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null - - .PHONY: $1.test-debug - $1.test-debug: $1 $(test-deps) - @env BASH=$(bash) $(bash) mk/debug-test.sh $1 < /dev/null + .PHONY: $1 + $1: $2 + @env BASH=$(bash) $(bash) $3 < /dev/null endef -define run-install-test-group +define run-test + + $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1 $2)) + $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1 $2)) + +endef + +define run-test-group .PHONY: $1.test-group diff --git a/tests/functional/common/vars-and-functions.sh.in b/tests/functional/common/vars-and-functions.sh.in index 967d6be54..848988af9 100644 --- a/tests/functional/common/vars-and-functions.sh.in +++ b/tests/functional/common/vars-and-functions.sh.in @@ -4,7 +4,7 @@ if [[ -z "${COMMON_VARS_AND_FUNCTIONS_SH_SOURCED-}" ]]; then COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1 -export PS4='+(${BASH_SOURCE[0]-$0}:$LINENO) ' +set +x export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default/tests\/functional//} export NIX_STORE_DIR From 20b95d622336cf982082d7daf3075339f6edce70 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 4 Nov 2023 15:35:38 -0400 Subject: [PATCH 581/909] Git object hashing in libutil This is the core functionality but just unit-tested and not yet made part of the store layer. This is because there is some tech debt around (a) repeated boilerplate hashing objects (b) better integration of the new `SourceAccessor` type that needs to be cleaned up first. Part of RFC 133 Co-Authored-By: Matthew Bauer Co-Authored-By: Carlo Nucera Co-authored-by: Robert Hensing Co-authored-by: Florian Klink --- src/libutil/experimental-features.cc | 8 + src/libutil/experimental-features.hh | 1 + src/libutil/git.cc | 263 +++++++++++++++++- src/libutil/git.hh | 141 +++++++++- src/libutil/serialise.cc | 4 + src/libutil/serialise.hh | 1 + src/libutil/tests/git.cc | 249 +++++++++++++++-- src/libutil/tests/local.mk | 4 + unit-test-data/libutil/git/check-data.sh | 31 +++ .../libutil/git/hello-world-blob.bin | Bin 0 -> 24 bytes unit-test-data/libutil/git/hello-world.bin | Bin 0 -> 16 bytes unit-test-data/libutil/git/tree.bin | Bin 0 -> 100 bytes unit-test-data/libutil/git/tree.txt | 3 + 13 files changed, 667 insertions(+), 38 deletions(-) create mode 100644 unit-test-data/libutil/git/check-data.sh create mode 100644 unit-test-data/libutil/git/hello-world-blob.bin create mode 100644 unit-test-data/libutil/git/hello-world.bin create mode 100644 unit-test-data/libutil/git/tree.bin create mode 100644 unit-test-data/libutil/git/tree.txt diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 88fb55713..ac4d189e1 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -96,6 +96,14 @@ constexpr std::array xpFeatureDetails [`nix`](@docroot@/command-ref/new-cli/nix.md) for details. )", }, + { + .tag = Xp::GitHashing, + .name = "git-hashing", + .description = R"( + Allow creating (content-addressed) store objects which are hashed via Git's hashing algorithm. + These store objects will not be understandable by older versions of Nix. + )", + }, { .tag = Xp::RecursiveNix, .name = "recursive-nix", diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index f580fd030..c355b8081 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -22,6 +22,7 @@ enum struct ExperimentalFeature Flakes, FetchTree, NixCommand, + GitHashing, RecursiveNix, NoUrlLiterals, FetchClosure, diff --git a/src/libutil/git.cc b/src/libutil/git.cc index f35c2fdb7..a4bd60096 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -1,9 +1,263 @@ -#include "git.hh" - +#include +#include +#include +#include #include +#include // for strcasecmp + +#include "signals.hh" +#include "config.hh" +#include "hash.hh" +#include "posix-source-accessor.hh" + +#include "git.hh" +#include "serialise.hh" + +namespace nix::git { + +using namespace nix; +using namespace std::string_literals; + +std::optional decodeMode(RawMode m) { + switch (m) { + case (RawMode) Mode::Directory: + case (RawMode) Mode::Executable: + case (RawMode) Mode::Regular: + case (RawMode) Mode::Symlink: + return (Mode) m; + default: + return std::nullopt; + } +} + + +static std::string getStringUntil(Source & source, char byte) +{ + std::string s; + char n[1]; + source(std::string_view { n, 1 }); + while (*n != byte) { + s += *n; + source(std::string_view { n, 1 }); + } + return s; +} + + +static std::string getString(Source & source, int n) +{ + std::string v; + v.resize(n); + source(v); + return v; +} + + +void parse( + ParseSink & sink, + const Path & sinkPath, + Source & source, + std::function hook, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + + auto type = getString(source, 5); + + if (type == "blob ") { + sink.createRegularFile(sinkPath); + + unsigned long long size = std::stoi(getStringUntil(source, 0)); + + sink.preallocateContents(size); + + unsigned long long left = size; + std::string buf; + buf.reserve(65536); + + while (left) { + checkInterrupt(); + buf.resize(std::min((unsigned long long)buf.capacity(), left)); + source(buf); + sink.receiveContents(buf); + left -= buf.size(); + } + } else if (type == "tree ") { + unsigned long long size = std::stoi(getStringUntil(source, 0)); + unsigned long long left = size; + + sink.createDirectory(sinkPath); + + while (left) { + std::string perms = getStringUntil(source, ' '); + left -= perms.size(); + left -= 1; + + RawMode rawMode = std::stoi(perms, 0, 8); + auto modeOpt = decodeMode(rawMode); + if (!modeOpt) + throw Error("Unknown Git permission: %o", perms); + auto mode = std::move(*modeOpt); + + std::string name = getStringUntil(source, '\0'); + left -= name.size(); + left -= 1; + + std::string hashs = getString(source, 20); + left -= 20; + + Hash hash(htSHA1); + std::copy(hashs.begin(), hashs.end(), hash.hash); + + hook(name, TreeEntry { + .mode = mode, + .hash = hash, + }); + + if (mode == Mode::Executable) + sink.isExecutable(); + } + } else throw Error("input doesn't look like a Git object"); +} + + +std::optional convertMode(SourceAccessor::Type type) +{ + switch (type) { + case SourceAccessor::tSymlink: return Mode::Symlink; + case SourceAccessor::tRegular: return Mode::Regular; + case SourceAccessor::tDirectory: return Mode::Directory; + case SourceAccessor::tMisc: return std::nullopt; + default: abort(); + } +} + + +void restore(ParseSink & sink, Source & source, std::function hook) +{ + parse(sink, "", source, [&](Path name, TreeEntry entry) { + auto [accessor, from] = hook(entry.hash); + auto stat = accessor->lstat(from); + auto gotOpt = convertMode(stat.type); + if (!gotOpt) + throw Error("file '%s' (git hash %s) has an unsupported type", + from, + entry.hash.to_string(HashFormat::Base16, false)); + auto & got = *gotOpt; + if (got != entry.mode) + throw Error("git mode of file '%s' (git hash %s) is %o but expected %o", + from, + entry.hash.to_string(HashFormat::Base16, false), + (RawMode) got, + (RawMode) entry.mode); + copyRecursive( + *accessor, from, + sink, name); + }); +} + + +void dumpBlobPrefix( + uint64_t size, Sink & sink, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + auto s = fmt("blob %d\0"s, std::to_string(size)); + sink(s); +} + + +void dumpTree(const Tree & entries, Sink & sink, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + + std::string v1; + + for (auto & [name, entry] : entries) { + auto name2 = name; + if (entry.mode == Mode::Directory) { + assert(name2.back() == '/'); + name2.pop_back(); + } + v1 += fmt("%o %s\0"s, static_cast(entry.mode), name2); + std::copy(entry.hash.hash, entry.hash.hash + entry.hash.hashSize, std::back_inserter(v1)); + } + + { + auto s = fmt("tree %d\0"s, v1.size()); + sink(s); + } + + sink(v1); +} + + +Mode dump( + SourceAccessor & accessor, const CanonPath & path, + Sink & sink, + std::function hook, + PathFilter & filter, + const ExperimentalFeatureSettings & xpSettings) +{ + auto st = accessor.lstat(path); + + switch (st.type) { + case SourceAccessor::tRegular: + { + accessor.readFile(path, sink, [&](uint64_t size) { + dumpBlobPrefix(size, sink, xpSettings); + }); + return st.isExecutable + ? Mode::Executable + : Mode::Regular; + } + + case SourceAccessor::tDirectory: + { + Tree entries; + for (auto & [name, _] : accessor.readDirectory(path)) { + auto child = path + name; + if (!filter(child.abs())) continue; + + auto entry = hook(child); + + auto name2 = name; + if (entry.mode == Mode::Directory) + name2 += "/"; + + entries.insert_or_assign(std::move(name2), std::move(entry)); + } + dumpTree(entries, sink, xpSettings); + return Mode::Directory; + } + + case SourceAccessor::tSymlink: + case SourceAccessor::tMisc: + default: + throw Error("file '%1%' has an unsupported type", path); + } +} + + +TreeEntry dumpHash( + HashType ht, + SourceAccessor & accessor, const CanonPath & path, PathFilter & filter) +{ + std::function hook; + hook = [&](const CanonPath & path) -> TreeEntry { + auto hashSink = HashSink(ht); + auto mode = dump(accessor, path, hashSink, hook, filter); + auto hash = hashSink.finish().first; + return { + .mode = mode, + .hash = hash, + }; + }; + + return hook(path); +} -namespace nix { -namespace git { std::optional parseLsRemoteLine(std::string_view line) { @@ -22,4 +276,3 @@ std::optional parseLsRemoteLine(std::string_view line) } } -} diff --git a/src/libutil/git.hh b/src/libutil/git.hh index bf2b9a286..303460072 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -5,9 +5,127 @@ #include #include -namespace nix { +#include "types.hh" +#include "serialise.hh" +#include "hash.hh" +#include "source-accessor.hh" +#include "fs-sink.hh" -namespace git { +namespace nix::git { + +using RawMode = uint32_t; + +enum struct Mode : RawMode { + Directory = 0040000, + Executable = 0100755, + Regular = 0100644, + Symlink = 0120000, +}; + +std::optional decodeMode(RawMode m); + +/** + * An anonymous Git tree object entry (no name part). + */ +struct TreeEntry +{ + Mode mode; + Hash hash; + + GENERATE_CMP(TreeEntry, me->mode, me->hash); +}; + +/** + * A Git tree object, fully decoded and stored in memory. + * + * Directory names must end in a `/` for sake of sorting. See + * https://github.com/mirage/irmin/issues/352 + */ +using Tree = std::map; + +/** + * Callback for processing a child hash with `parse` + * + * The function should + * + * 1. Obtain the file system objects denoted by `gitHash` + * + * 2. Ensure they match `mode` + * + * 3. Feed them into the same sink `parse` was called with + * + * Implementations may seek to memoize resources (bandwidth, storage, + * etc.) for the same Git hash. + */ +using SinkHook = void(const Path & name, TreeEntry entry); + +void parse( + ParseSink & sink, const Path & sinkPath, + Source & source, + std::function hook, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Assists with writing a `SinkHook` step (2). + */ +std::optional convertMode(SourceAccessor::Type type); + +/** + * Simplified version of `SinkHook` for `restore`. + * + * Given a `Hash`, return a `SourceAccessor` and `CanonPath` pointing to + * the file system object with that path. + */ +using RestoreHook = std::pair(Hash); + +/** + * Wrapper around `parse` and `RestoreSink` + */ +void restore(ParseSink & sink, Source & source, std::function hook); + +/** + * Dumps a single file to a sink + * + * @param xpSettings for testing purposes + */ +void dumpBlobPrefix( + uint64_t size, Sink & sink, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Dumps a representation of a git tree to a sink + */ +void dumpTree( + const Tree & entries, Sink & sink, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Callback for processing a child with `dump` + * + * The function should return the Git hash and mode of the file at the + * given path in the accessor passed to `dump`. + * + * Note that if the child is a directory, its child in must also be so + * processed in order to compute this information. + */ +using DumpHook = TreeEntry(const CanonPath & path); + +Mode dump( + SourceAccessor & accessor, const CanonPath & path, + Sink & sink, + std::function hook, + PathFilter & filter = defaultPathFilter, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Recursively dumps path, hashing as we go. + * + * A smaller wrapper around `dump`. + */ +TreeEntry dumpHash( + HashType ht, + SourceAccessor & accessor, const CanonPath & path, + PathFilter & filter = defaultPathFilter); /** * A line from the output of `git ls-remote --symref`. @@ -16,15 +134,17 @@ namespace git { * * - Symbolic references of the form * - * ref: {target} {reference} - * - * where {target} is itself a reference and {reference} is optional + * ``` + * ref: {target} {reference} + * ``` + * where {target} is itself a reference and {reference} is optional * * - Object references of the form * - * {target} {reference} - * - * where {target} is a commit id and {reference} is mandatory + * ``` + * {target} {reference} + * ``` + * where {target} is a commit id and {reference} is mandatory */ struct LsRemoteRefLine { enum struct Kind { @@ -36,8 +156,9 @@ struct LsRemoteRefLine { std::optional reference; }; +/** + * Parse an `LsRemoteRefLine` + */ std::optional parseLsRemoteLine(std::string_view line); } - -} diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 725ddbb8d..d7950b11b 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -74,6 +74,10 @@ void Source::operator () (char * data, size_t len) } } +void Source::operator () (std::string_view data) +{ + (*this)((char *)data.data(), data.size()); +} void Source::drainInto(Sink & sink) { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 9e07226bf..3f57ce88b 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -73,6 +73,7 @@ struct Source * an error if it is not going to be available. */ void operator () (char * data, size_t len); + void operator () (std::string_view data); /** * Store up to ‘len’ in the buffer pointed to by ‘data’, and diff --git a/src/libutil/tests/git.cc b/src/libutil/tests/git.cc index 5b5715fc2..2842ea4d0 100644 --- a/src/libutil/tests/git.cc +++ b/src/libutil/tests/git.cc @@ -1,33 +1,236 @@ -#include "git.hh" #include +#include "git.hh" +#include "memory-source-accessor.hh" + +#include "tests/characterization.hh" + namespace nix { - TEST(GitLsRemote, parseSymrefLineWithReference) { - auto line = "ref: refs/head/main HEAD"; - auto res = git::parseLsRemoteLine(line); - ASSERT_TRUE(res.has_value()); - ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic); - ASSERT_EQ(res->target, "refs/head/main"); - ASSERT_EQ(res->reference, "HEAD"); +using namespace git; + +class GitTest : public CharacterizationTest +{ + Path unitTestData = getUnitTestData() + "/libutil/git"; + +public: + + Path goldenMaster(std::string_view testStem) const override { + return unitTestData + "/" + testStem; } - TEST(GitLsRemote, parseSymrefLineWithNoReference) { - auto line = "ref: refs/head/main"; - auto res = git::parseLsRemoteLine(line); - ASSERT_TRUE(res.has_value()); - ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic); - ASSERT_EQ(res->target, "refs/head/main"); - ASSERT_EQ(res->reference, std::nullopt); - } + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; - TEST(GitLsRemote, parseObjectRefLine) { - auto line = "abc123 refs/head/main"; - auto res = git::parseLsRemoteLine(line); - ASSERT_TRUE(res.has_value()); - ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Object); - ASSERT_EQ(res->target, "abc123"); - ASSERT_EQ(res->reference, "refs/head/main"); +private: + + void SetUp() override + { + mockXpSettings.set("experimental-features", "git-hashing"); } +}; + +TEST(GitMode, gitMode_directory) { + Mode m = Mode::Directory; + RawMode r = 0040000; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST(GitMode, gitMode_executable) { + Mode m = Mode::Executable; + RawMode r = 0100755; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST(GitMode, gitMode_regular) { + Mode m = Mode::Regular; + RawMode r = 0100644; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST(GitMode, gitMode_symlink) { + Mode m = Mode::Symlink; + RawMode r = 0120000; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST_F(GitTest, blob_read) { + readTest("hello-world-blob.bin", [&](const auto & encoded) { + StringSource in { encoded }; + StringSink out; + RegularFileSink out2 { out }; + parse(out2, "", in, [](auto &, auto) {}, mockXpSettings); + + auto expected = readFile(goldenMaster("hello-world.bin")); + + ASSERT_EQ(out.s, expected); + }); } +TEST_F(GitTest, blob_write) { + writeTest("hello-world-blob.bin", [&]() { + auto decoded = readFile(goldenMaster("hello-world.bin")); + StringSink s; + dumpBlobPrefix(decoded.size(), s, mockXpSettings); + s(decoded); + return s.s; + }); +} + +/** + * This data is for "shallow" tree tests. However, we use "real" hashes + * so that we can check our test data in the corresponding functional + * test (`git-hashing/unit-test-data`). + */ +const static Tree tree = { + { + "Foo", + { + .mode = Mode::Regular, + // hello world with special chars from above + .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", htSHA1), + }, + }, + { + "bAr", + { + .mode = Mode::Executable, + // ditto + .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", htSHA1), + }, + }, + { + "baZ/", + { + .mode = Mode::Directory, + // Empty directory hash + .hash = Hash::parseAny("4b825dc642cb6eb9a060e54bf8d69288fbee4904", htSHA1), + }, + }, +}; + +TEST_F(GitTest, tree_read) { + readTest("tree.bin", [&](const auto & encoded) { + StringSource in { encoded }; + NullParseSink out; + Tree got; + parse(out, "", in, [&](auto & name, auto entry) { + auto name2 = name; + if (entry.mode == Mode::Directory) + name2 += '/'; + got.insert_or_assign(name2, std::move(entry)); + }, mockXpSettings); + + ASSERT_EQ(got, tree); + }); +} + +TEST_F(GitTest, tree_write) { + writeTest("tree.bin", [&]() { + StringSink s; + dumpTree(tree, s, mockXpSettings); + return s.s; + }); +} + +TEST_F(GitTest, both_roundrip) { + using File = MemorySourceAccessor::File; + + MemorySourceAccessor files; + files.root = File::Directory { + .contents { + { + "foo", + File::Regular { + .contents = "hello\n\0\n\tworld!", + }, + }, + { + "bar", + File::Directory { + .contents = { + { + "baz", + File::Regular { + .executable = true, + .contents = "good day,\n\0\n\tworld!", + }, + }, + }, + }, + }, + }, + }; + + std::map cas; + + std::function dumpHook; + dumpHook = [&](const CanonPath & path) { + StringSink s; + HashSink hashSink { htSHA1 }; + TeeSink s2 { s, hashSink }; + auto mode = dump( + files, path, s2, dumpHook, + defaultPathFilter, mockXpSettings); + auto hash = hashSink.finish().first; + cas.insert_or_assign(hash, std::move(s.s)); + return TreeEntry { + .mode = mode, + .hash = hash, + }; + }; + + auto root = dumpHook(CanonPath::root); + + MemorySourceAccessor files2; + + MemorySink sinkFiles2 { files2 }; + + std::function mkSinkHook; + mkSinkHook = [&](const Path prefix, const Hash & hash) { + StringSource in { cas[hash] }; + parse(sinkFiles2, prefix, in, [&](const Path & name, const auto & entry) { + mkSinkHook(prefix + "/" + name, entry.hash); + }, mockXpSettings); + }; + + mkSinkHook("", root.hash); + + ASSERT_EQ(files, files2); +} + +TEST(GitLsRemote, parseSymrefLineWithReference) { + auto line = "ref: refs/head/main HEAD"; + auto res = parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Symbolic); + ASSERT_EQ(res->target, "refs/head/main"); + ASSERT_EQ(res->reference, "HEAD"); +} + +TEST(GitLsRemote, parseSymrefLineWithNoReference) { + auto line = "ref: refs/head/main"; + auto res = parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Symbolic); + ASSERT_EQ(res->target, "refs/head/main"); + ASSERT_EQ(res->reference, std::nullopt); +} + +TEST(GitLsRemote, parseObjectRefLine) { + auto line = "abc123 refs/head/main"; + auto res = parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Object); + ASSERT_EQ(res->target, "abc123"); + ASSERT_EQ(res->reference, "refs/head/main"); +} + +} diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index 167915439..5a970c0f2 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -27,3 +27,7 @@ libutil-tests_CXXFLAGS += -I src/libutil libutil-tests_LIBS = libutil libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) + +check: unit-test-data/libutil/git/check-data.sh.test + +$(eval $(call run-test,unit-test-data/libutil/git/check-data.sh)) diff --git a/unit-test-data/libutil/git/check-data.sh b/unit-test-data/libutil/git/check-data.sh new file mode 100644 index 000000000..68b705c95 --- /dev/null +++ b/unit-test-data/libutil/git/check-data.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/git-hashing/unit-test-data +mkdir -p $TEST_ROOT + +repo="$TEST_ROOT/scratch" +git init "$repo" + +git -C "$repo" config user.email "you@example.com" +git -C "$repo" config user.name "Your Name" + +# `-w` to write for tree test +freshlyAddedHash=$(git -C "$repo" hash-object -w -t blob --stdin < "./hello-world.bin") +encodingHash=$(sha1sum -b < "./hello-world-blob.bin" | head -c 40) + +# If the hashes match, then `hello-world-blob.bin` must be the encoding +# of `hello-world.bin`. +[[ "$encodingHash" == "$freshlyAddedHash" ]] + +# Create empty directory object for tree test +echo -n | git -C "$repo" hash-object -w -t tree --stdin + +# Relies on both child hashes already existing in the git store +freshlyAddedHash=$(git -C "$repo" mktree < "./tree.txt") +encodingHash=$(sha1sum -b < "./tree.bin" | head -c 40) + +# If the hashes match, then `tree.bin` must be the encoding of the +# directory denoted by `tree.txt` interpreted as git directory listing. +[[ "$encodingHash" == "$freshlyAddedHash" ]] diff --git a/unit-test-data/libutil/git/hello-world-blob.bin b/unit-test-data/libutil/git/hello-world-blob.bin new file mode 100644 index 0000000000000000000000000000000000000000..255f5df55ccedb2dae5f541d516896ffffcdb526 GIT binary patch literal 24 fcmYew$xl)+G-L2c&B@7E;9}t7R0z*6%1HqLQkDjQ literal 0 HcmV?d00001 diff --git a/unit-test-data/libutil/git/hello-world.bin b/unit-test-data/libutil/git/hello-world.bin new file mode 100644 index 0000000000000000000000000000000000000000..63ddb340119baf8492d2da53af47e8c7cfcd5eb2 GIT binary patch literal 16 XcmeZB&B@7E;9}t7R0z*6%1HqLBqsz~ literal 0 HcmV?d00001 diff --git a/unit-test-data/libutil/git/tree.bin b/unit-test-data/libutil/git/tree.bin new file mode 100644 index 0000000000000000000000000000000000000000..5256ec140702fef5f88bd5750caf7cd57c03e5ac GIT binary patch literal 100 zcmXRZN=;R;G-5C`FfcPQQE(Ikg(Sx! ktkNb1K%kJ67{%b-6no6+bl%Pd2~WL$T$|MK`<*8X01f;sp#T5? literal 0 HcmV?d00001 diff --git a/unit-test-data/libutil/git/tree.txt b/unit-test-data/libutil/git/tree.txt new file mode 100644 index 000000000..be3d02920 --- /dev/null +++ b/unit-test-data/libutil/git/tree.txt @@ -0,0 +1,3 @@ +100644 blob 63ddb340119baf8492d2da53af47e8c7cfcd5eb2 Foo +100755 blob 63ddb340119baf8492d2da53af47e8c7cfcd5eb2 bAr +040000 tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904 baZ From fd5a4a846752873331b6549f0778181dc4ecc2f3 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 10 Nov 2023 12:12:13 -0500 Subject: [PATCH 582/909] nix upgrade-nix: make the source URL an option This new option enables organizations to more easily manage their Nix fleet's deployment, and ensure a consistent and planned rollout of Nix upgrades. --- src/libstore/globals.hh | 10 ++++++++++ src/nix/upgrade-nix.cc | 7 +++---- src/nix/upgrade-nix.md | 6 ++++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 8e034f5a9..27caf42c4 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1084,6 +1084,16 @@ public: true, // document default Xp::ConfigurableImpureEnv }; + + Setting upgradeNixStorePathUrl{ + this, + "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix", + "upgrade-nix-store-path-url", + R"( + Used by `nix upgrade-nix`, the URL of the file that contains the + store paths of the latest Nix release. + )" + }; }; diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index c529c2363..4c7a74e16 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -14,7 +14,6 @@ using namespace nix; struct CmdUpgradeNix : MixDryRun, StoreCommand { Path profileDir; - std::string storePathsUrl = "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix"; CmdUpgradeNix() { @@ -30,7 +29,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand .longName = "nix-store-paths-url", .description = "The URL of the file that contains the store paths of the latest Nix release.", .labels = {"url"}, - .handler = {&storePathsUrl} + .handler = {&(std::string&) settings.upgradeNixStorePathUrl} }); } @@ -44,7 +43,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand std::string description() override { - return "upgrade Nix to the stable version declared in Nixpkgs"; + return "upgrade Nix to the latest stable version"; } std::string doc() override @@ -145,7 +144,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); // FIXME: use nixos.org? - auto req = FileTransferRequest(storePathsUrl); + auto req = FileTransferRequest((std::string&) settings.upgradeNixStorePathUrl); auto res = getFileTransfer()->download(req); auto state = std::make_unique(SearchPath{}, store); diff --git a/src/nix/upgrade-nix.md b/src/nix/upgrade-nix.md index cce88c397..3a3bf61b9 100644 --- a/src/nix/upgrade-nix.md +++ b/src/nix/upgrade-nix.md @@ -16,8 +16,10 @@ R""( # Description -This command upgrades Nix to the stable version declared in Nixpkgs. -This stable version is defined in [nix-fallback-paths.nix](https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix) +This command upgrades Nix to the stable version. + +By default, the latest stable version is defined by Nixpkgs, in +[nix-fallback-paths.nix](https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix) and updated manually. It may not always be the latest tagged release. By default, it locates the directory containing the `nix` binary in the `$PATH` From 0be84c83b242b6e6a22400727752072b298e7cab Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Mon, 6 Nov 2023 10:29:37 -0500 Subject: [PATCH 583/909] key and cat: no need for progressBar otherwise the output will be invisible in common terminal configurations --- src/nix/cat.cc | 2 ++ src/nix/sigs.cc | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 6e5a736f2..4df086d4f 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -1,6 +1,7 @@ #include "command.hh" #include "store-api.hh" #include "nar-accessor.hh" +#include "progress-bar.hh" using namespace nix; @@ -13,6 +14,7 @@ struct MixCat : virtual Args auto st = accessor->lstat(CanonPath(path)); if (st.type != SourceAccessor::Type::tRegular) throw Error("path '%1%' is not a regular file", path); + stopProgressBar(); writeFull(STDOUT_FILENO, accessor->readFile(CanonPath(path))); } }; diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index a68616355..39555c9ea 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -3,6 +3,7 @@ #include "shared.hh" #include "store-api.hh" #include "thread-pool.hh" +#include "progress-bar.hh" #include @@ -174,6 +175,7 @@ struct CmdKeyGenerateSecret : Command if (!keyName) throw UsageError("required argument '--key-name' is missing"); + stopProgressBar(); writeFull(STDOUT_FILENO, SecretKey::generate(*keyName).to_string()); } }; @@ -195,6 +197,7 @@ struct CmdKeyConvertSecretToPublic : Command void run() override { SecretKey secretKey(drainFD(STDIN_FILENO)); + stopProgressBar(); writeFull(STDOUT_FILENO, secretKey.toPublicKey().to_string()); } }; From e4cbdd26e0e6a2a5907dff8e60c3645f7d94423a Mon Sep 17 00:00:00 2001 From: Bouke van der Bijl Date: Mon, 13 Nov 2023 17:13:52 +0100 Subject: [PATCH 584/909] Add TODO comment for include try/catch --- src/libutil/config.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 17380b6d8..ab873b4a8 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -125,7 +125,9 @@ static void applyConfigInner(const std::string & contents, const std::string & p try { std::string includedContents = readFile(path); applyConfigInner(includedContents, p, parsedContents); - } catch (SysError &) { } + } catch (SysError &) { + // TODO: Do we actually want to ignore this? Or is it better to fail? + } } else if (!ignoreMissing) { throw Error("file '%1%' included from '%2%' not found", p, path); } From d6898cd58b1a685404ba6878c317e60be9473a9a Mon Sep 17 00:00:00 2001 From: Bouke van der Bijl Date: Mon, 13 Nov 2023 17:14:05 +0100 Subject: [PATCH 585/909] Move applyConfigFile to lambda inside libstore --- src/libstore/globals.cc | 11 +++++++++-- src/libutil/config.cc | 8 -------- src/libutil/config.hh | 6 ------ 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 9c25d9868..0aecd2b6a 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -111,7 +111,14 @@ Settings::Settings() void loadConfFile() { - globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf"); + auto applyConfigFile = [&](const Path & path) { + try { + std::string contents = readFile(path); + globalConfig.applyConfig(contents, path); + } catch (SysError &) { } + }; + + applyConfigFile(settings.nixConfDir + "/nix.conf"); /* We only want to send overrides to the daemon, i.e. stuff from ~/.nix/nix.conf or the command line. */ @@ -119,7 +126,7 @@ void loadConfFile() auto files = settings.nixUserConfFiles; for (auto file = files.rbegin(); file != files.rend(); file++) { - globalConfig.applyConfigFile(*file); + applyConfigFile(*file); } auto nixConfEnv = getEnv("NIX_CONFIG"); diff --git a/src/libutil/config.cc b/src/libutil/config.cc index ab873b4a8..ad16c86bd 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -165,14 +165,6 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string set(name, value); } -void AbstractConfig::applyConfigFile(const Path & path) -{ - try { - std::string contents = readFile(path); - applyConfig(contents, path); - } catch (SysError &) { } -} - void Config::resetOverridden() { for (auto & s : _settings) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 38c3ce0c4..d49eb602d 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -82,12 +82,6 @@ public: */ void applyConfig(const std::string & contents, const std::string & path = ""); - /** - * Applies a nix configuration file - * - path: the location of the config file to apply - */ - void applyConfigFile(const Path & path); - /** * Resets the `overridden` flag of all Settings */ From 742a63b98f2008161fd00bdbbd39b8f1b14f6443 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 22:01:50 +0000 Subject: [PATCH 586/909] build(deps): bump zeebe-io/backport-action from 2.1.0 to 2.1.1 Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 2.1.0 to 2.1.1. - [Release notes](https://github.com/zeebe-io/backport-action/releases) - [Commits](https://github.com/zeebe-io/backport-action/compare/v2.1.0...v2.1.1) --- updated-dependencies: - dependency-name: zeebe-io/backport-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 893f4a56f..975c90b91 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v2.1.0 + uses: zeebe-io/backport-action@v2.1.1 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} From ad99c8950b86b8f354f5c72efe690d3cba045d99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20St=C3=BChrk?= Date: Mon, 13 Nov 2023 23:19:27 +0100 Subject: [PATCH 587/909] Update comment to reflect bind mounts are now used for store in chroot --- src/libstore/build/local-derivation-goal.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index adb011e30..a9f930773 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -652,8 +652,8 @@ void LocalDerivationGoal::startBuilder() #if __linux__ /* Create a temporary directory in which we set up the chroot environment using bind-mounts. We put it in the Nix store - to ensure that we can create hard-links to non-directory - inputs in the fake Nix store in the chroot (see below). */ + so that the build outputs can be moved efficiently from the + chroot to their final location. */ chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; deletePath(chrootRootDir); From 21bb180547118e29a66bf091bd6b1dd911b3114d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Nov 2023 13:30:51 +0100 Subject: [PATCH 588/909] Use libgit2 with ssh-exec support See https://github.com/libgit2/libgit2/pull/6617. This ensures that we get support for ~/.ssh/config, known_hosts etc. --- flake.lock | 17 +++++++++++++++++ flake.nix | 9 +++++++-- src/libfetchers/git-utils.cc | 17 ----------------- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/flake.lock b/flake.lock index 991cef1ee..2b1d96e4e 100644 --- a/flake.lock +++ b/flake.lock @@ -16,6 +16,22 @@ "type": "github" } }, + "libgit2": { + "flake": false, + "locked": { + "lastModified": 1697646580, + "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", + "owner": "libgit2", + "repo": "libgit2", + "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", + "type": "github" + }, + "original": { + "owner": "libgit2", + "repo": "libgit2", + "type": "github" + } + }, "lowdown-src": { "flake": false, "locked": { @@ -67,6 +83,7 @@ "root": { "inputs": { "flake-compat": "flake-compat", + "libgit2": "libgit2", "lowdown-src": "lowdown-src", "nixpkgs": "nixpkgs", "nixpkgs-regression": "nixpkgs-regression" diff --git a/flake.nix b/flake.nix index e71aa5374..d6a173081 100644 --- a/flake.nix +++ b/flake.nix @@ -7,8 +7,9 @@ inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; + inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; - outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, flake-compat }: + outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, flake-compat, libgit2 }: let inherit (nixpkgs) lib; @@ -194,7 +195,11 @@ bzip2 xz brotli editline openssl sqlite libarchive - libgit2 + (pkgs.libgit2.overrideAttrs (attrs: { + src = libgit2; + version = libgit2.lastModifiedDate; + cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"]; + })) boost lowdown-nix libsodium diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 3a0e2d02f..1ec50099b 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -336,9 +336,6 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this const std::string & url, const std::string & refspec) override { - /* FIXME: use libgit2. Unfortunately, it doesn't support - ssh_config at the moment. */ - #if 0 Remote remote; if (git_remote_create_anonymous(Setter(remote), *this, url.c_str())) @@ -352,20 +349,6 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this if (git_remote_fetch(remote.get(), &refspecs2, nullptr, nullptr)) throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message); - #endif - - // FIXME: git stderr messes up our progress indicator, so - // we're using --quiet for now. Should process its stderr. - runProgram("git", true, - { "-C", path.abs(), - "--bare", - "fetch", - "--quiet", - "--force", - "--", - url, - refspec - }, {}, true); } void verifyCommit( From d74d2fdaa721cd7cddceca2e0b4063a1d891bb9f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Nov 2023 13:35:26 +0100 Subject: [PATCH 589/909] Move statusCallbackTrampoline --- src/libfetchers/git-utils.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 1ec50099b..ffcc92fc7 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -130,11 +130,6 @@ T peelObject(git_repository * repo, git_object * obj, git_object_t type) return obj2; } -int statusCallbackTrampoline(const char * path, unsigned int statusFlags, void * payload) -{ - return (*((std::function *) payload))(path, statusFlags); -} - struct GitRepoImpl : GitRepo, std::enable_shared_from_this { CanonPath path; @@ -255,6 +250,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return result; } + // Helper for statusCallback below. + static int statusCallbackTrampoline(const char * path, unsigned int statusFlags, void * payload) + { + return (*((std::function *) payload))(path, statusFlags); + } + WorkdirInfo getWorkdirInfo() override { WorkdirInfo info; From 38b07d63479ebdd4f43145264a026a22a72d940b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Nov 2023 13:38:03 +0100 Subject: [PATCH 590/909] src/libfetchers/git.cc: Apply suggestion Co-authored-by: Robert Hensing --- src/libfetchers/git.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 9c2a7df16..12233ed0a 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -332,7 +332,7 @@ struct GitInputScheme : InputScheme whether the working directory is dirty compared to HEAD. */ GitRepo::WorkdirInfo workdirInfo; - /* URL of the repo, or its path if isLocal. */ + /* URL of the repo, or its path if isLocal. Never a `file` URL. */ std::string url; void warnDirty() const From 25cf8f107125eda79e7faece90e7e05093a39e65 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Nov 2023 13:57:24 +0100 Subject: [PATCH 591/909] src/libfetchers/union-input-accessor.cc: Apply suggestion Co-authored-by: Robert Hensing --- src/libfetchers/union-input-accessor.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libfetchers/union-input-accessor.cc b/src/libfetchers/union-input-accessor.cc index ae942cb41..f9472efa7 100644 --- a/src/libfetchers/union-input-accessor.cc +++ b/src/libfetchers/union-input-accessor.cc @@ -12,8 +12,7 @@ struct UnionInputAccessor : InputAccessor // Currently we require a root filesystem. This could be relaxed. assert(mounts.contains(CanonPath::root)); - // FIXME: should check that every mount point exists. Or we - // could return dummy parent directories automatically. + // FIXME: return dummy parent directories automatically? } std::string readFile(const CanonPath & path) override From 4329bdf6a30fadad66384f0b8c835d7dba9f87b3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Nov 2023 13:58:27 +0100 Subject: [PATCH 592/909] Move comment --- src/libfetchers/cache.hh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh index b517d496e..c8d3248bc 100644 --- a/src/libfetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -6,13 +6,14 @@ namespace nix::fetchers { +/* + * A cache for arbitrary `Attrs` -> `Attrs` mappings with a timestamp + * for expiration. + */ struct Cache { virtual ~Cache() { } - /* A cache for arbitrary Attrs -> Attrs mappings with a timestamp - for expiration. */ - /* * Add a value to the cache. The cache is an arbitrary mapping of * Attrs to Attrs. From 21140c987b7a301c01498864efbc3d92be04aced Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Nov 2023 13:59:00 +0100 Subject: [PATCH 593/909] Fix doxygen comments --- src/libfetchers/cache.hh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh index c8d3248bc..f70589267 100644 --- a/src/libfetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -6,7 +6,7 @@ namespace nix::fetchers { -/* +/** * A cache for arbitrary `Attrs` -> `Attrs` mappings with a timestamp * for expiration. */ @@ -14,7 +14,7 @@ struct Cache { virtual ~Cache() { } - /* + /** * Add a value to the cache. The cache is an arbitrary mapping of * Attrs to Attrs. */ @@ -22,13 +22,13 @@ struct Cache const Attrs & inAttrs, const Attrs & infoAttrs) = 0; - /* + /** * Look up a key with infinite TTL. */ virtual std::optional lookup( const Attrs & inAttrs) = 0; - /* + /** * Look up a key. Return nothing if its TTL has exceeded * `settings.tarballTTL`. */ @@ -41,7 +41,7 @@ struct Cache Attrs infoAttrs; }; - /* + /** * Look up a key. Return a bool denoting whether its TTL has * exceeded `settings.tarballTTL`. */ From 7f576f5dfe11c3f6b0e69179de95c921caddda18 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Nov 2023 14:01:38 +0100 Subject: [PATCH 594/909] Rename UnionInputAccessor to MountedInputAccessor --- src/libfetchers/git.cc | 10 +++++----- ...ion-input-accessor.cc => mounted-input-accessor.cc} | 10 +++++----- src/libfetchers/mounted-input-accessor.hh | 9 +++++++++ src/libfetchers/union-input-accessor.hh | 9 --------- 4 files changed, 19 insertions(+), 19 deletions(-) rename src/libfetchers/{union-input-accessor.cc => mounted-input-accessor.cc} (86%) create mode 100644 src/libfetchers/mounted-input-accessor.hh delete mode 100644 src/libfetchers/union-input-accessor.hh diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 12233ed0a..90c6ad531 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -9,7 +9,7 @@ #include "processes.hh" #include "git.hh" #include "fs-input-accessor.hh" -#include "union-input-accessor.hh" +#include "mounted-input-accessor.hh" #include "git-utils.hh" #include "fetch-settings.hh" @@ -587,7 +587,7 @@ struct GitInputScheme : InputScheme auto accessor = repo->getAccessor(rev); - /* If the repo has submodules, fetch them and return a union + /* If the repo has submodules, fetch them and return a mounted input accessor consisting of the accessor for the top-level repo and the accessors for the submodules. */ if (repoInfo.submodules) { @@ -611,7 +611,7 @@ struct GitInputScheme : InputScheme if (!mounts.empty()) { mounts.insert_or_assign(CanonPath::root, accessor); - accessor = makeUnionInputAccessor(std::move(mounts)); + accessor = makeMountedInputAccessor(std::move(mounts)); } } @@ -636,7 +636,7 @@ struct GitInputScheme : InputScheme ref accessor = makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url)); - /* If the repo has submodules, return a union input accessor + /* If the repo has submodules, return a mounted input accessor consisting of the accessor for the top-level repo and the accessors for the submodule workdirs. */ if (repoInfo.submodules && !repoInfo.workdirInfo.submodules.empty()) { @@ -660,7 +660,7 @@ struct GitInputScheme : InputScheme } mounts.insert_or_assign(CanonPath::root, accessor); - accessor = makeUnionInputAccessor(std::move(mounts)); + accessor = makeMountedInputAccessor(std::move(mounts)); } if (!repoInfo.workdirInfo.isDirty) { diff --git a/src/libfetchers/union-input-accessor.cc b/src/libfetchers/mounted-input-accessor.cc similarity index 86% rename from src/libfetchers/union-input-accessor.cc rename to src/libfetchers/mounted-input-accessor.cc index f9472efa7..49917f6e5 100644 --- a/src/libfetchers/union-input-accessor.cc +++ b/src/libfetchers/mounted-input-accessor.cc @@ -1,12 +1,12 @@ -#include "union-input-accessor.hh" +#include "mounted-input-accessor.hh" namespace nix { -struct UnionInputAccessor : InputAccessor +struct MountedInputAccessor : InputAccessor { std::map> mounts; - UnionInputAccessor(std::map> _mounts) + MountedInputAccessor(std::map> _mounts) : mounts(std::move(_mounts)) { // Currently we require a root filesystem. This could be relaxed. @@ -71,9 +71,9 @@ struct UnionInputAccessor : InputAccessor } }; -ref makeUnionInputAccessor(std::map> mounts) +ref makeMountedInputAccessor(std::map> mounts) { - return make_ref(std::move(mounts)); + return make_ref(std::move(mounts)); } } diff --git a/src/libfetchers/mounted-input-accessor.hh b/src/libfetchers/mounted-input-accessor.hh new file mode 100644 index 000000000..b557c5dad --- /dev/null +++ b/src/libfetchers/mounted-input-accessor.hh @@ -0,0 +1,9 @@ +#pragma once + +#include "input-accessor.hh" + +namespace nix { + +ref makeMountedInputAccessor(std::map> mounts); + +} diff --git a/src/libfetchers/union-input-accessor.hh b/src/libfetchers/union-input-accessor.hh deleted file mode 100644 index 6a1649c1d..000000000 --- a/src/libfetchers/union-input-accessor.hh +++ /dev/null @@ -1,9 +0,0 @@ -#pragma once - -#include "input-accessor.hh" - -namespace nix { - -ref makeUnionInputAccessor(std::map> mounts); - -} From c257c824475c92cdfda5daa027db334b6a0137f8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Nov 2023 14:47:17 +0100 Subject: [PATCH 595/909] Cleanup --- src/libfetchers/mounted-input-accessor.cc | 10 ++++------ src/libutil/canon-path.cc | 7 +++++++ src/libutil/canon-path.hh | 6 ++++++ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/libfetchers/mounted-input-accessor.cc b/src/libfetchers/mounted-input-accessor.cc index 49917f6e5..6f397eb17 100644 --- a/src/libfetchers/mounted-input-accessor.cc +++ b/src/libfetchers/mounted-input-accessor.cc @@ -54,18 +54,16 @@ struct MountedInputAccessor : InputAccessor std::pair, CanonPath> resolve(CanonPath path) { // Find the nearest parent of `path` that is a mount point. - std::vector ss; + std::vector subpath; while (true) { auto i = mounts.find(path); if (i != mounts.end()) { - auto subpath = CanonPath::root; - for (auto j = ss.rbegin(); j != ss.rend(); ++j) - subpath.push(*j); - return {i->second, std::move(subpath)}; + std::reverse(subpath.begin(), subpath.end()); + return {i->second, CanonPath(subpath)}; } assert(!path.isRoot()); - ss.push_back(std::string(*path.baseName())); + subpath.push_back(std::string(*path.baseName())); path.pop(); } } diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc index f678fae94..1e465f1f6 100644 --- a/src/libutil/canon-path.cc +++ b/src/libutil/canon-path.cc @@ -13,6 +13,13 @@ CanonPath::CanonPath(std::string_view raw, const CanonPath & root) : path(absPath((Path) raw, root.abs())) { } +CanonPath::CanonPath(const std::vector & elems) + : path("/") +{ + for (auto & s : elems) + push(s); +} + CanonPath CanonPath::fromCwd(std::string_view path) { return CanonPath(unchecked_t(), absPath((Path) path)); diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh index eefe05ed5..6d0519f4f 100644 --- a/src/libutil/canon-path.hh +++ b/src/libutil/canon-path.hh @@ -6,6 +6,7 @@ #include #include #include +#include namespace nix { @@ -46,6 +47,11 @@ public: : path(std::move(path)) { } + /** + * Construct a canon path from a vector of elements. + */ + CanonPath(const std::vector & elems); + static CanonPath fromCwd(std::string_view path = "."); static CanonPath root; From 4944cdb94d03742176cc7881f126e981c0e7e21c Mon Sep 17 00:00:00 2001 From: vicky1999 Date: Tue, 14 Nov 2023 19:59:48 +0530 Subject: [PATCH 596/909] nar dump-path command renamed to nar pack --- src/nix/dump-path.cc | 10 +++++++++- src/nix/nar-dump-path.md | 3 ++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index c4edc894b..0850d4c1c 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -61,4 +61,12 @@ struct CmdDumpPath2 : Command } }; -static auto rDumpPath2 = registerCommand2({"nar", "dump-path"}); +struct CmdNarDumpPath : CmdDumpPath2 { + void run() override { + warn("'nix nar dump-path' is a deprecated alias for 'nix nar pack'"); + CmdDumpPath2::run(); + } +}; + +static auto rCmdNarPack = registerCommand2({"nar", "pack"}); +static auto rCmdNarDumpPath = registerCommand2({"nar", "dump-path"}); diff --git a/src/nix/nar-dump-path.md b/src/nix/nar-dump-path.md index 26191ad25..29eaacfdb 100644 --- a/src/nix/nar-dump-path.md +++ b/src/nix/nar-dump-path.md @@ -5,7 +5,7 @@ R""( * To serialise directory `foo` as a NAR: ```console - # nix nar dump-path ./foo > foo.nar + # nix nar pack ./foo > foo.nar ``` # Description @@ -15,3 +15,4 @@ This command generates a NAR file containing the serialisation of symbolic links. The NAR is written to standard output. )"" + From e07e3c106a9ac0537210e62286c4e696573e9f6f Mon Sep 17 00:00:00 2001 From: vicky1999 Date: Tue, 14 Nov 2023 20:02:33 +0530 Subject: [PATCH 597/909] code cleanup --- src/nix/nar-dump-path.md | 1 - 1 file changed, 1 deletion(-) diff --git a/src/nix/nar-dump-path.md b/src/nix/nar-dump-path.md index 29eaacfdb..de82202de 100644 --- a/src/nix/nar-dump-path.md +++ b/src/nix/nar-dump-path.md @@ -15,4 +15,3 @@ This command generates a NAR file containing the serialisation of symbolic links. The NAR is written to standard output. )"" - From 6ec6b8aa363f566a8da0d6959753efa452b152cc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Nov 2023 15:52:18 +0100 Subject: [PATCH 598/909] Improve git submodule error reporting --- src/libfetchers/fetchers.cc | 10 ++++++++++ src/libfetchers/fetchers.hh | 2 ++ src/libfetchers/git.cc | 4 ++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 19e089aa8..c2513e076 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -220,6 +220,16 @@ std::pair Input::fetch(ref store) const return {std::move(storePath), input}; } +std::pair, Input> Input::getAccessor(ref store) const +{ + try { + return scheme->getAccessor(store, *this); + } catch (Error & e) { + e.addTrace({}, "while fetching the input '%s'", to_string()); + throw; + } +} + Input Input::applyOverrides( std::optional ref, std::optional rev) const diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 6db1615f2..ce5aa4c69 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -83,6 +83,8 @@ public: */ std::pair fetch(ref store) const; + std::pair, Input> getAccessor(ref store) const; + Input applyOverrides( std::optional ref, std::optional rev) const; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 90c6ad531..71ae74dde 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -605,7 +605,7 @@ struct GitInputScheme : InputScheme attrs.insert_or_assign("rev", submoduleRev.gitRev()); auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = - submoduleInput.scheme->getAccessor(store, submoduleInput); + submoduleInput.getAccessor(store); mounts.insert_or_assign(submodule.path, submoduleAccessor); } @@ -649,7 +649,7 @@ struct GitInputScheme : InputScheme attrs.insert_or_assign("url", submodulePath.abs()); auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = - submoduleInput.scheme->getAccessor(store, submoduleInput); + submoduleInput.getAccessor(store); /* If the submodule is dirty, mark this repo dirty as well. */ From 2964a9f562748cc698ee1f6ecf1e0da4e63211b9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Nov 2023 16:00:21 +0100 Subject: [PATCH 599/909] Fix relative submodule handling Tested on nix flake prefetch 'git+https://github.com/blender/blender.git?rev=4ed8a360e956daf2591add4d3c9ec0719e2628fe&submodules=1' --- src/libfetchers/git-utils.cc | 12 ++++++++++-- src/libfetchers/git-utils.hh | 4 +++- src/libfetchers/git.cc | 2 +- src/libutil/url.cc | 8 ++++++++ src/libutil/url.hh | 5 +++++ 5 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index ffcc92fc7..1edafbf33 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -308,13 +308,21 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this std::vector> getSubmodules(const Hash & rev) override; - std::string resolveSubmoduleUrl(const std::string & url) override + std::string resolveSubmoduleUrl( + const std::string & url, + const std::string & base) override { git_buf buf = GIT_BUF_INIT; if (git_submodule_resolve_url(&buf, *this, url.c_str())) throw Error("resolving Git submodule URL '%s'", url); Finally cleanup = [&]() { git_buf_dispose(&buf); }; - return buf.ptr; + + std::string res(buf.ptr); + + if (!hasPrefix(res, "/") && res.find("://") == res.npos) + res = parseURL(base + "/" + res).canonicalise().to_string(); + + return res; } bool hasObject(const Hash & oid_) override diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 7efbdedce..e0cb2c34f 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -59,7 +59,9 @@ struct GitRepo */ virtual std::vector> getSubmodules(const Hash & rev) = 0; - virtual std::string resolveSubmoduleUrl(const std::string & url) = 0; + virtual std::string resolveSubmoduleUrl( + const std::string & url, + const std::string & base) = 0; struct TarballInfo { diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 71ae74dde..177c8b66e 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -594,7 +594,7 @@ struct GitInputScheme : InputScheme std::map> mounts; for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev)) { - auto resolved = repo->resolveSubmoduleUrl(submodule.url); + auto resolved = repo->resolveSubmoduleUrl(submodule.url, repoInfo.url); debug("Git submodule %s: %s %s %s -> %s", submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved); fetchers::Attrs attrs; diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 9b438e6cd..57b64d607 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -2,6 +2,7 @@ #include "url-parts.hh" #include "util.hh" #include "split.hh" +#include "canon-path.hh" namespace nix { @@ -141,6 +142,13 @@ bool ParsedURL::operator ==(const ParsedURL & other) const && fragment == other.fragment; } +ParsedURL ParsedURL::canonicalise() +{ + ParsedURL res(*this); + res.path = CanonPath(res.path).abs(); + return res; +} + /** * Parse a URL scheme of the form '(applicationScheme\+)?transportScheme' * into a tuple '(applicationScheme, transportScheme)' diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 26c2dcc28..833f54678 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -19,6 +19,11 @@ struct ParsedURL std::string to_string() const; bool operator ==(const ParsedURL & other) const; + + /** + * Remove `.` and `..` path elements. + */ + ParsedURL canonicalise(); }; MakeError(BadURL, Error); From 9c7749e13508996eb9df83b1692664cc8cdbf952 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 14 Nov 2023 11:42:25 -0500 Subject: [PATCH 600/909] Fix makefile bug confusing `libnixutil-test` exe vs lib The `-exe` variant is the program, the unsuffixed variant is the library. The corrected usage matches `libnixstore-test`. --- src/libutil/tests/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index 5a970c0f2..c8b8557cb 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -1,6 +1,6 @@ check: libutil-tests_RUN -programs += libutil-tests +programs += libutil-tests-exe libutil-tests-exe_NAME = libnixutil-tests From 70b396649c127760e4b123da41451aa7456bc68d Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 15 Nov 2023 00:03:44 +0100 Subject: [PATCH 601/909] doc: logical implication is right-associative nix-repl> bools = [ false true ] nix-repl> combinations = builtins.concatMap (a: builtins.concatMap (b: map (c: { inherit a b c; }) bools) bools) bools nix-repl> builtins.all ({ a, b, c }: (a -> b -> c) == (a -> (b -> c))) combinations true nix-repl> builtins.all ({ a, b, c }: (a -> b -> c) == ((a -> b) -> c)) combinations false --- doc/manual/src/language/operators.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index cc825b4cf..e9cbb5c92 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -25,7 +25,7 @@ | Inequality | *expr* `!=` *expr* | none | 11 | | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | | Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | -| [Logical implication] | *bool* `->` *bool* | none | 14 | +| [Logical implication] | *bool* `->` *bool* | right | 14 | [string]: ./values.md#type-string [path]: ./values.md#type-path From 28909999116781e194e2eb1646f3ccec005e774f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 15 Nov 2023 13:57:20 +0100 Subject: [PATCH 602/909] Show Git fetch progress --- src/libfetchers/git-utils.cc | 30 +++++++++++++++++++++++++++++- src/libfetchers/git.cc | 2 -- src/libmain/progress-bar.cc | 8 ++++++++ src/libutil/logging.hh | 2 ++ 4 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 1edafbf33..b7ef05c10 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -3,6 +3,7 @@ #include "cache.hh" #include "finally.hh" #include "processes.hh" +#include "signals.hh" #include @@ -341,10 +342,32 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this ref getAccessor(const Hash & rev) override; + static int sidebandProgressCallback(const char * str, int len, void * payload) + { + auto act = (Activity *) payload; + act->result(resFetchStatus, trim(std::string_view(str, len))); + return _isInterrupted ? -1 : 0; + } + + static int transferProgressCallback(const git_indexer_progress * stats, void * payload) + { + auto act = (Activity *) payload; + act->result(resFetchStatus, + fmt("%d/%d objects received, %d/%d deltas indexed, %.1f MiB", + stats->received_objects, + stats->total_objects, + stats->indexed_deltas, + stats->total_deltas, + stats->received_bytes / (1024.0 * 1024.0))); + return _isInterrupted ? -1 : 0; + } + void fetch( const std::string & url, const std::string & refspec) override { + Activity act(*logger, lvlTalkative, actFetchTree, fmt("fetching Git repository '%s'", url)); + Remote remote; if (git_remote_create_anonymous(Setter(remote), *this, url.c_str())) @@ -356,7 +379,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this .count = 1 }; - if (git_remote_fetch(remote.get(), &refspecs2, nullptr, nullptr)) + git_fetch_options opts = GIT_FETCH_OPTIONS_INIT; + opts.callbacks.payload = &act; + opts.callbacks.sideband_progress = sidebandProgressCallback; + opts.callbacks.transfer_progress = transferProgressCallback; + + if (git_remote_fetch(remote.get(), &refspecs2, &opts, nullptr)) throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message); } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 177c8b66e..3e7dcd8de 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -520,8 +520,6 @@ struct GitInputScheme : InputScheme } if (doFetch) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", repoInfo.url)); - try { auto fetchRef = repoInfo.allRefs ? "refs/*" diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index a7aee47c3..3aa012ee1 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -340,6 +340,14 @@ public: state->activitiesByType[type].expected += j; update(*state); } + + else if (type == resFetchStatus) { + auto i = state->its.find(act); + assert(i != state->its.end()); + ActInfo & actInfo = *i->second; + actInfo.lastLine = getS(fields, 0); + update(*state); + } } void update(State & state) diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 5aa6bee95..183f2d8e1 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -23,6 +23,7 @@ typedef enum { actQueryPathInfo = 109, actPostBuildHook = 110, actBuildWaiting = 111, + actFetchTree = 112, } ActivityType; typedef enum { @@ -34,6 +35,7 @@ typedef enum { resProgress = 105, resSetExpected = 106, resPostBuildLogLine = 107, + resFetchStatus = 108, } ResultType; typedef uint64_t ActivityId; From 5dd4ae86877cedaf70ea70d80b89c66b850bdc5a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 15 Nov 2023 14:08:34 +0100 Subject: [PATCH 603/909] Remove unused cacheType field --- src/libfetchers/git.cc | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 3e7dcd8de..b066b384c 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -323,8 +323,6 @@ struct GitInputScheme : InputScheme bool submodules = false; bool allRefs = false; - std::string cacheType; - /* Whether this is a local, non-bare repository. */ bool isLocal = false; @@ -371,11 +369,6 @@ struct GitInputScheme : InputScheme .allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false) }; - repoInfo.cacheType = "git"; - if (repoInfo.shallow) repoInfo.cacheType += "-shallow"; - if (repoInfo.submodules) repoInfo.cacheType += "-submodules"; - if (repoInfo.allRefs) repoInfo.cacheType += "-all-refs"; - // file:// URIs are normally not cloned (but otherwise treated the // same as remote URIs, i.e. we don't use the working tree or // HEAD). Exception: If _NIX_FORCE_HTTP is set, or the repo is a bare git From 7ab91e72387b96d1926f1b9c95b919020d4ba962 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 15 Nov 2023 14:43:30 +0100 Subject: [PATCH 604/909] Implement shallow fetching --- src/libfetchers/git-utils.cc | 4 ++- src/libfetchers/git-utils.hh | 3 ++- src/libfetchers/git.cc | 48 +++++++++++++++++++----------------- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index b7ef05c10..f554dcc5f 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -364,7 +364,8 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this void fetch( const std::string & url, - const std::string & refspec) override + const std::string & refspec, + bool shallow) override { Activity act(*logger, lvlTalkative, actFetchTree, fmt("fetching Git repository '%s'", url)); @@ -380,6 +381,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this }; git_fetch_options opts = GIT_FETCH_OPTIONS_INIT; + opts.depth = shallow ? 1 : GIT_FETCH_DEPTH_FULL; opts.callbacks.payload = &act; opts.callbacks.sideband_progress = sidebandProgressCallback; opts.callbacks.transfer_progress = transferProgressCallback; diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index e0cb2c34f..1def82071 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -75,7 +75,8 @@ struct GitRepo virtual void fetch( const std::string & url, - const std::string & refspec) = 0; + const std::string & refspec, + bool shallow) = 0; /** * Verify that commit `rev` is signed by one of the keys in diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index b066b384c..7208a0b6d 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -219,9 +219,6 @@ struct GitInputScheme : InputScheme || name == "publicKeys") experimentalFeatureSettings.require(Xp::VerifiedFetches); - maybeGetBoolAttr(attrs, "shallow"); - maybeGetBoolAttr(attrs, "submodules"); - maybeGetBoolAttr(attrs, "allRefs"); maybeGetBoolAttr(attrs, "verifyCommit"); if (auto ref = maybeGetStrAttr(attrs, "ref")) { @@ -234,6 +231,9 @@ struct GitInputScheme : InputScheme auto url = fixGitURL(getStrAttr(attrs, "url")); parseURL(url); input.attrs["url"] = url; + getShallowAttr(input); + getSubmodulesAttr(input); + getAllRefsAttr(input); return input; } @@ -243,8 +243,10 @@ struct GitInputScheme : InputScheme if (url.scheme != "git") url.scheme = "git+" + url.scheme; if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev()); if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); - if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false)) + if (getShallowAttr(input)) url.query.insert_or_assign("shallow", "1"); + if (getSubmodulesAttr(input)) + url.query.insert_or_assign("submodules", "1"); if (maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(false)) url.query.insert_or_assign("verifyCommit", "1"); auto publicKeys = getPublicKeys(input.attrs); @@ -319,10 +321,6 @@ struct GitInputScheme : InputScheme struct RepoInfo { - bool shallow = false; - bool submodules = false; - bool allRefs = false; - /* Whether this is a local, non-bare repository. */ bool isLocal = false; @@ -347,11 +345,21 @@ struct GitInputScheme : InputScheme std::string gitDir = ".git"; }; + bool getShallowAttr(const Input & input) const + { + return maybeGetBoolAttr(input.attrs, "shallow").value_or(false); + } + bool getSubmodulesAttr(const Input & input) const { return maybeGetBoolAttr(input.attrs, "submodules").value_or(false); } + bool getAllRefsAttr(const Input & input) const + { + return maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); + } + RepoInfo getRepoInfo(const Input & input) const { auto checkHashType = [&](const std::optional & hash) @@ -363,11 +371,7 @@ struct GitInputScheme : InputScheme if (auto rev = input.getRev()) checkHashType(rev); - RepoInfo repoInfo { - .shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false), - .submodules = getSubmodulesAttr(input), - .allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false) - }; + RepoInfo repoInfo; // file:// URIs are normally not cloned (but otherwise treated the // same as remote URIs, i.e. we don't use the working tree or @@ -501,7 +505,7 @@ struct GitInputScheme : InputScheme if (auto rev = input.getRev()) { doFetch = !repo->hasObject(*rev); } else { - if (repoInfo.allRefs) { + if (getAllRefsAttr(input)) { doFetch = true; } else { /* If the local ref is older than ‘tarball-ttl’ seconds, do a @@ -514,7 +518,7 @@ struct GitInputScheme : InputScheme if (doFetch) { try { - auto fetchRef = repoInfo.allRefs + auto fetchRef = getAllRefsAttr(input) ? "refs/*" : ref.compare(0, 5, "refs/") == 0 ? ref @@ -522,7 +526,7 @@ struct GitInputScheme : InputScheme ? ref : "refs/heads/" + ref; - repo->fetch(repoInfo.url, fmt("%s:%s", fetchRef, fetchRef)); + repo->fetch(repoInfo.url, fmt("%s:%s", fetchRef, fetchRef), getShallowAttr(input)); } catch (Error & e) { if (!pathExists(localRefFile)) throw; logError(e.info()); @@ -556,7 +560,7 @@ struct GitInputScheme : InputScheme auto isShallow = repo->isShallow(); - if (isShallow && !repoInfo.shallow) + if (isShallow && !getShallowAttr(input)) throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", repoInfo.url); // FIXME: check whether rev is an ancestor of ref? @@ -568,7 +572,7 @@ struct GitInputScheme : InputScheme {"lastModified", getLastModified(repoInfo, repoDir, rev)}, }); - if (!repoInfo.shallow) + if (!getShallowAttr(input)) infoAttrs.insert_or_assign("revCount", getRevCount(repoInfo, repoDir, rev)); @@ -581,7 +585,7 @@ struct GitInputScheme : InputScheme /* If the repo has submodules, fetch them and return a mounted input accessor consisting of the accessor for the top-level repo and the accessors for the submodules. */ - if (repoInfo.submodules) { + if (getSubmodulesAttr(input)) { std::map> mounts; for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev)) { @@ -607,7 +611,7 @@ struct GitInputScheme : InputScheme } assert(!origRev || origRev == rev); - if (!repoInfo.shallow) + if (!getShallowAttr(input)) input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); @@ -619,7 +623,7 @@ struct GitInputScheme : InputScheme RepoInfo & repoInfo, Input && input) const { - if (repoInfo.submodules) + if (getSubmodulesAttr(input)) /* Create mountpoints for the submodules. */ for (auto & submodule : repoInfo.workdirInfo.submodules) repoInfo.workdirInfo.files.insert(submodule.path); @@ -630,7 +634,7 @@ struct GitInputScheme : InputScheme /* If the repo has submodules, return a mounted input accessor consisting of the accessor for the top-level repo and the accessors for the submodule workdirs. */ - if (repoInfo.submodules && !repoInfo.workdirInfo.submodules.empty()) { + if (getSubmodulesAttr(input) && !repoInfo.workdirInfo.submodules.empty()) { std::map> mounts; for (auto & submodule : repoInfo.workdirInfo.submodules) { From 84128461b68f6274f1cbf309fd019959132f3c2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 15 Nov 2023 09:23:26 +0100 Subject: [PATCH 605/909] Add a new `nix store add` command Deprecate `nix store add-file` and `nix store add-path`, and replace them with a single `nix store add` command. --- doc/manual/src/release-notes/rl-next.md | 2 + src/nix/add-file.md | 28 ---------- src/nix/add-to-store.cc | 70 +++++++++++++++++-------- src/nix/{add-path.md => add.md} | 2 +- tests/functional/add.sh | 17 ++++++ 5 files changed, 68 insertions(+), 51 deletions(-) delete mode 100644 src/nix/add-file.md rename src/nix/{add-path.md => add.md} (94%) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 1e6ad6922..422f1fce8 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -73,3 +73,5 @@ [`XDG_DATA_DIRS`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables) is now populated with the path to the `/share` subdirectory of the current profile. This means that command completion scripts, `.desktop` files, and similar artifacts installed via [`nix-env`](@docroot@/command-ref/nix-env.md) or [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). + +- A new command `nix store add` has been added. It replaces `nix store add-file` and `nix store add-path` which are now deprecated. diff --git a/src/nix/add-file.md b/src/nix/add-file.md deleted file mode 100644 index ed237a035..000000000 --- a/src/nix/add-file.md +++ /dev/null @@ -1,28 +0,0 @@ -R""( - -# Description - -Copy the regular file *path* to the Nix store, and print the resulting -store path on standard output. - -> **Warning** -> -> The resulting store path is not registered as a garbage -> collector root, so it could be deleted before you have a -> chance to register it. - -# Examples - -Add a regular file to the store: - -```console -# echo foo > bar - -# nix store add-file ./bar -/nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar - -# cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar -foo -``` - -)"" diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 39e5cc99d..f9d487ada 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -5,11 +5,22 @@ using namespace nix; +static FileIngestionMethod parseIngestionMethod(std::string_view input) +{ + if (input == "flat") { + return FileIngestionMethod::Flat; + } else if (input == "nar") { + return FileIngestionMethod::Recursive; + } else { + throw UsageError("Unknown hash mode '%s', expect `flat` or `nar`"); + } +} + struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; - FileIngestionMethod ingestionMethod; + FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive; CmdAddToStore() { @@ -23,6 +34,23 @@ struct CmdAddToStore : MixDryRun, StoreCommand .labels = {"name"}, .handler = {&namePart}, }); + + addFlag({ + .longName = "mode", + .shortName = 'n', + .description = R"( + How to compute the hash of the input. + One of: + + - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + + - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + )", + .labels = {"hash-mode"}, + .handler = {[this](std::string s) { + this->ingestionMethod = parseIngestionMethod(s); + }}, + }); } void run(ref store) override @@ -62,6 +90,22 @@ struct CmdAddToStore : MixDryRun, StoreCommand } }; +struct CmdAdd : CmdAddToStore +{ + + std::string description() override + { + return "Add a file or directory to the Nix store"; + } + + std::string doc() override + { + return + #include "add.md" + ; + } +}; + struct CmdAddFile : CmdAddToStore { CmdAddFile() @@ -71,36 +115,18 @@ struct CmdAddFile : CmdAddToStore std::string description() override { - return "add a regular file to the Nix store"; - } - - std::string doc() override - { - return - #include "add-file.md" - ; + return "Deprecated. Use [`nix store add --mode flat`](@docroot@/command-ref/new-cli/nix3-store-add.md) instead."; } }; struct CmdAddPath : CmdAddToStore { - CmdAddPath() - { - ingestionMethod = FileIngestionMethod::Recursive; - } - std::string description() override { - return "add a path to the Nix store"; - } - - std::string doc() override - { - return - #include "add-path.md" - ; + return "Deprecated alias to [`nix store add`](@docroot@/command-ref/new-cli/nix3-store-add.md)."; } }; static auto rCmdAddFile = registerCommand2({"store", "add-file"}); static auto rCmdAddPath = registerCommand2({"store", "add-path"}); +static auto rCmdAdd = registerCommand2({"store", "add"}); diff --git a/src/nix/add-path.md b/src/nix/add.md similarity index 94% rename from src/nix/add-path.md rename to src/nix/add.md index 87473611d..d38cd21d8 100644 --- a/src/nix/add-path.md +++ b/src/nix/add.md @@ -19,7 +19,7 @@ Add a directory to the store: # mkdir dir # echo foo > dir/bar -# nix store add-path ./dir +# nix store add ./dir /nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir # cat /nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir/bar diff --git a/tests/functional/add.sh b/tests/functional/add.sh index 5c3eed793..d0fedcb25 100644 --- a/tests/functional/add.sh +++ b/tests/functional/add.sh @@ -26,3 +26,20 @@ hash2=$(nix-hash --type sha256 --base32 ./dummy) echo $hash2 test "$hash1" = "sha256:$hash2" + +#### New style commands + +clearStore + +( + path1=$(nix store add ./dummy) + path2=$(nix store add --mode nar ./dummy) + path3=$(nix store add-path ./dummy) + [[ "$path1" == "$path2" ]] + [[ "$path1" == "$path3" ]] +) +( + path1=$(nix store add --mode flat ./dummy) + path2=$(nix store add-file ./dummy) + [[ "$path1" == "$path2" ]] +) From 5196613e8290a9ee81f1b9d88e7bc61cc3f64d2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 18 Nov 2022 11:13:32 +0100 Subject: [PATCH 606/909] Use boost small vectors instead of VLAs in the primops VLAs are a dangerous feature, and their usage triggers an undefined behavior since theire size can be zero in some cases. So replace them with `boost::small_vector`s which fit the same goal but are safer. It's also incidentally consistently 1% faster on the benchmarks. --- src/libexpr/primops.cc | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8d3a18526..e7587506a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -29,7 +29,6 @@ #include - namespace nix { @@ -2729,8 +2728,8 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs")); state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs"); - Value * res[args[1]->listSize()]; - unsigned int found = 0; + boost::container::small_vector res(args[1]->listSize()); + size_t found = 0; for (auto v2 : args[1]->listItems()) { state.forceAttrs(*v2, pos, "while evaluating an element in the list passed as second argument to builtins.catAttrs"); @@ -3064,9 +3063,8 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter"); - // FIXME: putting this on the stack is risky. - Value * vs[args[1]->listSize()]; - unsigned int k = 0; + boost::container::small_vector vs(args[1]->listSize()); + size_t k = 0; bool same = true; for (unsigned int n = 0; n < args[1]->listSize(); ++n) { @@ -3450,7 +3448,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args, state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap"); auto nrLists = args[1]->listSize(); - Value lists[nrLists]; + boost::container::small_vector lists(nrLists); size_t len = 0; for (unsigned int n = 0; n < nrLists; ++n) { From ba3cb4a04949e043669299da5497bea27b944598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 18 Nov 2022 13:02:06 +0100 Subject: [PATCH 607/909] Remove all the occurences of VLAs There's generally no strict reason for using them, and they are somewhat fishy, so let's avoid them. --- src/libexpr/eval.cc | 13 ++++++++----- src/libstore/derivations.cc | 9 ++++----- src/libstore/gc.cc | 9 ++------- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index dfe81cbf7..d853b104b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -41,6 +41,7 @@ #include #include #include +#include #endif @@ -1691,7 +1692,8 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & /* We have all the arguments, so call the primop with the previous and new arguments. */ - Value * vArgs[arity]; + assert(arity < 64); + Value * vArgs[64]; auto n = argsDone; for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left) vArgs[--n] = arg->primOpApp.right; @@ -1748,11 +1750,12 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v) Value vFun; fun->eval(state, env, vFun); - Value * vArgs[args.size()]; + + boost::container::small_vector vArgs(args.size()); for (size_t i = 0; i < args.size(); ++i) vArgs[i] = args[i]->maybeThunk(state, env); - state.callFunction(vFun, args.size(), vArgs, v, pos); + state.callFunction(vFun, args.size(), vArgs.data(), v, pos); } @@ -1991,8 +1994,8 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) return result; }; - Value values[es->size()]; - Value * vTmpP = values; + boost::container::small_vector values(es->size()); + Value * vTmpP = values.data(); for (auto & [i_pos, i] : *es) { Value & vTmp = *vTmpP++; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 1fecd1c97..6d9c8b9d6 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -151,11 +151,10 @@ StorePath writeDerivation(Store & store, /* Read string `s' from stream `str'. */ static void expect(std::istream & str, std::string_view s) { - char s2[s.size()]; - str.read(s2, s.size()); - std::string_view s2View { s2, s.size() }; - if (s2View != s) - throw FormatError("expected string '%s', got '%s'", s, s2View); + for (auto & c : s) { + if (str.get() != c) + throw FormatError("expected string '%1%'", s); + } } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 8d05ae4bd..ddec43fdc 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -330,9 +330,7 @@ typedef std::unordered_map> UncheckedRoots static void readProcLink(const std::string & file, UncheckedRoots & roots) { - /* 64 is the starting buffer size gnu readlink uses... */ - auto bufsiz = ssize_t{64}; -try_again: + constexpr auto bufsiz = PATH_MAX; char buf[bufsiz]; auto res = readlink(file.c_str(), buf, bufsiz); if (res == -1) { @@ -341,10 +339,7 @@ try_again: throw SysError("reading symlink"); } if (res == bufsiz) { - if (SSIZE_MAX / 2 < bufsiz) - throw Error("stupidly long symlink"); - bufsiz *= 2; - goto try_again; + throw Error("stupidly long symlink"); } if (res > 0 && buf[0] == '/') roots[std::string(static_cast(buf), res)] From 0daccb1121dfd5e98db3e41ba992b1b2c413dfc8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 16 Nov 2023 11:10:25 +0100 Subject: [PATCH 608/909] libexpr: Check primop arity earlier --- src/libexpr/eval.cc | 20 ++++++++++++++++++-- src/libexpr/eval.hh | 12 ++++++++++++ src/libexpr/tests/value/print.cc | 3 ++- src/libexpr/value.hh | 8 +------- 4 files changed, 33 insertions(+), 10 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d853b104b..1425eab97 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -723,6 +723,23 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info) } +void PrimOp::check() +{ + if (arity > maxPrimOpArity) { + throw Error("primop arity must not exceed %1%", maxPrimOpArity); + } +} + + +void Value::mkPrimOp(PrimOp * p) +{ + p->check(); + clearValue(); + internalType = tPrimOp; + primOp = p; +} + + Value * EvalState::addPrimOp(PrimOp && primOp) { /* Hack to make constants lazy: turn them into a application of @@ -1692,8 +1709,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & /* We have all the arguments, so call the primop with the previous and new arguments. */ - assert(arity < 64); - Value * vArgs[64]; + Value * vArgs[maxPrimOpArity]; auto n = argsDone; for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left) vArgs[--n] = arg->primOpApp.right; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 048dff42b..5ee6359a8 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -18,6 +18,12 @@ namespace nix { +/** + * We put a limit on primop arity because it lets us use a fixed size array on + * the stack. 16 is already an impractical number of arguments. Use an attrset + * argument for such overly complicated functions. + */ +constexpr size_t maxPrimOpArity = 64; class Store; class EvalState; @@ -71,6 +77,12 @@ struct PrimOp * Optional experimental for this to be gated on. */ std::optional experimentalFeature; + + /** + * Validity check to be performed by functions that introduce primops, + * such as RegisterPrimOp() and Value::mkPrimOp(). + */ + void check(); }; /** diff --git a/src/libexpr/tests/value/print.cc b/src/libexpr/tests/value/print.cc index 5e96e12ec..a4f6fc014 100644 --- a/src/libexpr/tests/value/print.cc +++ b/src/libexpr/tests/value/print.cc @@ -114,7 +114,8 @@ TEST_F(ValuePrintingTests, vLambda) TEST_F(ValuePrintingTests, vPrimOp) { Value vPrimOp; - vPrimOp.mkPrimOp(nullptr); + PrimOp primOp{}; + vPrimOp.mkPrimOp(&primOp); test(vPrimOp, ""); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 622e613ea..191cc30ba 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -354,13 +354,7 @@ public: // Value will be overridden anyways } - inline void mkPrimOp(PrimOp * p) - { - clearValue(); - internalType = tPrimOp; - primOp = p; - } - + void mkPrimOp(PrimOp * p); inline void mkPrimOpApp(Value * l, Value * r) { From 12c91a823e80b5e0a14a0abb0f34a6633b14bbfe Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 16 Nov 2023 11:27:31 +0100 Subject: [PATCH 609/909] maxPrimOpArity: 64 -> 8 This makes stack usage significantly more compact, allowing larger amounts of data to be processed on the same stack. PrimOp functions with more than 8 positional (curried) arguments should use an attrset instead. --- src/libexpr/eval.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 5ee6359a8..ce798ed96 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -20,10 +20,10 @@ namespace nix { /** * We put a limit on primop arity because it lets us use a fixed size array on - * the stack. 16 is already an impractical number of arguments. Use an attrset + * the stack. 8 is already an impractical number of arguments. Use an attrset * argument for such overly complicated functions. */ -constexpr size_t maxPrimOpArity = 64; +constexpr size_t maxPrimOpArity = 8; class Store; class EvalState; From 9fa133dde5610dfb0605399ffea83081bda1c6fc Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 16 Nov 2023 11:33:10 +0100 Subject: [PATCH 610/909] readProcLink: Replace unnecessary value judgement by actual info --- src/libstore/gc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ddec43fdc..93fa60682 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -339,7 +339,7 @@ static void readProcLink(const std::string & file, UncheckedRoots & roots) throw SysError("reading symlink"); } if (res == bufsiz) { - throw Error("stupidly long symlink"); + throw Error("overly long symlink starting with '%1%'", std::string_view(buf, bufsiz)); } if (res > 0 && buf[0] == '/') roots[std::string(static_cast(buf), res)] From 206ece0f41142536a856c62c49bd202282f12db8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 16 Nov 2023 12:18:37 +0100 Subject: [PATCH 611/909] builtins.{any,all}: Use constant errorCtx Clang warned that the expanded code used to have a buffer overflow. Very strange, but also very avoidable. --- src/libexpr/primops.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e7587506a..d104b7180 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3189,10 +3189,14 @@ static void anyOrAll(bool any, EvalState & state, const PosIdx pos, Value * * ar state.forceFunction(*args[0], pos, std::string("while evaluating the first argument passed to builtins.") + (any ? "any" : "all")); state.forceList(*args[1], pos, std::string("while evaluating the second argument passed to builtins.") + (any ? "any" : "all")); + std::string_view errorCtx = any + ? "while evaluating the return value of the function passed to builtins.any" + : "while evaluating the return value of the function passed to builtins.all"; + Value vTmp; for (auto elem : args[1]->listItems()) { state.callFunction(*args[0], *elem, vTmp, pos); - bool res = state.forceBool(vTmp, pos, std::string("while evaluating the return value of the function passed to builtins.") + (any ? "any" : "all")); + bool res = state.forceBool(vTmp, pos, errorCtx); if (res == any) { v.mkBool(any); return; From 91114a6fa48e2eb9399c23938eb12fdbd4fcda42 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 16 Nov 2023 12:44:10 +0100 Subject: [PATCH 612/909] ExprCall::eval: Heap allocate at arity 5+ --- src/libexpr/eval.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 1425eab97..bfbda52ef 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1766,8 +1766,13 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v) Value vFun; fun->eval(state, env, vFun); - - boost::container::small_vector vArgs(args.size()); + // Empirical arity of Nixpkgs lambdas by regex e.g. ([a-zA-Z]+:(\s|(/\*.*\/)|(#.*\n))*){5} + // 2: over 4000 + // 3: about 300 + // 4: about 60 + // 5: under 10 + // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total. + boost::container::small_vector vArgs(args.size()); for (size_t i = 0; i < args.size(); ++i) vArgs[i] = args[i]->maybeThunk(state, env); From 898c47384f651f51b3e4b63c271da274db8fca2e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 16 Nov 2023 12:48:37 +0100 Subject: [PATCH 613/909] primops: Err on the side of less stack usage Try to stay away from stack overflows. These small vectors use stack space. Most instances will not need to allocate because in general most things are small, and large things are worth heap allocating. 16 * 3 * word = 384 bytes is still quite a bit, but these functions tend not to be part of deep recursions. --- src/libexpr/eval.cc | 2 +- src/libexpr/primops.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index bfbda52ef..2fcbf3311 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2015,7 +2015,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) return result; }; - boost::container::small_vector values(es->size()); + boost::container::small_vector values(es->size()); Value * vTmpP = values.data(); for (auto & [i_pos, i] : *es) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d104b7180..7aa212281 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2549,7 +2549,7 @@ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args /* Get the attribute names to be removed. We keep them as Attrs instead of Symbols so std::set_difference can be used to remove them from attrs[0]. */ - boost::container::small_vector names; + boost::container::small_vector names; names.reserve(args[1]->listSize()); for (auto elem : args[1]->listItems()) { state.forceStringNoCtx(*elem, pos, "while evaluating the values of the second argument passed to builtins.removeAttrs"); @@ -3452,7 +3452,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args, state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap"); auto nrLists = args[1]->listSize(); - boost::container::small_vector lists(nrLists); + boost::container::small_vector lists(nrLists); size_t len = 0; for (unsigned int n = 0; n < nrLists; ++n) { From 1b9813e4e60836ddb1467efd50c572e7579ac945 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 16 Nov 2023 13:04:27 +0100 Subject: [PATCH 614/909] primops: Name stack reservation limits --- src/libexpr/eval.cc | 3 ++- src/libexpr/primops.cc | 6 +++--- src/libexpr/primops.hh | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 2fcbf3311..8b0ada517 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1,6 +1,7 @@ #include "eval.hh" #include "eval-settings.hh" #include "hash.hh" +#include "primops.hh" #include "types.hh" #include "util.hh" #include "store-api.hh" @@ -2015,7 +2016,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) return result; }; - boost::container::small_vector values(es->size()); + boost::container::small_vector values(es->size()); Value * vTmpP = values.data(); for (auto & [i_pos, i] : *es) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7aa212281..adce95bed 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2728,7 +2728,7 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs")); state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs"); - boost::container::small_vector res(args[1]->listSize()); + boost::container::small_vector res(args[1]->listSize()); size_t found = 0; for (auto v2 : args[1]->listItems()) { @@ -3063,7 +3063,7 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter"); - boost::container::small_vector vs(args[1]->listSize()); + boost::container::small_vector vs(args[1]->listSize()); size_t k = 0; bool same = true; @@ -3452,7 +3452,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args, state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap"); auto nrLists = args[1]->listSize(); - boost::container::small_vector lists(nrLists); + boost::container::small_vector lists(nrLists); size_t len = 0; for (unsigned int n = 0; n < nrLists; ++n) { diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 930e7f32a..1d5d5710d 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -8,6 +8,22 @@ namespace nix { +/** + * For functions where we do not expect deep recursion, we can use a sizable + * part of the stack a free allocation space. + * + * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes. + */ +constexpr size_t nonRecursiveStackReservation = 256; + +/** + * Functions that maybe applied to self-similar inputs, such as concatMap on a + * tree, should reserve a smaller part of the stack for allocation. + * + * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes. + */ +constexpr size_t conservativeStackReservation = 16; + struct RegisterPrimOp { typedef std::vector PrimOps; From a96be29db536177fdc284b51a3b2af44a70496e0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 16 Nov 2023 13:13:01 +0100 Subject: [PATCH 615/909] removeAttrs: increase stack reservation to 64 --- src/libexpr/primops.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index adce95bed..e274c3c0c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2549,7 +2549,8 @@ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args /* Get the attribute names to be removed. We keep them as Attrs instead of Symbols so std::set_difference can be used to remove them from attrs[0]. */ - boost::container::small_vector names; + // 64: large enough to fit the attributes of a derivation + boost::container::small_vector names; names.reserve(args[1]->listSize()); for (auto elem : args[1]->listItems()) { state.forceStringNoCtx(*elem, pos, "while evaluating the values of the second argument passed to builtins.removeAttrs"); From 4e27f1947a444a36d6a85f41cbf1afdc70ac6c4c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 16 Nov 2023 13:23:17 +0100 Subject: [PATCH 616/909] libexpr: Reduce nonRecursiveStackReservation 128 is still beyond the point where the allocation overhead is insignificant, but we don't anticipate to overflow for these use cases, so it's fine. --- src/libexpr/primops.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 1d5d5710d..45486608f 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -14,7 +14,7 @@ namespace nix { * * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes. */ -constexpr size_t nonRecursiveStackReservation = 256; +constexpr size_t nonRecursiveStackReservation = 128; /** * Functions that maybe applied to self-similar inputs, such as concatMap on a From 6c8f4ef3502aa214557541ec00538e41aeced6e3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 14 Nov 2023 10:52:57 -0500 Subject: [PATCH 617/909] Allow installing unit tests Closes #9343 See that issue for motivation. Installing these is disabled by default, but we enable it (and the additional output we want isntall these too so as not to clutter the existing ones) to use in cross builds and dev shells. --- Makefile.config.in | 3 +++ configure.ac | 12 ++++++++++++ flake.nix | 10 ++++++++-- src/libexpr/tests/local.mk | 6 +++++- src/libstore/tests/local.mk | 12 ++++++++++-- src/libutil/tests/local.mk | 12 ++++++++++-- 6 files changed, 48 insertions(+), 7 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index 19992fa20..1482db81f 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -28,6 +28,8 @@ SODIUM_LIBS = @SODIUM_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@ bash = @bash@ bindir = @bindir@ +checkbindir = @checkbindir@ +checklibdir = @checklibdir@ datadir = @datadir@ datarootdir = @datarootdir@ doc_generate = @doc_generate@ @@ -48,4 +50,5 @@ sysconfdir = @sysconfdir@ system = @system@ ENABLE_BUILD = @ENABLE_BUILD@ ENABLE_TESTS = @ENABLE_TESTS@ +INSTALL_UNIT_TESTS = @INSTALL_UNIT_TESTS@ internal_api_docs = @internal_api_docs@ diff --git a/configure.ac b/configure.ac index 75ce7d01d..281ba2c32 100644 --- a/configure.ac +++ b/configure.ac @@ -167,6 +167,18 @@ AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]), ENABLE_TESTS=$enableval, ENABLE_TESTS=yes) AC_SUBST(ENABLE_TESTS) +AC_ARG_ENABLE(install-unit-tests, AS_HELP_STRING([--enable-install-unit-tests],[Install the unit tests for running later (default no)]), + INSTALL_UNIT_TESTS=$enableval, INSTALL_UNIT_TESTS=no) +AC_SUBST(INSTALL_UNIT_TESTS) + +AC_ARG_WITH(check-bin-dir, AS_HELP_STRING([--with-check-bin-dir=PATH],[path to install unit tests for running later (defaults to $libexecdir/nix)]), + checkbindir=$withval, checkbindir=$libexecdir/nix) +AC_SUBST(checkbindir) + +AC_ARG_WITH(check-lib-dir, AS_HELP_STRING([--with-check-lib-dir=PATH],[path to install unit tests for running later (defaults to $libdir)]), + checklibdir=$withval, checklibdir=$libdir) +AC_SUBST(checklibdir) + # Building without API docs is the default as Nix' C++ interfaces are internal and unstable. AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]), internal_api_docs=$enableval, internal_api_docs=no) diff --git a/flake.nix b/flake.nix index 51d818423..05ab7b06d 100644 --- a/flake.nix +++ b/flake.nix @@ -164,6 +164,10 @@ testConfigureFlags = [ "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" + ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ + "--enable-install-unit-tests" + "--with-check-bin-dir=${builtins.placeholder "check"}/bin" + "--with-check-lib-dir=${builtins.placeholder "check"}/lib" ]; internalApiDocsConfigureFlags = [ @@ -404,7 +408,8 @@ src = nixSrc; VERSION_SUFFIX = versionSuffix; - outputs = [ "out" "dev" "doc" ]; + outputs = [ "out" "dev" "doc" ] + ++ lib.optional (currentStdenv.hostPlatform != currentStdenv.buildPlatform) "check"; nativeBuildInputs = nativeBuildDeps; buildInputs = buildDeps @@ -710,7 +715,8 @@ stdenv.mkDerivation { name = "nix"; - outputs = [ "out" "dev" "doc" ]; + outputs = [ "out" "dev" "doc" ] + ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "check"; nativeBuildInputs = nativeBuildDeps ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk index 331a5ead6..6d2a04aaf 100644 --- a/src/libexpr/tests/local.mk +++ b/src/libexpr/tests/local.mk @@ -6,7 +6,11 @@ libexpr-tests_NAME := libnixexpr-tests libexpr-tests_DIR := $(d) -libexpr-tests_INSTALL_DIR := +ifeq ($(INSTALL_UNIT_TESTS), yes) + libexpr-tests_INSTALL_DIR := $(checkbindir) +else + libexpr-tests_INSTALL_DIR := +endif libexpr-tests_SOURCES := \ $(wildcard $(d)/*.cc) \ diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk index 03becc7d1..e9b8b4f99 100644 --- a/src/libstore/tests/local.mk +++ b/src/libstore/tests/local.mk @@ -6,7 +6,11 @@ libstore-tests-exe_NAME = libnixstore-tests libstore-tests-exe_DIR := $(d) -libstore-tests-exe_INSTALL_DIR := +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-tests-exe_INSTALL_DIR := $(checkbindir) +else + libstore-tests-exe_INSTALL_DIR := +endif libstore-tests-exe_LIBS = libstore-tests @@ -18,7 +22,11 @@ libstore-tests_NAME = libnixstore-tests libstore-tests_DIR := $(d) -libstore-tests_INSTALL_DIR := +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-tests_INSTALL_DIR := $(checklibdir) +else + libstore-tests_INSTALL_DIR := +endif libstore-tests_SOURCES := $(wildcard $(d)/*.cc) diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index c8b8557cb..e6fc4e364 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -6,7 +6,11 @@ libutil-tests-exe_NAME = libnixutil-tests libutil-tests-exe_DIR := $(d) -libutil-tests-exe_INSTALL_DIR := +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-tests-exe_INSTALL_DIR := $(checkbindir) +else + libutil-tests-exe_INSTALL_DIR := +endif libutil-tests-exe_LIBS = libutil-tests @@ -18,7 +22,11 @@ libutil-tests_NAME = libnixutil-tests libutil-tests_DIR := $(d) -libutil-tests_INSTALL_DIR := +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-tests_INSTALL_DIR := $(checklibdir) +else + libutil-tests_INSTALL_DIR := +endif libutil-tests_SOURCES := $(wildcard $(d)/*.cc) From 31ebc6028b3682969d86a7b39ae87131c41cc604 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Nov 2023 16:45:14 +0100 Subject: [PATCH 618/909] Fix symlink handling This restores the symlink handling behaviour prior to 94812cca98fbb157e5f64a15a85a2b852d289feb. Fixes #9298. --- src/libexpr/eval.hh | 2 +- src/libexpr/parser.y | 18 +++++++++++++----- .../lang/eval-okay-symlink-resolution.exp | 1 + .../lang/eval-okay-symlink-resolution.nix | 1 + .../symlink-resolution/foo/lib/default.nix | 1 + .../lang/symlink-resolution/foo/overlays | 1 + .../symlink-resolution/overlays/overlay.nix | 1 + 7 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 tests/functional/lang/eval-okay-symlink-resolution.exp create mode 100644 tests/functional/lang/eval-okay-symlink-resolution.nix create mode 100644 tests/functional/lang/symlink-resolution/foo/lib/default.nix create mode 120000 tests/functional/lang/symlink-resolution/foo/overlays create mode 100644 tests/functional/lang/symlink-resolution/overlays/overlay.nix diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 048dff42b..9257a0e48 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -827,7 +827,7 @@ std::string showType(const Value & v); /** * If `path` refers to a directory, then append "/default.nix". */ -SourcePath resolveExprPath(const SourcePath & path); +SourcePath resolveExprPath(SourcePath path); struct InvalidPathError : EvalError { diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index b86cef217..f6cf1f689 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -686,17 +686,25 @@ Expr * EvalState::parse( } -SourcePath resolveExprPath(const SourcePath & path) +SourcePath resolveExprPath(SourcePath path) { + unsigned int followCount = 0, maxFollow = 1024; + /* If `path' is a symlink, follow it. This is so that relative path references work. */ - auto path2 = path.resolveSymlinks(); + while (true) { + // Basic cycle/depth limit to avoid infinite loops. + if (++followCount >= maxFollow) + throw Error("too many symbolic links encountered while traversing the path '%s'", path); + if (path.lstat().type != InputAccessor::tSymlink) break; + path = {path.accessor, CanonPath(path.readLink(), path.path.parent().value_or(CanonPath::root))}; + } /* If `path' refers to a directory, append `/default.nix'. */ - if (path2.lstat().type == InputAccessor::tDirectory) - return path2 + "default.nix"; + if (path.lstat().type == InputAccessor::tDirectory) + return path + "default.nix"; - return path2; + return path; } diff --git a/tests/functional/lang/eval-okay-symlink-resolution.exp b/tests/functional/lang/eval-okay-symlink-resolution.exp new file mode 100644 index 000000000..8b8441b91 --- /dev/null +++ b/tests/functional/lang/eval-okay-symlink-resolution.exp @@ -0,0 +1 @@ +"test" diff --git a/tests/functional/lang/eval-okay-symlink-resolution.nix b/tests/functional/lang/eval-okay-symlink-resolution.nix new file mode 100644 index 000000000..ffb1818bd --- /dev/null +++ b/tests/functional/lang/eval-okay-symlink-resolution.nix @@ -0,0 +1 @@ +import symlink-resolution/foo/overlays/overlay.nix diff --git a/tests/functional/lang/symlink-resolution/foo/lib/default.nix b/tests/functional/lang/symlink-resolution/foo/lib/default.nix new file mode 100644 index 000000000..8b8441b91 --- /dev/null +++ b/tests/functional/lang/symlink-resolution/foo/lib/default.nix @@ -0,0 +1 @@ +"test" diff --git a/tests/functional/lang/symlink-resolution/foo/overlays b/tests/functional/lang/symlink-resolution/foo/overlays new file mode 120000 index 000000000..0d44a21c5 --- /dev/null +++ b/tests/functional/lang/symlink-resolution/foo/overlays @@ -0,0 +1 @@ +../overlays \ No newline at end of file diff --git a/tests/functional/lang/symlink-resolution/overlays/overlay.nix b/tests/functional/lang/symlink-resolution/overlays/overlay.nix new file mode 100644 index 000000000..b0368308e --- /dev/null +++ b/tests/functional/lang/symlink-resolution/overlays/overlay.nix @@ -0,0 +1 @@ +import ../lib From 96d67620d551c7143b6682cfff74a2ee2edbe863 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Nov 2023 17:12:06 +0100 Subject: [PATCH 619/909] Fix a broken generated header file dependency https://hydra.nixos.org/build/240882042 --- src/libexpr/local.mk | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index d243b9cec..ed7bf9490 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -43,7 +43,9 @@ $(foreach i, $(wildcard src/libexpr/value/*.hh), \ $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) -$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh +$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh + +$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh From c81937576928ca494af0be6e7c61f2070be5d353 Mon Sep 17 00:00:00 2001 From: Dominic Shelton Date: Fri, 17 Nov 2023 17:50:17 +1100 Subject: [PATCH 620/909] doc: Add example of inherit in a let expression --- doc/manual/src/language/constructs.md | 54 ++++++++++++++++++++------- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md index a3590f55d..bd7bd92a5 100644 --- a/doc/manual/src/language/constructs.md +++ b/doc/manual/src/language/constructs.md @@ -132,6 +132,32 @@ a = src-set.a; b = src-set.b; c = src-set.c; when used while defining local variables in a let-expression or while defining a set. +in a let expression, inherit can be used to selectively bring specific attributes of a set into scope. For example + + +```nix +let + x = { a = 1; b = 2; }; + inherit (builtins) attrNames; +in +{ + names = attrNames x; +} +``` + +is equivalent to + +```nix +let + x = { a = 1; b = 2; }; +in +{ + names = builtins.attrNames x; +} +``` + +both resolve to `{ names = [ "a" "b" ]; }`. + ## Functions Functions have the following form: @@ -146,65 +172,65 @@ three kinds of patterns: - If a pattern is a single identifier, then the function matches any argument. Example: - + ```nix let negate = x: !x; concat = x: y: x + y; in if negate true then concat "foo" "bar" else "" ``` - + Note that `concat` is a function that takes one argument and returns a function that takes another argument. This allows partial parameterisation (i.e., only filling some of the arguments of a function); e.g., - + ```nix map (concat "foo") [ "bar" "bla" "abc" ] ``` - + evaluates to `[ "foobar" "foobla" "fooabc" ]`. - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a set containing the listed attributes, and binds the values of those attributes to variables in the function body. For example, the function - + ```nix { x, y, z }: z + y + x ``` - + can only be called with a set containing exactly the attributes `x`, `y` and `z`. No other attributes are allowed. If you want to allow additional arguments, you can use an ellipsis (`...`): - + ```nix { x, y, z, ... }: z + y + x ``` - + This works on any set that contains at least the three named attributes. - + It is possible to provide *default values* for attributes, in which case they are allowed to be missing. A default value is specified by writing `name ? e`, where *e* is an arbitrary expression. For example, - + ```nix { x, y ? "foo", z ? "bar" }: z + y + x ``` - + specifies a function that only requires an attribute named `x`, but optionally accepts `y` and `z`. - An `@`-pattern provides a means of referring to the whole value being matched: - + ```nix args@{ x, y, z, ... }: z + y + x + args.a ``` - + but can also be written as: - + ```nix { x, y, z, ... } @ args: z + y + x + args.a ``` From 2eb59c34b531f03a85f67b9246ccaf0ff5fcad23 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Nov 2023 03:02:02 +0100 Subject: [PATCH 621/909] Value: extract Value::StringWithContext --- src/libexpr/value.hh | 54 +++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 191cc30ba..0f8cef418 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -158,37 +158,39 @@ public: inline bool isPrimOp() const { return internalType == tPrimOp; }; inline bool isPrimOpApp() const { return internalType == tPrimOpApp; }; + /** + * Strings in the evaluator carry a so-called `context` which + * is a list of strings representing store paths. This is to + * allow users to write things like + * + * "--with-freetype2-library=" + freetype + "/lib" + * + * where `freetype` is a derivation (or a source to be copied + * to the store). If we just concatenated the strings without + * keeping track of the referenced store paths, then if the + * string is used as a derivation attribute, the derivation + * will not have the correct dependencies in its inputDrvs and + * inputSrcs. + + * The semantics of the context is as follows: when a string + * with context C is used as a derivation attribute, then the + * derivations in C will be added to the inputDrvs of the + * derivation, and the other store paths in C will be added to + * the inputSrcs of the derivations. + + * For canonicity, the store paths should be in sorted order. + */ + struct StringWithContext { + const char * c_str; + const char * * context; // must be in sorted order + }; + union { NixInt integer; bool boolean; - /** - * Strings in the evaluator carry a so-called `context` which - * is a list of strings representing store paths. This is to - * allow users to write things like - - * "--with-freetype2-library=" + freetype + "/lib" - - * where `freetype` is a derivation (or a source to be copied - * to the store). If we just concatenated the strings without - * keeping track of the referenced store paths, then if the - * string is used as a derivation attribute, the derivation - * will not have the correct dependencies in its inputDrvs and - * inputSrcs. - - * The semantics of the context is as follows: when a string - * with context C is used as a derivation attribute, then the - * derivations in C will be added to the inputDrvs of the - * derivation, and the other store paths in C will be added to - * the inputSrcs of the derivations. - - * For canonicity, the store paths should be in sorted order. - */ - struct { - const char * c_str; - const char * * context; // must be in sorted order - } string; + StringWithContext string; struct { InputAccessor * accessor; From d8ff5cfe8eba34c8b4b5cc53f3b40cd3dfd84224 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Nov 2023 03:03:37 +0100 Subject: [PATCH 622/909] Value: extract Value::Path --- src/libexpr/value.hh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 0f8cef418..34f994997 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -185,6 +185,11 @@ public: const char * * context; // must be in sorted order }; + struct Path { + InputAccessor * accessor; + const char * path; + }; + union { NixInt integer; @@ -192,10 +197,7 @@ public: StringWithContext string; - struct { - InputAccessor * accessor; - const char * path; - } _path; + Path _path; Bindings * attrs; struct { From b55203e874f8e4b2fc5289129efba791937c23d0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Nov 2023 03:06:04 +0100 Subject: [PATCH 623/909] Value: extract Value::ClosureThunk --- src/libexpr/value.hh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 34f994997..4c51c52e4 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -190,6 +190,11 @@ public: const char * path; }; + struct ClosureThunk { + Env * env; + Expr * expr; + }; + union { NixInt integer; @@ -205,10 +210,7 @@ public: Value * * elems; } bigList; Value * smallList[2]; - struct { - Env * env; - Expr * expr; - } thunk; + ClosureThunk thunk; struct { Value * left, * right; } app; From 6af1d9f7b94da454252b62f0cfff4ce800c5a46b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Nov 2023 03:06:55 +0100 Subject: [PATCH 624/909] Value: extract Value::FunctionApplicationThunk --- src/libexpr/value.hh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 4c51c52e4..cfb3f5276 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -195,6 +195,10 @@ public: Expr * expr; }; + struct FunctionApplicationThunk { + Value * left, * right; + }; + union { NixInt integer; @@ -211,17 +215,13 @@ public: } bigList; Value * smallList[2]; ClosureThunk thunk; - struct { - Value * left, * right; - } app; + FunctionApplicationThunk app; struct { Env * env; ExprLambda * fun; } lambda; PrimOp * primOp; - struct { - Value * left, * right; - } primOpApp; + FunctionApplicationThunk primOpApp; ExternalValueBase * external; NixFloat fpoint; }; From 7055c6528532fdd3b0ce9b8f5282b002fc011470 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Nov 2023 03:07:32 +0100 Subject: [PATCH 625/909] Value: extract Value::Lambda --- src/libexpr/value.hh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index cfb3f5276..93ccdbc2e 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -199,6 +199,11 @@ public: Value * left, * right; }; + struct Lambda { + Env * env; + ExprLambda * fun; + }; + union { NixInt integer; @@ -216,10 +221,7 @@ public: Value * smallList[2]; ClosureThunk thunk; FunctionApplicationThunk app; - struct { - Env * env; - ExprLambda * fun; - } lambda; + Lambda lambda; PrimOp * primOp; FunctionApplicationThunk primOpApp; ExternalValueBase * external; From 260c6147625e95e4772ccdee80d6463d242c7b64 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Nov 2023 03:31:45 +0100 Subject: [PATCH 626/909] Value: use std::span, change use of const **`Value` and `const`** These two deserve some explanation. We'll get to lists later. Values can normally be thought of as immutable, except they are are also the vehicle for call by need, which must be implemented using mutation. This circumstance makes a `const Value` a rather useless thing: - If it's a thunk, you can't evaluate it, except by copying, but that would not be call by need. - If it's not a thunk, you know the type, so the method that acquired it for you should have returned something more specific, such as a `const Bindings &` (which actually does make sense because that's an immutable span of pointers to mutable `Value`s. - If you don't care about the type yet, you might establish the convention that `const Value` means `deepSeq`-ed data, but this is hardly useful and not actually as safe as you would supposedly want to trust it to be - just convention. **Lists** `std::span` is a tuple of pointer and size - just what we need. We don't return them as `const Value`, because considering the first bullet point we discussed before, we'd have to force all the list values, which isn't what we want. So what we end up with is a nice representation of a list in weak head normal form: the spine is immutable, but the items may need some evaluation later. --- src/libexpr/value.hh | 37 ++++++++----------------------------- 1 file changed, 8 insertions(+), 29 deletions(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 93ccdbc2e..bcff8ae55 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -3,6 +3,7 @@ #include #include +#include #include "symbol-table.hh" #include "value/context.hh" @@ -395,7 +396,13 @@ public: return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; } - const Value * const * listElems() const + std::span listItems() const + { + assert(isList()); + return std::span(listElems(), listSize()); + } + + Value * const * listElems() const { return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; } @@ -414,34 +421,6 @@ public: */ bool isTrivial() const; - auto listItems() - { - struct ListIterable - { - typedef Value * const * iterator; - iterator _begin, _end; - iterator begin() const { return _begin; } - iterator end() const { return _end; } - }; - assert(isList()); - auto begin = listElems(); - return ListIterable { begin, begin + listSize() }; - } - - auto listItems() const - { - struct ConstListIterable - { - typedef const Value * const * iterator; - iterator _begin, _end; - iterator begin() const { return _begin; } - iterator end() const { return _end; } - }; - assert(isList()); - auto begin = listElems(); - return ConstListIterable { begin, begin + listSize() }; - } - SourcePath path() const { assert(internalType == tPath); From 121665f3773bc46ca6df0dda6f66b1a86e7d9e72 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Nov 2023 17:51:09 +0100 Subject: [PATCH 627/909] nix-env: Use state.mkList, required for correct stats --- src/nix-env/nix-env.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 213a20d93..ab1d8f713 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -172,7 +172,7 @@ static void loadSourceExpr(EvalState & state, const SourcePath & path, Value & v directory). */ else if (st.type == InputAccessor::tDirectory) { auto attrs = state.buildBindings(maxAttrs); - attrs.alloc("_combineChannels").mkList(0); + state.mkList(attrs.alloc("_combineChannels"), 0); StringSet seen; getAllExprs(state, path, seen, attrs); v.mkAttrs(attrs); From 7b0e8c5c2c09146722349d3fd2dd69211d8b8945 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 17 Nov 2023 10:56:23 +0100 Subject: [PATCH 628/909] Apply suggestions from code review Co-authored-by: Valentin Gagarin --- doc/manual/src/language/constructs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md index bd7bd92a5..a82ec5960 100644 --- a/doc/manual/src/language/constructs.md +++ b/doc/manual/src/language/constructs.md @@ -132,7 +132,7 @@ a = src-set.a; b = src-set.b; c = src-set.c; when used while defining local variables in a let-expression or while defining a set. -in a let expression, inherit can be used to selectively bring specific attributes of a set into scope. For example +In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example ```nix @@ -156,7 +156,7 @@ in } ``` -both resolve to `{ names = [ "a" "b" ]; }`. +both evaluate to `{ names = [ "a" "b" ]; }`. ## Functions From f7d59d0dda5e4a793e701bc8fb9136b3ef22948c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Nov 2023 14:21:17 +0100 Subject: [PATCH 629/909] Release notes --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/release-notes/rl-2.19.md | 77 +++++++++++++++++++++++++ doc/manual/src/release-notes/rl-next.md | 75 ------------------------ 3 files changed, 78 insertions(+), 75 deletions(-) create mode 100644 doc/manual/src/release-notes/rl-2.19.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 794f78a07..8dc464abd 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -115,6 +115,7 @@ - [C++ style guide](contributing/cxx.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md) - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md) diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md new file mode 100644 index 000000000..4eecaf929 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -0,0 +1,77 @@ +# Release 2.19 (2023-11-17) + +- The experimental `nix` command can now act as a [shebang interpreter](@docroot@/command-ref/new-cli/nix.md#shebang-interpreter) + by appending the contents of any `#! nix` lines and the script's location into a single call. + +- [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. + +- [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). + +- The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly. + +- There is a new flake installable syntax `flakeref#.attrPath` where the "." prefix specifies that `attrPath` is interpreted from the root of the flake outputs, with no searching of default attribute prefixes like `packages.` or `legacyPackages.`. + +- Nix adds `apple-virt` to the default system features on macOS systems that support virtualization. This is similar to what's done for the `kvm` system feature on Linux hosts. + +- Add a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash). + +- `nix-shell` shebang lines now support single-quoted arguments. + +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). + As described in the documentation for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of flakes. + +- The interface for creating and updating lock files has been overhauled: + + - [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now. + It will *never* update existing inputs. + + - [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) does the same, but *will* update inputs. + - Passing no arguments will update all inputs of the current flake, just like it already did. + - Passing input names as arguments will ensure only those are updated. This replaces the functionality of `nix flake lock --update-input` + - To operate on a flake outside the current directory, you must now pass `--flake path/to/flake`. + + - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. + They are superceded by `nix flake update`. + +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + +- [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) + (experimental) now returns a JSON map rather than JSON list. + The `path` field of each object has instead become the key in the outer map, since it is unique. + The `valid` field also goes away because we just use `null` instead. + + - Old way: + + ```json5 + [ + { + "path": "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15", + "valid": true, + // ... + }, + { + "path": "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path", + "valid": false + } + ] + ``` + + - New way + + ```json5 + { + "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15": { + // ... + }, + "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path": null, + } + ``` + + This makes it match `nix derivation show`, which also maps store paths to information. + +- When Nix is installed using the [binary installer](@docroot@/installation/installing-binary.md), in supported shells (Bash, Zsh, Fish) + [`XDG_DATA_DIRS`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables) is now populated with the path to the `/share` subdirectory of the current profile. + This means that command completion scripts, `.desktop` files, and similar artifacts installed via [`nix-env`](@docroot@/command-ref/nix-env.md) or [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) + (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). + +- A new command `nix store add` has been added. It replaces `nix store add-file` and `nix store add-path` which are now deprecated. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 422f1fce8..78ae99f4b 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,77 +1,2 @@ # Release X.Y (202?-??-??) -- The experimental nix command can now act as a [shebang interpreter](@docroot@/command-ref/new-cli/nix.md#shebang-interpreter) - by appending the contents of any `#! nix` lines and the script's location to a single call. - -- [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. - -- [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). - -- The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly. - -- Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.` or `legacyPackages.`. - -- Nix adds `apple-virt` to the default system features on macOS systems that support virtualization. This is similar to what's done for the `kvm` system feature on Linux hosts. - -- Introduce a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash). - -- `nix-shell` shebang lines now support single-quoted arguments. - -- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - As described in the document for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of Flakes. - -- The interface for creating and updating lock files has been overhauled: - - - [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now. - It will *never* update existing inputs. - - - [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) does the same, but *will* update inputs. - - Passing no arguments will update all inputs of the current flake, just like it already did. - - Passing input names as arguments will ensure only those are updated. This replaces the functionality of `nix flake lock --update-input` - - To operate on a flake outside the current directory, you must now pass `--flake path/to/flake`. - - - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. - They are superceded by `nix flake update`. - -- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). - -- [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) - (experimental) now returns a JSON map rather than JSON list. - The `path` field of each object has instead become the key in th outer map, since it is unique. - The `valid` field also goes away because we just use null instead. - - - Old way: - - ```json5 - [ - { - "path": "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15", - "valid": true, - // ... - }, - { - "path": "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path", - "valid": false - } - ] - ``` - - - New way - - ```json5 - { - "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15": { - // ... - }, - "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path": null, - } - ``` - - This makes it match `nix derivation show`, which also maps store paths to information. - -- When Nix is installed using the [binary installer](@docroot@/installation/installing-binary.md), in supported shells (Bash, Zsh, Fish) - [`XDG_DATA_DIRS`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables) is now populated with the path to the `/share` subdirectory of the current profile. - This means that command completion scripts, `.desktop` files, and similar artifacts installed via [`nix-env`](@docroot@/command-ref/nix-env.md) or [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) - (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). - -- A new command `nix store add` has been added. It replaces `nix store add-file` and `nix store add-path` which are now deprecated. From 293ae592576bb9c48975466613fcba6a30d06f5e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 17 Nov 2023 11:26:45 -0500 Subject: [PATCH 630/909] Fix `make check` After 9c7749e13508996eb9df83b1692664cc8cdbf952, `libutil-tests_RUN` doesn't exist. It needs to become `libutil-tests-exe_RUN`. --- src/libutil/tests/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index e6fc4e364..66886c45f 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -1,4 +1,4 @@ -check: libutil-tests_RUN +check: libutil-tests-exe_RUN programs += libutil-tests-exe From 4a539ac3eac90b2c2f839cae885df89a03240348 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 17 Nov 2023 17:38:08 +0100 Subject: [PATCH 631/909] Fix buildNoGc Fixes https://hydra.nixos.org/build/241067941/nixlog/1 src/libexpr/eval.cc:1776:54: error: variable 'boost::container::small_vector vArgs' has initializer but incomplete type --- src/libexpr/eval.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 8b0ada517..e9b8cacfd 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -31,6 +31,7 @@ #include #include +#include #if HAVE_BOEHMGC @@ -42,7 +43,6 @@ #include #include #include -#include #endif From 251fb23aeab8f85afb4f8376c2e6fc3d8b229d23 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 01:46:48 +0100 Subject: [PATCH 632/909] Shebang parser: add virtual destructor Fixes: warning: destructor called on non-final 'nix::ParseUnquoted' that has virtual functions but non-virtual destructor [-Wdelete-non-abstract-non-virtual-dtor] --- src/libutil/args.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 4359c5e8e..4480a03f5 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -97,6 +97,8 @@ struct Parser { virtual void operator()(std::shared_ptr & state, Strings & r) = 0; Parser(std::string_view s) : remaining(s) {}; + + virtual ~Parser() { }; }; struct ParseQuoted : public Parser { From 70ddf298e0882075dcb1cf69562c629a195718f7 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Sun, 19 Nov 2023 04:09:14 +0100 Subject: [PATCH 633/909] doc: Add link to filterSource from path --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e274c3c0c..dda409955 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2374,7 +2374,7 @@ static RegisterPrimOp primop_path({ like `@`. - filter\ - A function of the type expected by `builtins.filterSource`, + A function of the type expected by [`builtins.filterSource`](#builtins-filterSource), with the same semantics. - recursive\ From fe4f573d49a5c47cf9ffd0bd3fe8868104550818 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 11:38:47 +0100 Subject: [PATCH 634/909] flake.nix: Update nixpkgs: release-23.05 -> nixos-23.05-small MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/9eb24edd6a0027fed010ccfe300a9734d029983c' (2023-11-01) → 'github:NixOS/nixpkgs/decdf666c833a325cb4417041a90681499e06a41' (2023-11-18) --- flake.lock | 8 ++++---- flake.nix | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/flake.lock b/flake.lock index 991cef1ee..825166717 100644 --- a/flake.lock +++ b/flake.lock @@ -34,16 +34,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1698876495, - "narHash": "sha256-nsQo2/mkDUFeAjuu92p0dEqhRvHHiENhkKVIV1y0/Oo=", + "lastModified": 1700342017, + "narHash": "sha256-HaibwlWH5LuqsaibW3sIVjZQtEM/jWtOHX4Nk93abGE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "9eb24edd6a0027fed010ccfe300a9734d029983c", + "rev": "decdf666c833a325cb4417041a90681499e06a41", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-23.05", + "ref": "nixos-23.05-small", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 05ab7b06d..f21b1a63f 100644 --- a/flake.nix +++ b/flake.nix @@ -1,9 +1,7 @@ { description = "The purely functional package manager"; - # FIXME go back to nixos-23.05-small once - # https://github.com/NixOS/nixpkgs/pull/264875 is included. - inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.05"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; From 796a7eb92d2b0caf75685126adc7460a4c39cfec Mon Sep 17 00:00:00 2001 From: DavHau Date: Sun, 19 Nov 2023 20:32:23 +0700 Subject: [PATCH 635/909] fetchTree: clarify docs for shallow flag --- src/libexpr/primops/fetchTree.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 8031bf809..383ec7c58 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -425,7 +425,8 @@ static RegisterPrimOp primop_fetchGit({ - `shallow` (default: `false`) - A Boolean parameter that specifies whether fetching a shallow clone is allowed. + A Boolean parameter that specifies whether fetching from a shallow remote repository is allowed. + This still performs a full clone of what is available on the remote. - `allRefs` From d5928085d5a542b19cc21b1f299392ee7a0c960b Mon Sep 17 00:00:00 2001 From: nicoo Date: Sun, 19 Nov 2023 19:57:07 +0100 Subject: [PATCH 636/909] builtins.concatMap: Fix typo in error message --- src/libexpr/primops.cc | 2 +- src/libexpr/tests/error_traces.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index dda409955..27f502830 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3459,7 +3459,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args, for (unsigned int n = 0; n < nrLists; ++n) { Value * vElem = args[1]->listElems()[n]; state.callFunction(*args[0], *vElem, lists[n], pos); - state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)), "while evaluating the return value of the function passed to buitlins.concatMap"); + state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)), "while evaluating the return value of the function passed to builtins.concatMap"); len += lists[n].listSize(); } diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc index 139366bcd..81498f65a 100644 --- a/src/libexpr/tests/error_traces.cc +++ b/src/libexpr/tests/error_traces.cc @@ -906,12 +906,12 @@ namespace nix { ASSERT_TRACE2("concatMap (x: 1) [ \"foo\" ] # TODO", TypeError, hintfmt("value is %s while a list was expected", "an integer"), - hintfmt("while evaluating the return value of the function passed to buitlins.concatMap")); + hintfmt("while evaluating the return value of the function passed to builtins.concatMap")); ASSERT_TRACE2("concatMap (x: \"foo\") [ 1 2 ] # TODO", TypeError, hintfmt("value is %s while a list was expected", "a string"), - hintfmt("while evaluating the return value of the function passed to buitlins.concatMap")); + hintfmt("while evaluating the return value of the function passed to builtins.concatMap")); } From 19993398a12069a868b0fb10b63f7d06f0f993e6 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Mon, 20 Nov 2023 03:37:02 -0700 Subject: [PATCH 637/909] flakes: check for flake.nix before complaining that lstat on it fails getFlake currently calls lstat (via isLink via canonPath) before it performs the sanity check that a flake.nix exists in the first place. This commit moves the check to before path canonicalization, so that failed symlink check operations don't throw before the check does. --- src/libexpr/flake/flake.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 54de53e0b..b128de31e 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -212,8 +212,16 @@ static Flake getFlake( auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( state, originalRef, allowLookup, flakeCache); + // We need to guard against symlink attacks, but before we start doing + // filesystem operations we should make sure there's a flake.nix in the + // first place. + auto unsafeFlakeDir = state.store->toRealPath(storePath) + "/" + lockedRef.subdir; + auto unsafeFlakeFile = unsafeFlakeDir + "/flake.nix"; + if (!pathExists(unsafeFlakeFile)) + throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); + // Guard against symlink attacks. - auto flakeDir = canonPath(state.store->toRealPath(storePath) + "/" + lockedRef.subdir, true); + auto flakeDir = canonPath(unsafeFlakeDir, true); auto flakeFile = canonPath(flakeDir + "/flake.nix", true); if (!isInDir(flakeFile, state.store->toRealPath(storePath))) throw Error("'flake.nix' file of flake '%s' escapes from '%s'", @@ -226,9 +234,6 @@ static Flake getFlake( .storePath = storePath, }; - if (!pathExists(flakeFile)) - throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); - Value vInfo; state.evalFile(state.rootPath(CanonPath(flakeFile)), vInfo, true); // FIXME: symlink attack From 1d6abec993a371091459d5e23f985c6d69621ce7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Nov 2023 12:35:35 +0100 Subject: [PATCH 638/909] Revert use of boost::container::small_vector in the evaluator It caused random crashes (https://hydra.nixos.org/build/241514506, https://hydra.nixos.org/build/241443330) because the heap allocation done by small_vector in the not-small case is not scanned for GC roots. --- src/libexpr/eval.cc | 11 +++++------ src/libexpr/primops.cc | 7 ++++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e9b8cacfd..46a49c891 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -31,7 +31,6 @@ #include #include -#include #if HAVE_BOEHMGC @@ -1710,7 +1709,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & /* We have all the arguments, so call the primop with the previous and new arguments. */ - Value * vArgs[maxPrimOpArity]; + Value * vArgs[arity]; auto n = argsDone; for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left) vArgs[--n] = arg->primOpApp.right; @@ -1773,11 +1772,11 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v) // 4: about 60 // 5: under 10 // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total. - boost::container::small_vector vArgs(args.size()); + Value * vArgs[args.size()]; for (size_t i = 0; i < args.size(); ++i) vArgs[i] = args[i]->maybeThunk(state, env); - state.callFunction(vFun, args.size(), vArgs.data(), v, pos); + state.callFunction(vFun, args.size(), vArgs, v, pos); } @@ -2016,8 +2015,8 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) return result; }; - boost::container::small_vector values(es->size()); - Value * vTmpP = values.data(); + Value values[es->size()]; + Value * vTmpP = values; for (auto & [i_pos, i] : *es) { Value & vTmp = *vTmpP++; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 27f502830..a8d44d8b7 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2729,7 +2729,7 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs")); state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs"); - boost::container::small_vector res(args[1]->listSize()); + Value * res[args[1]->listSize()]; size_t found = 0; for (auto v2 : args[1]->listItems()) { @@ -3064,7 +3064,8 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter"); - boost::container::small_vector vs(args[1]->listSize()); + // FIXME: putting this on the stack is risky. + Value * vs[args[1]->listSize()]; size_t k = 0; bool same = true; @@ -3453,7 +3454,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args, state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap"); auto nrLists = args[1]->listSize(); - boost::container::small_vector lists(nrLists); + Value lists[nrLists]; size_t len = 0; for (unsigned int n = 0; n < nrLists; ++n) { From 7ac39ff05c8353c665174e8df61dd76a2b0b93db Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Nov 2023 13:11:58 +0100 Subject: [PATCH 639/909] refactor Store::buildPaths: convert to string earlier Preparation for RFC 92 dynamic derivations. --- src/libstore/build/entry-points.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 13ff22f45..74eca63f3 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -15,7 +15,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod worker.run(goals); - StorePathSet failed; + StringSet failed; std::optional ex; for (auto & i : goals) { if (i->ex) { @@ -26,9 +26,9 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod } if (i->exitCode != Goal::ecSuccess) { if (auto i2 = dynamic_cast(i.get())) - failed.insert(i2->drvPath); + failed.insert(std::string { i2->drvPath.to_string() }); else if (auto i2 = dynamic_cast(i.get())) - failed.insert(i2->storePath); + failed.insert(std::string { i2->storePath.to_string()}); } } @@ -37,7 +37,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); - throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed)); + throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed))); } } From a5e51a9e02efe2813170fdf0093c98b3d56aed84 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Nov 2023 13:18:25 +0100 Subject: [PATCH 640/909] refactor Worker::childStarted/Terminated: use switch Preparation for RFC 92 dynamic derivations. --- src/libstore/build/worker.cc | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 01914e2d6..01f52e7ab 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -199,8 +199,16 @@ void Worker::childStarted(GoalPtr goal, const std::set & fds, child.respectTimeouts = respectTimeouts; children.emplace_back(child); if (inBuildSlot) { - if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++; - else nrLocalBuilds++; + switch (goal->jobCategory()) { + case JobCategory::Substitution: + nrSubstitutions++; + break; + case JobCategory::Build: + nrLocalBuilds++; + break; + default: + abort(); + } } } @@ -212,12 +220,17 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers) if (i == children.end()) return; if (i->inBuildSlot) { - if (goal->jobCategory() == JobCategory::Substitution) { + switch (goal->jobCategory()) { + case JobCategory::Substitution: assert(nrSubstitutions > 0); nrSubstitutions--; - } else { + break; + case JobCategory::Build: assert(nrLocalBuilds > 0); nrLocalBuilds--; + break; + default: + abort(); } } From 2a96445d7505cb0a82ed2a49c7210b3073ffd153 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Nov 2023 15:06:28 +0100 Subject: [PATCH 641/909] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index ef0f38abe..7329e21c3 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.19.0 +2.20.0 From e2b6821ca0147f36bcb9404aab080f80746984c8 Mon Sep 17 00:00:00 2001 From: roblabla Date: Mon, 20 Nov 2023 15:41:38 +0100 Subject: [PATCH 642/909] Fix bad_format_string error when builder stdout contains % --- src/libutil/serialise.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index d7950b11b..f465bd0de 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -448,7 +448,7 @@ Error readError(Source & source) auto msg = readString(source); ErrorInfo info { .level = level, - .msg = hintformat(fmt("%s", msg)), + .msg = hintfmt(msg), }; auto havePos = readNum(source); assert(havePos == 0); @@ -457,7 +457,7 @@ Error readError(Source & source) havePos = readNum(source); assert(havePos == 0); info.traces.push_back(Trace { - .hint = hintformat(fmt("%s", readString(source))) + .hint = hintfmt(readString(source)) }); } return Error(std::move(info)); From e4066c04442f86a5a12d492d588e3e82b533053d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 15 Nov 2023 16:15:47 +0100 Subject: [PATCH 643/909] Fetch specific Git revisions This is more efficient, and necessary when using shallow=1 with a rev. --- src/libfetchers/git.cc | 5 ++++- tests/functional/fetchGit.sh | 4 +--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7208a0b6d..2fd3fb41e 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -518,8 +518,11 @@ struct GitInputScheme : InputScheme if (doFetch) { try { - auto fetchRef = getAllRefsAttr(input) + auto fetchRef = + getAllRefsAttr(input) ? "refs/*" + : input.getRev() + ? input.getRev()->gitRev() : ref.compare(0, 5, "refs/") == 0 ? ref : ref == "HEAD" diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index c38cd27eb..4985c7764 100644 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -51,9 +51,7 @@ git -C $repo add differentbranch git -C $repo commit -m 'Test2' git -C $repo checkout master devrev=$(git -C $repo rev-parse devtest) -out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; }" 2>&1) || status=$? -[[ $status == 1 ]] -[[ $out =~ 'Cannot find Git revision' ]] +nix eval --impure --raw --expr "builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; }" [[ $(nix eval --raw --expr "builtins.readFile (builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; allRefs = true; } + \"/differentbranch\")") = 'different file' ]] From 071f14a0bb25ffa8e5aaf8ad37031d205f49ef7d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 15 Nov 2023 17:17:07 +0100 Subject: [PATCH 644/909] Don't do shallow fetches over ssh --- src/libfetchers/git-utils.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index f554dcc5f..19eae0e1d 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -381,7 +381,9 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this }; git_fetch_options opts = GIT_FETCH_OPTIONS_INIT; - opts.depth = shallow ? 1 : GIT_FETCH_DEPTH_FULL; + // FIXME: for some reason, shallow fetching over ssh barfs + // with "could not read from remote repository". + opts.depth = shallow && parseURL(url).scheme != "ssh" ? 1 : GIT_FETCH_DEPTH_FULL; opts.callbacks.payload = &act; opts.callbacks.sideband_progress = sidebandProgressCallback; opts.callbacks.transfer_progress = transferProgressCallback; From a0162d5732b23e7fdc1f65df28826611e3a424e5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Nov 2023 18:54:36 +0100 Subject: [PATCH 645/909] Improve SourceAccessor path display MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backported from lazy-trees. This allows SourceAccessors to show the origin of the accessor. E.g. we now get copying '«git+https://github.com/blender/blender.git?ref=refs/heads/main&rev=4edc1389337dd3679ff66969c332d2aff52e1992»/' to the store instead of copying '/' to the store --- src/libexpr/eval.cc | 3 +++ src/libfetchers/fs-input-accessor.cc | 1 + src/libfetchers/git.cc | 2 ++ src/libutil/posix-source-accessor.hh | 2 +- src/libutil/source-accessor.cc | 9 ++++++++- src/libutil/source-accessor.hh | 4 ++++ 6 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 46a49c891..bf6b6f8c1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -532,6 +532,9 @@ EvalState::EvalState( , baseEnv(allocEnv(128)) , staticBaseEnv{std::make_shared(false, nullptr)} { + corepkgsFS->setPathDisplay(""); + internalFS->setPathDisplay("«nix-internal»", ""); + countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; assert(gcInitialised); diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index 81be64482..2efee932d 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -18,6 +18,7 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor , allowedPaths(std::move(allowedPaths)) , makeNotAllowedError(std::move(makeNotAllowedError)) { + displayPrefix = root.isRoot() ? "" : root.abs(); } void readFile( diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7208a0b6d..6bca87304 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -582,6 +582,8 @@ struct GitInputScheme : InputScheme auto accessor = repo->getAccessor(rev); + accessor->setPathDisplay("«" + input.to_string() + "»"); + /* If the repo has submodules, fetch them and return a mounted input accessor consisting of the accessor for the top-level repo and the accessors for the submodules. */ diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh index cf087d26e..a45d96bf8 100644 --- a/src/libutil/posix-source-accessor.hh +++ b/src/libutil/posix-source-accessor.hh @@ -7,7 +7,7 @@ namespace nix { /** * A source accessor that uses the Unix filesystem. */ -struct PosixSourceAccessor : SourceAccessor +struct PosixSourceAccessor : virtual SourceAccessor { /** * The most recent mtime seen by lstat(). This is a hack to diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index e2114e18f..7813433a7 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -7,6 +7,7 @@ static std::atomic nextNumber{0}; SourceAccessor::SourceAccessor() : number(++nextNumber) + , displayPrefix{"«unknown»"} { } @@ -55,9 +56,15 @@ SourceAccessor::Stat SourceAccessor::lstat(const CanonPath & path) throw Error("path '%s' does not exist", showPath(path)); } +void SourceAccessor::setPathDisplay(std::string displayPrefix, std::string displaySuffix) +{ + this->displayPrefix = std::move(displayPrefix); + this->displaySuffix = std::move(displaySuffix); +} + std::string SourceAccessor::showPath(const CanonPath & path) { - return path.abs(); + return displayPrefix + path.abs() + displaySuffix; } } diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index 1a4e80361..264caab16 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -17,6 +17,8 @@ struct SourceAccessor { const size_t number; + std::string displayPrefix, displaySuffix; + SourceAccessor(); virtual ~SourceAccessor() @@ -117,6 +119,8 @@ struct SourceAccessor return number < x.number; } + void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); + virtual std::string showPath(const CanonPath & path); }; From 99d5204baaef211234d50f20610fa43d304888ce Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Nov 2023 20:04:37 +0100 Subject: [PATCH 646/909] Persistently cache InputAccessor::fetchToStore() This avoids repeated copying of the same source tree between Nix invocations. It requires the accessor to have a "fingerprint" (e.g. a Git revision) that uniquely determines its contents. --- src/libfetchers/fetchers.cc | 5 +++++ src/libfetchers/fetchers.hh | 9 +++++++++ src/libfetchers/git.cc | 14 +++++++++++++- src/libfetchers/github.cc | 8 ++++++++ src/libfetchers/input-accessor.cc | 30 ++++++++++++++++++++++++++++++ src/libfetchers/input-accessor.hh | 2 ++ src/libfetchers/mercurial.cc | 8 ++++++++ src/libstore/content-address.hh | 4 ++-- 8 files changed, 77 insertions(+), 3 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index c2513e076..60208619e 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -108,6 +108,11 @@ Input Input::fromAttrs(Attrs && attrs) return std::move(*res); } +std::optional Input::getFingerprint(ref store) const +{ + return scheme ? scheme->getFingerprint(store, *this) : std::nullopt; +} + ParsedURL Input::toURL() const { if (!scheme) diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index ce5aa4c69..5f3254b6d 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -113,6 +113,12 @@ public: std::optional getRev() const; std::optional getRevCount() const; std::optional getLastModified() const; + + /** + * For locked inputs, return a string that uniquely specifies the + * content of the input (typically a commit hash or content hash). + */ + std::optional getFingerprint(ref store) const; }; @@ -180,6 +186,9 @@ struct InputScheme virtual bool isDirect(const Input & input) const { return true; } + + virtual std::optional getFingerprint(ref store, const Input & input) const + { return std::nullopt; } }; void registerInputScheme(std::shared_ptr && fetcher); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7208a0b6d..6b461499b 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -700,10 +700,22 @@ struct GitInputScheme : InputScheme auto repoInfo = getRepoInfo(input); - return + auto [accessor, final] = input.getRef() || input.getRev() || !repoInfo.isLocal ? getAccessorFromCommit(store, repoInfo, std::move(input)) : getAccessorFromWorkdir(store, repoInfo, std::move(input)); + + accessor->fingerprint = final.getFingerprint(store); + + return {accessor, std::move(final)}; + } + + std::optional getFingerprint(ref store, const Input & input) const override + { + if (auto rev = input.getRev()) + return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : ""); + else + return std::nullopt; } }; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 6c9b29721..661ad4884 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -229,6 +229,14 @@ struct GitArchiveInputScheme : InputScheme { return Xp::Flakes; } + + std::optional getFingerprint(ref store, const Input & input) const override + { + if (auto rev = input.getRev()) + return rev->gitRev(); + else + return std::nullopt; + } }; struct GitHubInputScheme : GitArchiveInputScheme diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index d1d450cf7..53502c621 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -1,5 +1,6 @@ #include "input-accessor.hh" #include "store-api.hh" +#include "cache.hh" namespace nix { @@ -11,6 +12,30 @@ StorePath InputAccessor::fetchToStore( PathFilter * filter, RepairFlag repair) { + // FIXME: add an optimisation for the case where the accessor is + // an FSInputAccessor pointing to a store path. + + std::optional cacheKey; + + if (!filter && fingerprint) { + cacheKey = fetchers::Attrs{ + {"_what", "fetchToStore"}, + {"store", store->storeDir}, + {"name", std::string(name)}, + {"fingerprint", *fingerprint}, + {"method", (uint8_t) method}, + {"path", path.abs()} + }; + if (auto res = fetchers::getCache()->lookup(*cacheKey)) { + StorePath storePath(fetchers::getStrAttr(*res, "storePath")); + if (store->isValidPath(storePath)) { + debug("store path cache hit for '%s'", showPath(path)); + return storePath; + } + } + } else + debug("source path '%s' is uncacheable", showPath(path)); + Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path))); auto source = sinkToSource([&](Sink & sink) { @@ -25,6 +50,11 @@ StorePath InputAccessor::fetchToStore( ? store->computeStorePathFromDump(*source, name, method, htSHA256).first : store->addToStoreFromDump(*source, name, method, htSHA256, repair); + if (cacheKey) + fetchers::getCache()->upsert( + *cacheKey, + fetchers::Attrs{{"storePath", std::string(storePath.to_string())}}); + return storePath; } diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 9c688a234..26d17f064 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -18,6 +18,8 @@ class Store; struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this { + std::optional fingerprint; + /** * Return the maximum last-modified time of the files in this * tree, if available. diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 9244acf39..aa991a75d 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -339,6 +339,14 @@ struct MercurialInputScheme : InputScheme return makeResult(infoAttrs, std::move(storePath)); } + + std::optional getFingerprint(ref store, const Input & input) const override + { + if (auto rev = input.getRev()) + return rev->gitRev(); + else + return std::nullopt; + } }; static auto rMercurialInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index c4d619bdc..bdb558907 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -39,12 +39,12 @@ enum struct FileIngestionMethod : uint8_t { /** * Flat-file hashing. Directly ingest the contents of a single file */ - Flat = false, + Flat = 0, /** * Recursive (or NAR) hashing. Serializes the file-system object in Nix * Archive format and ingest that */ - Recursive = true + Recursive = 1 }; /** From 64827360be35a3d16e818aa9d8426ca40b2c4dc2 Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Tue, 21 Nov 2023 14:49:48 +0100 Subject: [PATCH 647/909] Fix "unbound variable" errors in bash Fixes #9414 --- scripts/nix-profile-daemon.sh.in | 2 +- scripts/nix-profile.sh.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index c63db4648..d256b24ed 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -31,7 +31,7 @@ fi export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" # Populate bash completions, .desktop files, etc -if [ -z "$XDG_DATA_DIRS" ]; then +if [ -z "${XDG_DATA_DIRS-}" ]; then # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" else diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 56e070ae1..44bc96e89 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -33,7 +33,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" # Populate bash completions, .desktop files, etc - if [ -z "$XDG_DATA_DIRS" ]; then + if [ -z "${XDG_DATA_DIRS-}" ]; then # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" else From 4d8decbd135a78389c463fbb4c844f1bf22aed69 Mon Sep 17 00:00:00 2001 From: Lin Jian Date: Tue, 21 Nov 2023 15:29:36 +0800 Subject: [PATCH 648/909] doc: fix number of template attributes This number is not updated when welcomeText is added[1][2]. [1]: f3a2940e70dea2c35dcae3fca019e94bf8758b4d [2]: https://github.com/NixOS/nix/pull/6103 --- src/nix/flake-init.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/flake-init.md b/src/nix/flake-init.md index fc1f4f805..ea274bf29 100644 --- a/src/nix/flake-init.md +++ b/src/nix/flake-init.md @@ -30,7 +30,7 @@ using `-t`. # Template definitions A flake can declare templates through its `templates` output -attribute. A template has two attributes: +attribute. A template has the following attributes: * `description`: A one-line description of the template, in CommonMark syntax. From f880469173061a07f0b2a24734932c5a9ad633c6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 19 Nov 2023 10:17:57 -0500 Subject: [PATCH 649/909] Put `canonicaliseTimestampAndPermissions` in its own header/file It is not inherently tied to `LocalStore`, it could probably even go in `libnixutil`. Functions not attached to `LocalStore` should not be declared in `local-store.hh`. I am moving it to facilitate experimenting for #9344. If canonicalisation should be done client-side in client-side builds, there wouldn't be a `LocalStore` at all so having to include that header to get this freestanding function is cumbersome and wrong. Perhaps canonicalisation should still be done server-side for security reasons --- I don't mean to make that judgement call now --- but even if so, this freestanding function still isn't connected to `LocalStore` so while less urgent it is still better to move out of this header. --- src/libstore/build/local-derivation-goal.cc | 1 + src/libstore/local-store.cc | 159 +----------------- src/libstore/local-store.hh | 34 ---- src/libstore/optimise-store.cc | 1 + src/libstore/posix-fs-canonicalise.cc | 169 ++++++++++++++++++++ src/libstore/posix-fs-canonicalise.hh | 45 ++++++ src/nix-store/nix-store.cc | 1 + 7 files changed, 218 insertions(+), 192 deletions(-) create mode 100644 src/libstore/posix-fs-canonicalise.cc create mode 100644 src/libstore/posix-fs-canonicalise.hh diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index a9f930773..198402ff7 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -19,6 +19,7 @@ #include "namespaces.hh" #include "child.hh" #include "unix-domain-socket.hh" +#include "posix-fs-canonicalise.hh" #include #include diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2a3582ad8..4ff75f528 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -11,6 +11,7 @@ #include "finally.hh" #include "compression.hh" #include "signals.hh" +#include "posix-fs-canonicalise.hh" #include #include @@ -581,164 +582,6 @@ void LocalStore::makeStoreWritable() } -const time_t mtimeStore = 1; /* 1 second into the epoch */ - - -static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st) -{ - if (!S_ISLNK(st.st_mode)) { - - /* Mask out all type related bits. */ - mode_t mode = st.st_mode & ~S_IFMT; - - if (mode != 0444 && mode != 0555) { - mode = (st.st_mode & S_IFMT) - | 0444 - | (st.st_mode & S_IXUSR ? 0111 : 0); - if (chmod(path.c_str(), mode) == -1) - throw SysError("changing mode of '%1%' to %2$o", path, mode); - } - - } - - if (st.st_mtime != mtimeStore) { - struct timeval times[2]; - times[0].tv_sec = st.st_atime; - times[0].tv_usec = 0; - times[1].tv_sec = mtimeStore; - times[1].tv_usec = 0; -#if HAVE_LUTIMES - if (lutimes(path.c_str(), times) == -1) - if (errno != ENOSYS || - (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) -#else - if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) -#endif - throw SysError("changing modification time of '%1%'", path); - } -} - - -void canonicaliseTimestampAndPermissions(const Path & path) -{ - canonicaliseTimestampAndPermissions(path, lstat(path)); -} - - -static void canonicalisePathMetaData_( - const Path & path, - std::optional> uidRange, - InodesSeen & inodesSeen) -{ - checkInterrupt(); - -#if __APPLE__ - /* Remove flags, in particular UF_IMMUTABLE which would prevent - the file from being garbage-collected. FIXME: Use - setattrlist() to remove other attributes as well. */ - if (lchflags(path.c_str(), 0)) { - if (errno != ENOTSUP) - throw SysError("clearing flags of path '%1%'", path); - } -#endif - - auto st = lstat(path); - - /* Really make sure that the path is of a supported type. */ - if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) - throw Error("file '%1%' has an unsupported type", path); - -#if __linux__ - /* Remove extended attributes / ACLs. */ - ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); - - if (eaSize < 0) { - if (errno != ENOTSUP && errno != ENODATA) - throw SysError("querying extended attributes of '%s'", path); - } else if (eaSize > 0) { - std::vector eaBuf(eaSize); - - if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) - throw SysError("querying extended attributes of '%s'", path); - - for (auto & eaName: tokenizeString(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { - if (settings.ignoredAcls.get().count(eaName)) continue; - if (lremovexattr(path.c_str(), eaName.c_str()) == -1) - throw SysError("removing extended attribute '%s' from '%s'", eaName, path); - } - } -#endif - - /* Fail if the file is not owned by the build user. This prevents - us from messing up the ownership/permissions of files - hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). - However, ignore files that we chown'ed ourselves previously to - ensure that we don't fail on hard links within the same build - (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ - if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) { - if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino))) - throw BuildError("invalid ownership on file '%1%'", path); - mode_t mode = st.st_mode & ~S_IFMT; - assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); - return; - } - - inodesSeen.insert(Inode(st.st_dev, st.st_ino)); - - canonicaliseTimestampAndPermissions(path, st); - - /* Change ownership to the current uid. If it's a symlink, use - lchown if available, otherwise don't bother. Wrong ownership - of a symlink doesn't matter, since the owning user can't change - the symlink and can't delete it because the directory is not - writable. The only exception is top-level paths in the Nix - store (since that directory is group-writable for the Nix build - users group); we check for this case below. */ - if (st.st_uid != geteuid()) { -#if HAVE_LCHOWN - if (lchown(path.c_str(), geteuid(), getegid()) == -1) -#else - if (!S_ISLNK(st.st_mode) && - chown(path.c_str(), geteuid(), getegid()) == -1) -#endif - throw SysError("changing owner of '%1%' to %2%", - path, geteuid()); - } - - if (S_ISDIR(st.st_mode)) { - DirEntries entries = readDirectory(path); - for (auto & i : entries) - canonicalisePathMetaData_(path + "/" + i.name, uidRange, inodesSeen); - } -} - - -void canonicalisePathMetaData( - const Path & path, - std::optional> uidRange, - InodesSeen & inodesSeen) -{ - canonicalisePathMetaData_(path, uidRange, inodesSeen); - - /* On platforms that don't have lchown(), the top-level path can't - be a symlink, since we can't change its ownership. */ - auto st = lstat(path); - - if (st.st_uid != geteuid()) { - assert(S_ISLNK(st.st_mode)); - throw Error("wrong ownership of top-level store path '%1%'", path); - } -} - - -void canonicalisePathMetaData(const Path & path, - std::optional> uidRange) -{ - InodesSeen inodesSeen; - canonicalisePathMetaData(path, uidRange, inodesSeen); -} - - void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) { experimentalFeatureSettings.require(Xp::CaDerivations); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 6d589bee5..8f0ffd2a2 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -371,38 +371,4 @@ private: friend struct DerivationGoal; }; - -typedef std::pair Inode; -typedef std::set InodesSeen; - - -/** - * "Fix", or canonicalise, the meta-data of the files in a store path - * after it has been built. In particular: - * - * - the last modification date on each file is set to 1 (i.e., - * 00:00:01 1/1/1970 UTC) - * - * - the permissions are set of 444 or 555 (i.e., read-only with or - * without execute permission; setuid bits etc. are cleared) - * - * - the owner and group are set to the Nix user and group, if we're - * running as root. - * - * If uidRange is not empty, this function will throw an error if it - * encounters files owned by a user outside of the closed interval - * [uidRange->first, uidRange->second]. - */ -void canonicalisePathMetaData( - const Path & path, - std::optional> uidRange, - InodesSeen & inodesSeen); -void canonicalisePathMetaData( - const Path & path, - std::optional> uidRange); - -void canonicaliseTimestampAndPermissions(const Path & path); - -MakeError(PathInUse, Error); - } diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index a4ac413b3..0fa977545 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -1,6 +1,7 @@ #include "local-store.hh" #include "globals.hh" #include "signals.hh" +#include "posix-fs-canonicalise.hh" #include #include diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc new file mode 100644 index 000000000..cc3ab0b74 --- /dev/null +++ b/src/libstore/posix-fs-canonicalise.cc @@ -0,0 +1,169 @@ +#include + +#include "posix-fs-canonicalise.hh" +#include "file-system.hh" +#include "signals.hh" +#include "util.hh" +#include "globals.hh" +#include "store-api.hh" + +namespace nix { + +const time_t mtimeStore = 1; /* 1 second into the epoch */ + + +static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st) +{ + if (!S_ISLNK(st.st_mode)) { + + /* Mask out all type related bits. */ + mode_t mode = st.st_mode & ~S_IFMT; + + if (mode != 0444 && mode != 0555) { + mode = (st.st_mode & S_IFMT) + | 0444 + | (st.st_mode & S_IXUSR ? 0111 : 0); + if (chmod(path.c_str(), mode) == -1) + throw SysError("changing mode of '%1%' to %2$o", path, mode); + } + + } + + if (st.st_mtime != mtimeStore) { + struct timeval times[2]; + times[0].tv_sec = st.st_atime; + times[0].tv_usec = 0; + times[1].tv_sec = mtimeStore; + times[1].tv_usec = 0; +#if HAVE_LUTIMES + if (lutimes(path.c_str(), times) == -1) + if (errno != ENOSYS || + (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) +#else + if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) +#endif + throw SysError("changing modification time of '%1%'", path); + } +} + + +void canonicaliseTimestampAndPermissions(const Path & path) +{ + canonicaliseTimestampAndPermissions(path, lstat(path)); +} + + +static void canonicalisePathMetaData_( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen) +{ + checkInterrupt(); + +#if __APPLE__ + /* Remove flags, in particular UF_IMMUTABLE which would prevent + the file from being garbage-collected. FIXME: Use + setattrlist() to remove other attributes as well. */ + if (lchflags(path.c_str(), 0)) { + if (errno != ENOTSUP) + throw SysError("clearing flags of path '%1%'", path); + } +#endif + + auto st = lstat(path); + + /* Really make sure that the path is of a supported type. */ + if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) + throw Error("file '%1%' has an unsupported type", path); + +#if __linux__ + /* Remove extended attributes / ACLs. */ + ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); + + if (eaSize < 0) { + if (errno != ENOTSUP && errno != ENODATA) + throw SysError("querying extended attributes of '%s'", path); + } else if (eaSize > 0) { + std::vector eaBuf(eaSize); + + if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) + throw SysError("querying extended attributes of '%s'", path); + + for (auto & eaName: tokenizeString(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { + if (settings.ignoredAcls.get().count(eaName)) continue; + if (lremovexattr(path.c_str(), eaName.c_str()) == -1) + throw SysError("removing extended attribute '%s' from '%s'", eaName, path); + } + } +#endif + + /* Fail if the file is not owned by the build user. This prevents + us from messing up the ownership/permissions of files + hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). + However, ignore files that we chown'ed ourselves previously to + ensure that we don't fail on hard links within the same build + (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ + if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) { + if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino))) + throw BuildError("invalid ownership on file '%1%'", path); + mode_t mode = st.st_mode & ~S_IFMT; + assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); + return; + } + + inodesSeen.insert(Inode(st.st_dev, st.st_ino)); + + canonicaliseTimestampAndPermissions(path, st); + + /* Change ownership to the current uid. If it's a symlink, use + lchown if available, otherwise don't bother. Wrong ownership + of a symlink doesn't matter, since the owning user can't change + the symlink and can't delete it because the directory is not + writable. The only exception is top-level paths in the Nix + store (since that directory is group-writable for the Nix build + users group); we check for this case below. */ + if (st.st_uid != geteuid()) { +#if HAVE_LCHOWN + if (lchown(path.c_str(), geteuid(), getegid()) == -1) +#else + if (!S_ISLNK(st.st_mode) && + chown(path.c_str(), geteuid(), getegid()) == -1) +#endif + throw SysError("changing owner of '%1%' to %2%", + path, geteuid()); + } + + if (S_ISDIR(st.st_mode)) { + DirEntries entries = readDirectory(path); + for (auto & i : entries) + canonicalisePathMetaData_(path + "/" + i.name, uidRange, inodesSeen); + } +} + + +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen) +{ + canonicalisePathMetaData_(path, uidRange, inodesSeen); + + /* On platforms that don't have lchown(), the top-level path can't + be a symlink, since we can't change its ownership. */ + auto st = lstat(path); + + if (st.st_uid != geteuid()) { + assert(S_ISLNK(st.st_mode)); + throw Error("wrong ownership of top-level store path '%1%'", path); + } +} + + +void canonicalisePathMetaData(const Path & path, + std::optional> uidRange) +{ + InodesSeen inodesSeen; + canonicalisePathMetaData(path, uidRange, inodesSeen); +} + +} diff --git a/src/libstore/posix-fs-canonicalise.hh b/src/libstore/posix-fs-canonicalise.hh new file mode 100644 index 000000000..35644af12 --- /dev/null +++ b/src/libstore/posix-fs-canonicalise.hh @@ -0,0 +1,45 @@ +#pragma once +///@file + +#include +#include + +#include "types.hh" +#include "error.hh" + +namespace nix { + +typedef std::pair Inode; +typedef std::set InodesSeen; + + +/** + * "Fix", or canonicalise, the meta-data of the files in a store path + * after it has been built. In particular: + * + * - the last modification date on each file is set to 1 (i.e., + * 00:00:01 1/1/1970 UTC) + * + * - the permissions are set of 444 or 555 (i.e., read-only with or + * without execute permission; setuid bits etc. are cleared) + * + * - the owner and group are set to the Nix user and group, if we're + * running as root. + * + * If uidRange is not empty, this function will throw an error if it + * encounters files owned by a user outside of the closed interval + * [uidRange->first, uidRange->second]. + */ +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen); +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange); + +void canonicaliseTimestampAndPermissions(const Path & path); + +MakeError(PathInUse, Error); + +} diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 123283dfe..25f0107bc 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -14,6 +14,7 @@ #include "graphml.hh" #include "legacy.hh" #include "path-with-outputs.hh" +#include "posix-fs-canonicalise.hh" #include #include From 949f5841f8a8611d0f49793bd8c4963462d62e3a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 21 Nov 2023 13:12:55 -0500 Subject: [PATCH 650/909] Add the `MountedSSHStore` experimental feature It will be implemented in the subsequent commits of this PR. --- src/libutil/experimental-features.cc | 7 +++++++ src/libutil/experimental-features.hh | 1 + 2 files changed, 8 insertions(+) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index ac4d189e1..2418e3f4c 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -262,6 +262,13 @@ constexpr std::array xpFeatureDetails Allow the use of the [impure-env](@docroot@/command-ref/conf-file.md#conf-impure-env) setting. )", }, + { + .tag = Xp::MountedSSHStore, + .name = "mounted-ssh-store", + .description = R"( + Allow the use of the [`mounted SSH store`](@docroot@/command-ref/new-cli/nix3-help-stores.html#experimental-ssh-store-with-filesytem-mounted). + )", + }, { .tag = Xp::VerifiedFetches, .name = "verified-fetches", diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index c355b8081..eae4fa9b8 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -34,6 +34,7 @@ enum struct ExperimentalFeature ParseTomlTimestamps, ReadOnlyLocalStore, ConfigurableImpureEnv, + MountedSSHStore, VerifiedFetches, }; From 9796ebd7ef8c9a23ee8128273d925acae00e43b0 Mon Sep 17 00:00:00 2001 From: Matej Urbas Date: Mon, 17 Apr 2023 12:08:42 -0400 Subject: [PATCH 651/909] Add `--process-ops` flag to `nix-daemon` --- src/nix/daemon.cc | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 373dedf7c..4dada8e0e 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -443,16 +443,23 @@ static void processStdioConnection(ref store, TrustedFlag trustClient) * * @param forceTrustClientOpt See `daemonLoop()` and the parameter with * the same name over there for details. + * + * @param procesOps Whether to force processing ops even if the next + * store also is a remote store and could process it directly. */ -static void runDaemon(bool stdio, std::optional forceTrustClientOpt) +static void runDaemon(bool stdio, std::optional forceTrustClientOpt, bool processOps) { if (stdio) { auto store = openUncachedStore(); + std::shared_ptr remoteStore; + // If --force-untrusted is passed, we cannot forward the connection and // must process it ourselves (before delegating to the next store) to // force untrusting the client. - if (auto remoteStore = store.dynamic_pointer_cast(); remoteStore && (!forceTrustClientOpt || *forceTrustClientOpt != NotTrusted)) + processOps |= !forceTrustClientOpt || *forceTrustClientOpt != NotTrusted; + + if (!processOps && (remoteStore = store.dynamic_pointer_cast())) forwardStdioConnection(*remoteStore); else // `Trusted` is passed in the auto (no override case) because we @@ -468,6 +475,7 @@ static int main_nix_daemon(int argc, char * * argv) { auto stdio = false; std::optional isTrustedOpt = std::nullopt; + auto processOps = false; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--daemon") @@ -487,11 +495,14 @@ static int main_nix_daemon(int argc, char * * argv) } else if (*arg == "--default-trust") { experimentalFeatureSettings.require(Xp::DaemonTrustOverride); isTrustedOpt = std::nullopt; + } else if (*arg == "--process-ops") { + experimentalFeatureSettings.require(Xp::MountedSSHStore); + processOps = true; } else return false; return true; }); - runDaemon(stdio, isTrustedOpt); + runDaemon(stdio, isTrustedOpt, processOps); return 0; } @@ -503,6 +514,7 @@ struct CmdDaemon : StoreCommand { bool stdio = false; std::optional isTrustedOpt = std::nullopt; + bool processOps = false; CmdDaemon() { @@ -538,6 +550,19 @@ struct CmdDaemon : StoreCommand }}, .experimentalFeature = Xp::DaemonTrustOverride, }); + + addFlag({ + .longName = "process-ops", + .description = R"( + Forces the daemon to process received commands itself rather than forwarding the commands straight to the remote store. + + This is useful for the `mounted-ssh://` store where some actions need to be performed on the remote end but as connected user, and not as the user of the underlying daemon on the remote end. + )", + .handler = {[&]() { + processOps = true; + }}, + .experimentalFeature = Xp::MountedSSHStore, + }); } std::string description() override @@ -556,7 +581,7 @@ struct CmdDaemon : StoreCommand void run(ref store) override { - runDaemon(stdio, isTrustedOpt); + runDaemon(stdio, isTrustedOpt, processOps); } }; From 226b0f3956ef83ec51b6556d8f27e13a966b4ebf Mon Sep 17 00:00:00 2001 From: Matej Urbas Date: Mon, 17 Apr 2023 12:04:18 -0400 Subject: [PATCH 652/909] Extend the worker protocol with `wopAddPermRoot` --- src/libstore/daemon.cc | 15 +++++++++++++++ src/libstore/worker-protocol.hh | 3 ++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 105d92f25..be9b0b0d3 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -657,6 +657,21 @@ static void performOp(TunnelLogger * logger, ref store, break; } + case WorkerProto::Op::AddPermRoot: { + if (!trusted) + throw Error( + "you are not privileged to create perm roots\n\n" + "hint: you can just do this client-side without special privileges, and probably want to do that instead."); + auto storePath = WorkerProto::Serialise::read(*store, rconn); + Path gcRoot = absPath(readString(from)); + logger->startWork(); + auto & localFSStore = require(*store); + localFSStore.addPermRoot(storePath, gcRoot); + logger->stopWork(); + to << gcRoot; + break; + } + case WorkerProto::Op::AddIndirectRoot: { Path path = absPath(readString(from)); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 25d544ba7..8a26c09c5 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -9,7 +9,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 35) +#define PROTOCOL_VERSION (1 << 8 | 36) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -161,6 +161,7 @@ enum struct WorkerProto::Op : uint64_t AddMultipleToStore = 44, AddBuildLog = 45, BuildPathsWithResults = 46, + AddPermRoot = 47, }; /** From 06b8902562089811e5724aa0b8d719f891ab73f2 Mon Sep 17 00:00:00 2001 From: Matej Urbas Date: Sat, 15 Apr 2023 11:02:41 +0100 Subject: [PATCH 653/909] MountedSSHStore: stores on shared filesystems --- src/libstore/indirect-root-store.hh | 24 ++++++ src/libstore/mounted-ssh-store.md | 18 ++++ src/libstore/ssh-store.cc | 123 +++++++++++++++++++++++++++- 3 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 src/libstore/mounted-ssh-store.md diff --git a/src/libstore/indirect-root-store.hh b/src/libstore/indirect-root-store.hh index 59e45af45..c11679fe8 100644 --- a/src/libstore/indirect-root-store.hh +++ b/src/libstore/indirect-root-store.hh @@ -11,6 +11,30 @@ namespace nix { * reference. * * See methods for details on the operations it represents. + * + * @note + * To understand the purpose of this class, it might help to do some + * "closed-world" rather than "open-world" reasoning, and consider the + * problem it solved for us. This class was factored out from + * `LocalFSStore` in order to support the following table, which + * contains 4 concrete store types (non-abstract classes, exposed to the + * user), and how they implemented the two GC root methods: + * + * @note + * | | `addPermRoot()` | `addIndirectRoot()` | + * |-------------------|-----------------|---------------------| + * | `LocalStore` | local | local | + * | `UDSRemoteStore` | local | remote | + * | `SSHStore` | doesn't have | doesn't have | + * | `MountedSSHStore` | remote | doesn't have | + * + * @note + * Note how only the local implementations of `addPermRoot()` need + * `addIndirectRoot()`; that is what this class enforces. Without it, + * and with `addPermRoot()` and `addIndirectRoot()` both `virtual`, we + * would accidentally be allowing for a combinatorial explosion of + * possible implementations many of which make no sense. Having this and + * that invariant enforced cuts down that space. */ struct IndirectRootStore : public virtual LocalFSStore { diff --git a/src/libstore/mounted-ssh-store.md b/src/libstore/mounted-ssh-store.md new file mode 100644 index 000000000..1ebfe3081 --- /dev/null +++ b/src/libstore/mounted-ssh-store.md @@ -0,0 +1,18 @@ +R"( + +**Store URL format**: `mounted-ssh-ng://[username@]hostname` + +Experimental store type that allows full access to a Nix store on a remote machine, +and additionally requires that store be mounted in the local file system. + +The mounting of that store is not managed by Nix, and must by managed manually. +It could be accomplished with SSHFS or NFS, for example. + +The local file system is used to optimize certain operations. +For example, rather than serializing Nix archives and sending over the Nix channel, +we can directly access the file system data via the mount-point. + +The local file system is also used to make certain operations possible that wouldn't otherwise be. +For example, persistent GC roots can be created if they reside on the same file system as the remote store: +the remote side will create the symlinks necessary to avoid race conditions. +)" diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 4a6aad449..d4c8ab5b2 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -3,9 +3,10 @@ #include "local-fs-store.hh" #include "remote-store.hh" #include "remote-store-connection.hh" -#include "remote-fs-accessor.hh" +#include "source-accessor.hh" #include "archive.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "pool.hh" #include "ssh.hh" @@ -78,6 +79,8 @@ protected: std::string host; + std::vector extraRemoteProgramArgs; + SSHMaster master; void setOptions(RemoteStore::Connection & conn) override @@ -91,6 +94,121 @@ protected: }; }; +struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig +{ + using SSHStoreConfig::SSHStoreConfig; + using LocalFSStoreConfig::LocalFSStoreConfig; + + MountedSSHStoreConfig(StringMap params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) + { + } + + const std::string name() override { return "Experimental SSH Store with filesytem mounted"; } + + std::string doc() override + { + return + #include "mounted-ssh-store.md" + ; + } + + std::optional experimentalFeature() const override + { + return ExperimentalFeature::MountedSSHStore; + } +}; + +/** + * The mounted ssh store assumes that filesystems on the remote host are + * shared with the local host. This means that the remote nix store is + * available locally and is therefore treated as a local filesystem + * store. + * + * MountedSSHStore is very similar to UDSRemoteStore --- ignoring the + * superficial differnce of SSH vs Unix domain sockets, they both are + * accessing remote stores, and they both assume the store will be + * mounted in the local filesystem. + * + * The difference lies in how they manage GC roots. See addPermRoot + * below for details. + */ +class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSHStore, public virtual LocalFSStore +{ +public: + + MountedSSHStore(const std::string & scheme, const std::string & host, const Params & params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) + , MountedSSHStoreConfig(params) + , Store(params) + , RemoteStore(params) + , SSHStore(scheme, host, params) + , LocalFSStore(params) + { + extraRemoteProgramArgs = { + "--process-ops", + }; + } + + static std::set uriSchemes() + { + return {"mounted-ssh-ng"}; + } + + std::string getUri() override + { + return *uriSchemes().begin() + "://" + host; + } + + void narFromPath(const StorePath & path, Sink & sink) override + { + return LocalFSStore::narFromPath(path, sink); + } + + ref getFSAccessor(bool requireValidPath) override + { + return LocalFSStore::getFSAccessor(requireValidPath); + } + + std::optional getBuildLogExact(const StorePath & path) override + { + return LocalFSStore::getBuildLogExact(path); + } + + /** + * This is the key difference from UDSRemoteStore: UDSRemote store + * has the client create the direct root, and the remote side create + * the indirect root. + * + * We could also do that, but the race conditions (will the remote + * side see the direct root the client made?) seems bigger. + * + * In addition, the remote-side will have a process associated with + * the authenticating user handling the connection (even if there + * is a system-wide daemon or similar). This process can safely make + * the direct and indirect roots without there being such a risk of + * privilege escalation / symlinks in directories owned by the + * originating requester that they cannot delete. + */ + Path addPermRoot(const StorePath & path, const Path & gcRoot) override + { + auto conn(getConnection()); + conn->to << WorkerProto::Op::AddPermRoot; + WorkerProto::write(*this, *conn, path); + WorkerProto::write(*this, *conn, gcRoot); + conn.processStderr(); + return readString(conn->from); + } +}; + ref SSHStore::openConnection() { auto conn = make_ref(); @@ -98,6 +216,8 @@ ref SSHStore::openConnection() std::string command = remoteProgram + " --stdio"; if (remoteStore.get() != "") command += " --store " + shellEscape(remoteStore.get()); + for (auto & arg : extraRemoteProgramArgs) + command += " " + shellEscape(arg); conn->sshConn = master.startCommand(command); conn->to = FdSink(conn->sshConn->in.get()); @@ -106,5 +226,6 @@ ref SSHStore::openConnection() } static RegisterStoreImplementation regSSHStore; +static RegisterStoreImplementation regMountedSSHStore; } From b32b20a6d7cf3a9cf2c81a133c255e9c2ee8e308 Mon Sep 17 00:00:00 2001 From: mupdt <25388474+mupdt@users.noreply.github.com> Date: Mon, 31 Jul 2023 09:07:43 -0400 Subject: [PATCH 654/909] release note entry for the `mounted-ssh-ng://` store --- doc/manual/src/release-notes/rl-next.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index dbe2692f9..0e3d8b462 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,3 +1,6 @@ # Release X.Y (202?-??-??) -- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set. \ No newline at end of file +- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set. + +- Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md). + This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem. From 03c3af1bf97354e281230a82f76d84b2db65db91 Mon Sep 17 00:00:00 2001 From: Matej Urbas Date: Tue, 29 Aug 2023 17:01:09 +0100 Subject: [PATCH 655/909] mounted-ssh-ng store: integration tests --- .../build-remote-with-mounted-ssh-ng.sh | 22 +++++++++++++++++++ tests/functional/local.mk | 1 + 2 files changed, 23 insertions(+) create mode 100644 tests/functional/build-remote-with-mounted-ssh-ng.sh diff --git a/tests/functional/build-remote-with-mounted-ssh-ng.sh b/tests/functional/build-remote-with-mounted-ssh-ng.sh new file mode 100644 index 000000000..443acb6ca --- /dev/null +++ b/tests/functional/build-remote-with-mounted-ssh-ng.sh @@ -0,0 +1,22 @@ +source common.sh + +requireSandboxSupport +[[ $busybox =~ busybox ]] || skipTest "no busybox" + +enableFeatures mounted-ssh-store + +nix build -Lvf simple.nix \ + --arg busybox $busybox \ + --out-link $TEST_ROOT/result-from-remote \ + --store mounted-ssh-ng://localhost + +nix build -Lvf simple.nix \ + --arg busybox $busybox \ + --out-link $TEST_ROOT/result-from-remote-new-cli \ + --store 'mounted-ssh-ng://localhost?remote-program=nix daemon' + +# This verifies that the out link was actually created and valid. The ability +# to create out links (permanent gc roots) is the distinguishing feature of +# the mounted-ssh-ng store. +cat $TEST_ROOT/result-from-remote/hello | grepQuiet 'Hello World!' +cat $TEST_ROOT/result-from-remote-new-cli/hello | grepQuiet 'Hello World!' diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 21dabca88..8d584142a 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -69,6 +69,7 @@ nix_tests = \ build-remote-trustless-should-pass-2.sh \ build-remote-trustless-should-pass-3.sh \ build-remote-trustless-should-fail-0.sh \ + build-remote-with-mounted-ssh-ng.sh \ nar-access.sh \ pure-eval.sh \ eval.sh \ From 4e790efade0c3073292ff73be44351f29badd935 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Nov 2023 13:38:52 +0100 Subject: [PATCH 656/909] Use boost::container::small_vector in place of VLAs --- boehmgc-traceable_allocator-public.diff | 12 +++++++ flake.nix | 3 ++ src/libexpr/eval.cc | 13 +++++--- src/libexpr/gc-small-vector.hh | 42 +++++++++++++++++++++++++ src/libexpr/primops.cc | 9 +++--- 5 files changed, 70 insertions(+), 9 deletions(-) create mode 100644 boehmgc-traceable_allocator-public.diff create mode 100644 src/libexpr/gc-small-vector.hh diff --git a/boehmgc-traceable_allocator-public.diff b/boehmgc-traceable_allocator-public.diff new file mode 100644 index 000000000..903c707a6 --- /dev/null +++ b/boehmgc-traceable_allocator-public.diff @@ -0,0 +1,12 @@ +diff --git a/include/gc_allocator.h b/include/gc_allocator.h +index 597c7f13..587286be 100644 +--- a/include/gc_allocator.h ++++ b/include/gc_allocator.h +@@ -312,6 +312,7 @@ public: + + template<> + class traceable_allocator { ++public: + typedef size_t size_type; + typedef ptrdiff_t difference_type; + typedef void* pointer; diff --git a/flake.nix b/flake.nix index 9030a74f7..570a099ab 100644 --- a/flake.nix +++ b/flake.nix @@ -230,6 +230,9 @@ }).overrideAttrs(o: { patches = (o.patches or []) ++ [ ./boehmgc-coroutine-sp-fallback.diff + + # https://github.com/ivmai/bdwgc/pull/586 + ./boehmgc-traceable_allocator-public.diff ]; }) ) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 46a49c891..90f04d40a 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -16,6 +16,7 @@ #include "fs-input-accessor.hh" #include "memory-input-accessor.hh" #include "signals.hh" +#include "gc-small-vector.hh" #include #include @@ -31,6 +32,7 @@ #include #include +#include #if HAVE_BOEHMGC @@ -1709,7 +1711,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & /* We have all the arguments, so call the primop with the previous and new arguments. */ - Value * vArgs[arity]; + Value * vArgs[maxPrimOpArity]; auto n = argsDone; for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left) vArgs[--n] = arg->primOpApp.right; @@ -1772,11 +1774,11 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v) // 4: about 60 // 5: under 10 // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total. - Value * vArgs[args.size()]; + SmallValueVector<4> vArgs(args.size()); for (size_t i = 0; i < args.size(); ++i) vArgs[i] = args[i]->maybeThunk(state, env); - state.callFunction(vFun, args.size(), vArgs, v, pos); + state.callFunction(vFun, args.size(), vArgs.data(), v, pos); } @@ -2015,8 +2017,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) return result; }; - Value values[es->size()]; - Value * vTmpP = values; + // List of returned strings. References to these Values must NOT be persisted. + SmallTemporaryValueVector values(es->size()); + Value * vTmpP = values.data(); for (auto & [i_pos, i] : *es) { Value & vTmp = *vTmpP++; diff --git a/src/libexpr/gc-small-vector.hh b/src/libexpr/gc-small-vector.hh new file mode 100644 index 000000000..7f4f08fc7 --- /dev/null +++ b/src/libexpr/gc-small-vector.hh @@ -0,0 +1,42 @@ +#pragma once + +#include + +#if HAVE_BOEHMGC + +#include +#include +#include + +#endif + +namespace nix { + +struct Value; + +/** + * A GC compatible vector that may used a reserved portion of `nItems` on the stack instead of allocating on the heap. + */ +#if HAVE_BOEHMGC +template +using SmallVector = boost::container::small_vector>; +#else +template +using SmallVector = boost::container::small_vector; +#endif + +/** + * A vector of value pointers. See `SmallVector`. + */ +template +using SmallValueVector = SmallVector; + +/** + * A vector of values that must not be referenced after the vector is destroyed. + * + * See also `SmallValueVector`. + */ +template +using SmallTemporaryValueVector = SmallVector; + +} \ No newline at end of file diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a8d44d8b7..54a5da817 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4,6 +4,7 @@ #include "eval-inline.hh" #include "eval.hh" #include "eval-settings.hh" +#include "gc-small-vector.hh" #include "globals.hh" #include "json-to-value.hh" #include "names.hh" @@ -2729,7 +2730,7 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs")); state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs"); - Value * res[args[1]->listSize()]; + SmallValueVector res(args[1]->listSize()); size_t found = 0; for (auto v2 : args[1]->listItems()) { @@ -3064,8 +3065,7 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter"); - // FIXME: putting this on the stack is risky. - Value * vs[args[1]->listSize()]; + SmallValueVector vs(args[1]->listSize()); size_t k = 0; bool same = true; @@ -3454,7 +3454,8 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args, state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap"); auto nrLists = args[1]->listSize(); - Value lists[nrLists]; + // List of returned lists before concatenation. References to these Values must NOT be persisted. + SmallTemporaryValueVector lists(nrLists); size_t len = 0; for (unsigned int n = 0; n < nrLists; ++n) { From 46131567da96ffac298b9ec54016b37114b0dfd5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 21 Nov 2023 23:19:25 -0500 Subject: [PATCH 657/909] Add missing `-lrapidcheck` fixing build with shared lib https://github.com/NixOS/nixpkgs/pull/269064 makes rapidcheck be build as a shared lib, but that broke Nix because the `-lrapidcheck` was missing. This fixes that (and doesn't break Nix what the library is a static archive as today). --- src/libexpr/tests/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk index 6d2a04aaf..7689a03e0 100644 --- a/src/libexpr/tests/local.mk +++ b/src/libexpr/tests/local.mk @@ -20,4 +20,4 @@ libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/l libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers -libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock +libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock From 61b76f5f34db7f863a6f22bd9083f677b339fcf6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Nov 2023 11:26:12 +0100 Subject: [PATCH 658/909] Apply suggestion Co-authored-by: John Ericson --- src/libfetchers/input-accessor.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index 53502c621..8e10cf2e2 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -27,7 +27,7 @@ StorePath InputAccessor::fetchToStore( {"path", path.abs()} }; if (auto res = fetchers::getCache()->lookup(*cacheKey)) { - StorePath storePath(fetchers::getStrAttr(*res, "storePath")); + StorePath storePath{fetchers::getStrAttr(*res, "storePath")}; if (store->isValidPath(storePath)) { debug("store path cache hit for '%s'", showPath(path)); return storePath; From 5292f364267eb74005a2e06dfe69c0d0dc8bd2a3 Mon Sep 17 00:00:00 2001 From: r-vdp Date: Wed, 22 Nov 2023 11:33:25 +0100 Subject: [PATCH 659/909] Fix compile warning due to unused variable binding. We still need the check, since we don't have narinfo for locally built store paths. --- src/nix/path-info.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 23198a120..080d6bbf1 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -54,7 +54,7 @@ static json pathInfoToJSON( jsonObject["closureSize"] = getStoreObjectsTotalSize(store, closure); - if (auto * narInfo = dynamic_cast(&*info)) { + if (dynamic_cast(&*info)) { uint64_t totalDownloadSize = 0; for (auto & p : closure) { auto depInfo = store.queryPathInfo(p); From b1ab592f28f08da5dc7c060e5c3b19dc66dbc111 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Nov 2023 11:44:02 +0100 Subject: [PATCH 660/909] Use the StorePath-based cache interface --- src/libfetchers/input-accessor.cc | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index 8e10cf2e2..85dc4609f 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -26,12 +26,9 @@ StorePath InputAccessor::fetchToStore( {"method", (uint8_t) method}, {"path", path.abs()} }; - if (auto res = fetchers::getCache()->lookup(*cacheKey)) { - StorePath storePath{fetchers::getStrAttr(*res, "storePath")}; - if (store->isValidPath(storePath)) { - debug("store path cache hit for '%s'", showPath(path)); - return storePath; - } + if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) { + debug("store path cache hit for '%s'", showPath(path)); + return res->second; } } else debug("source path '%s' is uncacheable", showPath(path)); @@ -51,9 +48,7 @@ StorePath InputAccessor::fetchToStore( : store->addToStoreFromDump(*source, name, method, htSHA256, repair); if (cacheKey) - fetchers::getCache()->upsert( - *cacheKey, - fetchers::Attrs{{"storePath", std::string(storePath.to_string())}}); + fetchers::getCache()->add(store, *cacheKey, {}, storePath, true); return storePath; } From 2ce8c9650b3e714f28d8685e48996141cba2df2c Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Thu, 23 Nov 2023 22:02:20 +0100 Subject: [PATCH 661/909] doc: primops: add more info for foldl (#9254) * doc: primops: add more info for foldl From the existing doc it is not obvious whether the first or the second argument is the accumulator. This is however relevant to know, as for certain scenarios, this might change the behavior. Co-authored-by: Valentin Gagarin --- src/libexpr/primops.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a8d44d8b7..7c0561413 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3179,9 +3179,16 @@ static RegisterPrimOp primop_foldlStrict({ .doc = R"( Reduce a list by applying a binary operator, from left to right, e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) - ...`. For example, `foldl' (x: y: x + y) 0 [1 2 3]` evaluates to 6. - The return value of each application of `op` is evaluated immediately, - even for intermediate values. + ...`. + + For example, `foldl' (acc: elem: acc + elem) 0 [1 2 3]` evaluates + to `6` and `foldl' (acc: elem: { "${elem}" = elem; } // acc) {} + ["a" "b"]` evaluates to `{ a = "a"; b = "b"; }`. + + The first argument of `op` is the accumulator wheres the second + argument is the current element being processed. The return value + of each application of `op` is evaluated immediately, even for + intermediate values. )", .fun = prim_foldlStrict, }); From 5be0e6b314c216b0b51499fc488ca08272297469 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Fri, 24 Nov 2023 10:50:01 +0100 Subject: [PATCH 662/909] doc: primops: fix typo --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7c0561413..ba735b435 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3185,7 +3185,7 @@ static RegisterPrimOp primop_foldlStrict({ to `6` and `foldl' (acc: elem: { "${elem}" = elem; } // acc) {} ["a" "b"]` evaluates to `{ a = "a"; b = "b"; }`. - The first argument of `op` is the accumulator wheres the second + The first argument of `op` is the accumulator whereas the second argument is the current element being processed. The return value of each application of `op` is evaluated immediately, even for intermediate values. From 6a94755b1240be654cadb463a9f528eeccf3787c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 24 Nov 2023 11:45:37 +0100 Subject: [PATCH 663/909] Allow user input in `git commit` We occasionnally commit to git repositories (like with `nix flake update --commit-lock-file`). This shells out to `git commit`, which might wait for user input (for a signing key passphrase for instance). Disable the progress bar while this is running to make sure that the user can enter it. --- src/libfetchers/git.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 2c5b70f53..8cd74057c 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -11,6 +11,8 @@ #include "fs-input-accessor.hh" #include "mounted-input-accessor.hh" #include "git-utils.hh" +#include "logging.hh" +#include "finally.hh" #include "fetch-settings.hh" @@ -314,6 +316,9 @@ struct GitInputScheme : InputScheme runProgram("git", true, { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) }); + // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit` + logger->pause(); + Finally restoreLogger([]() { logger->resume(); }); if (commitMsg) runProgram("git", true, { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-m", *commitMsg }); From b7982372d234b1fd15bab01d09093471c1870bb4 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 15:05:21 +0100 Subject: [PATCH 664/909] Compile hand-written release notes with changelog-d --- .gitignore | 1 + doc/manual/local.mk | 5 +++- doc/manual/rl-next/config | 2 ++ doc/manual/src/contributing/hacking.md | 38 ++++++++++++++++++++++++++ flake.nix | 11 +++++++- 5 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 doc/manual/rl-next/config diff --git a/.gitignore b/.gitignore index 04d96ca2c..38e7e2b5a 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ perl/Makefile.config /doc/manual/src/contributing/experimental-feature-descriptions.md /doc/manual/src/language/builtins.md /doc/manual/src/language/builtin-constants.md +/doc/manual/src/release-notes/rl-next.md # /scripts/ /scripts/nix-profile.sh diff --git a/doc/manual/local.mk b/doc/manual/local.mk index db3daf252..74a4103b3 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -144,6 +144,9 @@ $(d)/language.json: $(bindir)/nix $(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp @mv $@.tmp $@ +$(d)/src/release-notes/rl-next.md: $(d)/rl-next/* + $(trace-gen) changelog-d doc/manual/rl-next > $@ + # Generate the HTML manual. .PHONY: manual-html manual-html: $(docdir)/manual/index.html @@ -177,7 +180,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli # `@docroot@` is to be preserved for documenting the mechanism # FIXME: maybe contributing guides should live right next to the code # instead of in the manual -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/rl-next/config b/doc/manual/rl-next/config new file mode 100644 index 000000000..b3c2e868f --- /dev/null +++ b/doc/manual/rl-next/config @@ -0,0 +1,2 @@ +organization: NixOS +repository: nix diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index fe08ceb94..855900d7a 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -220,3 +220,41 @@ Configure your editor to use the `clangd` from the shell, either by running it i > For some editors (e.g. Visual Studio Code), you may need to install a [special extension](https://open-vsx.org/extension/llvm-vs-code-extensions/vscode-clangd) for the editor to interact with `clangd`. > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). > Editor-specific setup is typically opinionated, so we will not cover it here in more detail. + +## Add a release note + +`doc/manual/rl-next` contains release notes entries for all unreleased changes. + +User-visible changes should come with a release note. + +### Add an entry + +Here's what a complete entry looks like. The file name is not incorporated in the document. + +``` +synopsis: Basically a title +issues: #1234 +prs: #1238 +description: { + +Here's one or more paragraphs that describe the change. + +- It's markdown +- Add references to the manual using @docroot@ + +} +``` + +Significant changes should add the following header, which moves them to the top. + +``` +significance: significant +``` + + +See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). + +### Build process + +Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. +Non-releases build the release notes on the fly. diff --git a/flake.nix b/flake.nix index 9030a74f7..ca37948a7 100644 --- a/flake.nix +++ b/flake.nix @@ -173,6 +173,10 @@ "--enable-internal-api-docs" ]; + # TODO: after backport of https://github.com/NixOS/nixpkgs/pull/268487, remove `haskellPackages.` - + # vastly improves output closure, and adds shell completions + changelog-d = pkgs.buildPackages.haskellPackages.changelog-d; + nativeBuildDeps = [ buildPackages.bison @@ -190,7 +194,10 @@ buildPackages.jq # Also for custom mdBook preprocessor. buildPackages.openssh # only needed for tests (ssh-keygen) ] - ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]; + ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)] + # Official releases don't have rl-next, so we don't need to compile a changelog + ++ lib.optional (!officialRelease) changelog-d + ; buildDeps = [ curl @@ -727,6 +734,8 @@ ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools + # We want changelog-d in the shell even if it's an official release + ++ lib.optional officialRelease changelog-d ; buildInputs = buildDeps ++ propagatedDeps From b26038c517ed10feae751ad6733244c00b715d34 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 16:08:34 +0100 Subject: [PATCH 665/909] doc: Rename 2X.XX to "Upcoming release", and only generate if applicable --- .gitignore | 1 + doc/manual/local.mk | 13 +++++++++++-- doc/manual/src/SUMMARY.md.in | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 38e7e2b5a..5c359d7a0 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ perl/Makefile.config /doc/manual/language.json /doc/manual/xp-features.json /doc/manual/src/SUMMARY.md +/doc/manual/src/SUMMARY-rl-next.md /doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md /doc/manual/src/command-ref/experimental-features-shortlist.md diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 74a4103b3..265a4649d 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -92,7 +92,7 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md @cp $< $@ @$(call process-includes,$@,$@) @@ -144,9 +144,18 @@ $(d)/language.json: $(bindir)/nix $(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp @mv $@.tmp $@ -$(d)/src/release-notes/rl-next.md: $(d)/rl-next/* +# Generate "Upcoming release" notes (or clear it and remove from menu) +$(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/* $(trace-gen) changelog-d doc/manual/rl-next > $@ +$(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md + $(trace-gen) true + @if [ -s $< ]; then \ + echo ' - [Upcoming release](release-notes/rl-next.md)' > $@; \ + else \ + true > $@; \ + fi + # Generate the HTML manual. .PHONY: manual-html manual-html: $(docdir)/manual/index.html diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 8dc464abd..8e7b4eeab 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -114,7 +114,7 @@ - [CLI guideline](contributing/cli-guideline.md) - [C++ style guide](contributing/cxx.md) - [Release Notes](release-notes/release-notes.md) - - [Release X.Y (202?-??-??)](release-notes/rl-next.md) +{{#include ./SUMMARY-rl-next.md}} - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md) - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) From 9aa63f70d7d861ba74764188410c0add730cd48d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 24 Nov 2023 15:32:02 +0100 Subject: [PATCH 666/909] fricklerhandwerk: subscribe to documentation changes (#9422) * fricklerhandwerk: subscribe to documentation changes --- .github/CODEOWNERS | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ab5908649..39d595199 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -14,5 +14,12 @@ /doc @fricklerhandwerk *.md @fricklerhandwerk +# Documentation of built-in functions +src/libexpr/primops.cc @fricklerhandwerk @roberth +# Documentation on experimental features +src/libutil/experimental-features.cc @fricklerhandwerk +# Documentation on configuration settings +src/libstore/globals.hh @fricklerhandwerk + # Libstore layer /src/libstore @thufschmitt From 2a538c571b13877fa426f2cff2749cf17d140216 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 16:49:11 +0100 Subject: [PATCH 667/909] Add scripts/release-notes --- maintainers/release-process.md | 45 ++++------ scripts/release-notes | 148 +++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 27 deletions(-) create mode 100755 scripts/release-notes diff --git a/maintainers/release-process.md b/maintainers/release-process.md index d85266b81..e542d8be5 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -24,34 +24,23 @@ release: * In a checkout of the Nix repo, make sure you're on `master` and run `git pull`. -* Move the contents of `doc/manual/src/release-notes/rl-next.md` - (except the first line) to - `doc/manual/src/release-notes/rl-$VERSION.md` (where `$VERSION` is - the contents of `.version` *without* the patch level, e.g. `2.12` - rather than `2.12.0`). - -* Add a header to `doc/manual/src/release-notes/rl-$VERSION.md` like - - ``` - # Release 2.12 (2022-12-06) - ``` - -* Proof-read / edit / rearrange the release notes. Breaking changes - and highlights should go to the top. - -* Add a link to the release notes to `doc/manual/src/SUMMARY.md.in` - (*not* `SUMMARY.md`), e.g. - - ``` - - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md) - ``` - -* Run +* Compile the release notes by running ```console $ git checkout -b release-notes - $ git add doc/manual/src/release-notes/rl-$VERSION.md - $ git commit -a -m 'Release notes' + $ VERSION=X.YY ./scripts/release-notes + ``` + + where `X.YY` is *without* the patch level, e.g. `2.12` rather than ~~`2.12.0`~~. + + A commit is created. + +* Proof-read / edit / rearrange the release notes if needed. Breaking changes + and highlights should go to the top. + +* Push. + + ```console $ git push --set-upstream $REMOTE release-notes ``` @@ -67,15 +56,17 @@ release: $ git checkout -b $VERSION-maintenance ``` -* Mark the release as stable: +* Mark the release as official: ```console - $ git cherry-pick f673551e71942a52b6d7ae66af8b67140904a76a + $ sed -e 's/officialRelease = false;/officialRelease = true;/' -i flake.nix ``` This removes the link to `rl-next.md` from the manual and sets `officialRelease = true` in `flake.nix`. +* Commit + * Push the release branch: ```console diff --git a/scripts/release-notes b/scripts/release-notes new file mode 100755 index 000000000..e5ee39d11 --- /dev/null +++ b/scripts/release-notes @@ -0,0 +1,148 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash ../shell.nix -I nixpkgs=channel:nixos-unstable-small +# ^^^^^^^ +# Only used for bash. shell.nix goes to the flake. + +# --- CONFIGURATION --- + +# This does double duty for +# - including rl-next +# - marking where to insert new links (right after) +SUMMARY_MARKER_LINE='{{#include ./SUMMARY-rl-next.md}}' + +# --- LIB --- + +log() { + echo 1>&2 "release-notes:" "$@" +} +logcmd() { + local cmd="$1" + shift + logcmd2 "$cmd" "${*@Q}" "$cmd" "$@" +} +logcmd2() { + local fakecmd="$1" + local fakeargs="$2" + shift + shift + printf 1>&2 "release-notes: \033[34;1m$fakecmd\033[0m " + echo "$fakeargs" 1>&2 + "$@" +} +die() { + # ANSI red + printf 1>&2 "release-notes: \033[31;1merror:\033[0m" + echo 1>&2 "" "$@" + exit 1 +} +confirm() { + local answer + echo 1>&2 "$@" "[y/n]" + read -r answer + case "$answer" in + y|Y|yes|Yes|YES) + return 0 + ;; + n|N|no|No|NO) + return 1 + ;; + *) + echo 1>&2 "please answer y or n" + confirm "$@" + ;; + esac +} +report_done() { + logcmd2 "git" "show" git -c pager.show=false show + printf 1>&2 "release-notes: \033[32;1mdone\033[0m\n" +} + +# --- PARSE ARGS --- + +if [[ $# -gt 0 ]]; then + die "Release notes takes no arguments, but make sure to set VERSION." +fi + +# --- CHECKS --- + +if [[ ! -e flake.nix ]] || [[ ! -e .git ]]; then + die "must run in repo root" + exit 1 +fi + +# repo must be clean +if ! git diff --quiet; then + die "repo is dirty, please commit or stash changes" +fi + +if ! git diff --quiet --cached; then + die "repo has staged changes, please commit or stash them" +fi + +if ! grep "$SUMMARY_MARKER_LINE" doc/manual/src/SUMMARY.md.in >/dev/null; then + # would have been nice to catch this early, but won't be worth the extra infra + die "SUMMARY.md.in is missing the marker line '$SUMMARY_MARKER_LINE', which would be used for inserting a new release notes page. Please fix the script." +fi + +if [[ ! -n "${VERSION:-}" ]]; then + die "please set the VERSION environment variable before invoking this script" + exit 1 +fi + +case "$VERSION" in + # FIXME: accepts "." without any real digits + [[:digit:]]*.[[:digit:]]*) + ;; + *) + die "VERSION must be MAJOR.MINOR, where each is a number, e.g. 2.20 (VERSION was set to $VERSION)" + ;; +esac + +# --- DEFAULTS --- + +if [[ ! -n "${DATE:-}" ]]; then + DATE="$(date +%Y-%m-%d)" + log "DATE not set, using $DATE" +fi + +case "$DATE" in + [[:digit:]]*-[[:digit:]]*-[[:digit:]]*) + ;; + *) + die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)" + ;; +esac + +# --- DO THE WORK --- + +basename=rl-$VERSION.md +file=doc/manual/src/release-notes/$basename +title="Release $VERSION ($DATE)" + +( + # TODO add minor number, and append? + echo "# $title" + echo + changelog-d doc/manual/rl-next | sed -e 's/ *$//' +) > $file + +log "Wrote $file" + +NEW_SUMMARY_LINE=" - [$title](release-notes/$basename)" + +# find the marker line, insert new link after it +escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" +escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" +logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in + +for f in doc/manual/rl-next/*.md; do + if [[ config != "$(basename $f)" ]]; then + logcmd git rm $f + fi +done + +logcmd git add $file doc/manual/src/SUMMARY.md.in +logcmd git status +logcmd git commit -m "release notes: $VERSION" + +report_done From b1ea30f21d24df9afc4eb1635eee9a080e4f81f3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 18:13:59 +0100 Subject: [PATCH 668/909] scripts/release-notes: Support patch releases This also fixes the broken case statement, which has globs, not regexes. --- maintainers/release-process.md | 24 +++++++++++++ scripts/release-notes | 65 ++++++++++++++++++++++++---------- 2 files changed, 71 insertions(+), 18 deletions(-) diff --git a/maintainers/release-process.md b/maintainers/release-process.md index e542d8be5..b1259a9bd 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -150,6 +150,30 @@ release: ## Creating a point release +* Checkout. + + ```console + $ git checkout XX.YY-maintenance + ``` + +* Determine the next patch version. + + ```console + $ export VERSION=XX.YY.ZZ + ``` + +* Update release notes. + + ```console + $ ./scripts/release-notes + ``` + +* Push. + + ```console + $ git push + ``` + * Wait for the desired evaluation of the maintenance jobset to finish building. diff --git a/scripts/release-notes b/scripts/release-notes index e5ee39d11..43f20a547 100755 --- a/scripts/release-notes +++ b/scripts/release-notes @@ -89,14 +89,40 @@ if [[ ! -n "${VERSION:-}" ]]; then exit 1 fi -case "$VERSION" in - # FIXME: accepts "." without any real digits - [[:digit:]]*.[[:digit:]]*) - ;; - *) - die "VERSION must be MAJOR.MINOR, where each is a number, e.g. 2.20 (VERSION was set to $VERSION)" - ;; -esac +# mutate/initialize: +# VERSION: MAJOR.MINOR +# FULL_VERSION: MAJOR.MINOR.PATCH +# IS_PATCH: true if this is a patch release; append instead of create +if grep -E '^[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then + log 'is minor' + IS_PATCH=false + FULL_VERSION="$VERSION.0" +elif grep -E '^[0-9]+\.[0-9]+\.0$' <<< "$VERSION" >/dev/null; then + log 'is minor (.0)' + IS_PATCH=false + FULL_VERSION="$VERSION" + VERSION="$(echo "$VERSION" | sed -e 's/\.0$//')" +elif grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then + log 'is patch' + IS_PATCH=true + FULL_VERSION="$VERSION" + VERSION="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')" +else + die "VERSION must be MAJOR.MINOR[.PATCH], where each is a number, e.g. 2.20 or 2.20.1 (VERSION was set to $VERSION)" +fi + +log "VERSION=$VERSION" +log "FULL_VERSION=$FULL_VERSION" +log "IS_PATCH=$IS_PATCH" + +basename=rl-$VERSION.md +file=doc/manual/src/release-notes/$basename + +if ! $IS_PATCH; then + if [[ -e $file ]]; then + die "release notes file $file already exists. If you'd like to make a minor release, pass a patch version, e.g. 2.20.1" + fi +fi # --- DEFAULTS --- @@ -106,7 +132,7 @@ if [[ ! -n "${DATE:-}" ]]; then fi case "$DATE" in - [[:digit:]]*-[[:digit:]]*-[[:digit:]]*) + [0-9]*-[0-9]*-[0-9]*) ;; *) die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)" @@ -115,25 +141,28 @@ esac # --- DO THE WORK --- -basename=rl-$VERSION.md -file=doc/manual/src/release-notes/$basename +# menu title="Release $VERSION ($DATE)" +# section on page +section_title="Release $FULL_VERSION ($DATE)" ( # TODO add minor number, and append? - echo "# $title" + echo "# $section_title" echo changelog-d doc/manual/rl-next | sed -e 's/ *$//' -) > $file +) | tee -a $file log "Wrote $file" -NEW_SUMMARY_LINE=" - [$title](release-notes/$basename)" +if ! $IS_PATCH; then + NEW_SUMMARY_LINE=" - [$title](release-notes/$basename)" -# find the marker line, insert new link after it -escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" -escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" -logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in + # find the marker line, insert new link after it + escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" + escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" + logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in +fi for f in doc/manual/rl-next/*.md; do if [[ config != "$(basename $f)" ]]; then From 7c4ee5c8135fae65602791f0b89d0dbae7e94f3e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 18:36:50 +0100 Subject: [PATCH 669/909] scripts/release-notes: Avoid mutating variables --- scripts/release-notes | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/scripts/release-notes b/scripts/release-notes index 43f20a547..2e1be64a4 100755 --- a/scripts/release-notes +++ b/scripts/release-notes @@ -89,33 +89,35 @@ if [[ ! -n "${VERSION:-}" ]]; then exit 1 fi -# mutate/initialize: -# VERSION: MAJOR.MINOR -# FULL_VERSION: MAJOR.MINOR.PATCH -# IS_PATCH: true if this is a patch release; append instead of create +# version_major_minor: MAJOR.MINOR +# version_full: MAJOR.MINOR.PATCH +# IS_PATCH: true if this is a patch release; append instead of create if grep -E '^[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then log 'is minor' IS_PATCH=false - FULL_VERSION="$VERSION.0" + version_full="$VERSION.0" + version_major_minor="$VERSION" elif grep -E '^[0-9]+\.[0-9]+\.0$' <<< "$VERSION" >/dev/null; then log 'is minor (.0)' IS_PATCH=false - FULL_VERSION="$VERSION" - VERSION="$(echo "$VERSION" | sed -e 's/\.0$//')" + version_full="$VERSION" + version_major_minor="$(echo "$VERSION" | sed -e 's/\.0$//')" elif grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then log 'is patch' IS_PATCH=true - FULL_VERSION="$VERSION" - VERSION="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')" + version_full="$VERSION" + version_major_minor="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')" else die "VERSION must be MAJOR.MINOR[.PATCH], where each is a number, e.g. 2.20 or 2.20.1 (VERSION was set to $VERSION)" fi -log "VERSION=$VERSION" -log "FULL_VERSION=$FULL_VERSION" +unset VERSION + +log "version_major_minor=$version_major_minor" +log "version_full=$version_full" log "IS_PATCH=$IS_PATCH" -basename=rl-$VERSION.md +basename=rl-${version_major_minor}.md file=doc/manual/src/release-notes/$basename if ! $IS_PATCH; then @@ -142,9 +144,9 @@ esac # --- DO THE WORK --- # menu -title="Release $VERSION ($DATE)" +title="Release $version_major_minor ($DATE)" # section on page -section_title="Release $FULL_VERSION ($DATE)" +section_title="Release $version_full ($DATE)" ( # TODO add minor number, and append? @@ -172,6 +174,6 @@ done logcmd git add $file doc/manual/src/SUMMARY.md.in logcmd git status -logcmd git commit -m "release notes: $VERSION" +logcmd git commit -m "release notes: $version_full" report_done From 6971c4adc06d574cbe1e9ab6da19814e11e2ba6c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 19:08:42 +0100 Subject: [PATCH 670/909] maintainers/release-notes <- scripts/release-notes --- {scripts => maintainers}/release-notes | 0 maintainers/release-process.md | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename {scripts => maintainers}/release-notes (100%) diff --git a/scripts/release-notes b/maintainers/release-notes similarity index 100% rename from scripts/release-notes rename to maintainers/release-notes diff --git a/maintainers/release-process.md b/maintainers/release-process.md index b1259a9bd..db8b064a5 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -28,7 +28,7 @@ release: ```console $ git checkout -b release-notes - $ VERSION=X.YY ./scripts/release-notes + $ VERSION=X.YY ./maintainers/release-notes ``` where `X.YY` is *without* the patch level, e.g. `2.12` rather than ~~`2.12.0`~~. @@ -165,7 +165,7 @@ release: * Update release notes. ```console - $ ./scripts/release-notes + $ ./maintainers/release-notes ``` * Push. From 857f9168f7b48aa491052f24fb571c21398f9826 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 24 Nov 2023 15:27:54 +0100 Subject: [PATCH 671/909] Migrate rl-next.md to doc/manual/rl-next directory --- doc/manual/rl-next/mounted-ssh-store.md | 9 +++++++++ doc/manual/rl-next/nix-env-json-drv-path.md | 9 +++++++++ doc/manual/src/release-notes/rl-next.md | 6 ------ 3 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 doc/manual/rl-next/mounted-ssh-store.md create mode 100644 doc/manual/rl-next/nix-env-json-drv-path.md delete mode 100644 doc/manual/src/release-notes/rl-next.md diff --git a/doc/manual/rl-next/mounted-ssh-store.md b/doc/manual/rl-next/mounted-ssh-store.md new file mode 100644 index 000000000..39fac5283 --- /dev/null +++ b/doc/manual/rl-next/mounted-ssh-store.md @@ -0,0 +1,9 @@ +synopsis: Mounted SSH Store +issues: #7890 +prs: #7912 +description: { + +Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md). +This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem. + +} diff --git a/doc/manual/rl-next/nix-env-json-drv-path.md b/doc/manual/rl-next/nix-env-json-drv-path.md new file mode 100644 index 000000000..fbe2b67d8 --- /dev/null +++ b/doc/manual/rl-next/nix-env-json-drv-path.md @@ -0,0 +1,9 @@ +synopsis: Fix `nix-env --query --drv-path --json` +prs: #9257 +description: { + +Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set. + +} + + diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md deleted file mode 100644 index 0e3d8b462..000000000 --- a/doc/manual/src/release-notes/rl-next.md +++ /dev/null @@ -1,6 +0,0 @@ -# Release X.Y (202?-??-??) - -- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set. - -- Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md). - This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem. From 54b684765519dee1e74dca71605f8e7f6c8b0a25 Mon Sep 17 00:00:00 2001 From: ivan770 Date: Fri, 24 Nov 2023 11:17:35 -0500 Subject: [PATCH 672/909] doc: fix machine-specific capabilities leaking --- src/libstore/globals.hh | 14 +++++++++++--- src/libstore/store-api.hh | 5 ++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 27caf42c4..838d2aba2 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -183,7 +183,9 @@ public: command line switch and defaults to `1`. The value `0` means that the builder should use all available CPU cores in the system. )", - {"build-cores"}, false}; + {"build-cores"}, + // Don't document the machine-specific default value + false}; /** * Read-only mode. Don't copy stuff to the store, don't change @@ -699,7 +701,10 @@ public: Build systems will usually detect the target platform to be the current physical system and therefore produce machine code incompatible with what may be intended in the derivation. You should design your derivation's `builder` accordingly and cross-check the results when using this option against natively-built versions of your derivation. - )", {}, false}; + )", + {}, + // Don't document the machine-specific default value + false}; Setting systemFeatures{ this, @@ -744,7 +749,10 @@ public: [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. Included by default on Linux if the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting is enabled. - )", {}, false}; + )", + {}, + // Don't document the machine-specific default value + false}; Setting substituters{ this, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 32ad2aa44..8b6bf9aed 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -167,7 +167,10 @@ struct StoreConfig : public Config Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. Example: `"kvm"` - )" }; + )", + {}, + // Don't document the machine-specific default value + false}; }; class Store : public std::enable_shared_from_this, public virtual StoreConfig From d2f5e263e3c095dfe9d874387665b88c4bfff6f1 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Mon, 6 Feb 2023 16:36:57 +0100 Subject: [PATCH 673/909] Switch from std::regex to boost::regex --- flake.nix | 24 +++++++++++++++++------- src/libexpr/local.mk | 2 +- src/libexpr/primops.cc | 35 +++++++++++++++++++++++------------ 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/flake.nix b/flake.nix index 9030a74f7..1f7e7b2af 100644 --- a/flake.nix +++ b/flake.nix @@ -154,7 +154,7 @@ configureFlags = lib.optionals stdenv.isLinux [ - "--with-boost=${boost}/lib" + "--with-boost=${boost-nix}/lib" "--with-sandbox-shell=${sh}/bin/busybox" ] ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [ @@ -202,7 +202,7 @@ version = libgit2.lastModifiedDate; cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"]; })) - boost + boost-nix lowdown-nix libsodium ] @@ -423,14 +423,14 @@ propagatedBuildInputs = propagatedDeps; - disallowedReferences = [ boost ]; + disallowedReferences = [ boost-nix ]; preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic) '' # Copy libboost_context so we don't get all of Boost in our closure. # https://github.com/NixOS/nixpkgs/issues/45462 mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib + cp -pd ${boost-nix}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib rm -f $out/lib/*.a ${lib.optionalString currentStdenv.hostPlatform.isLinux '' chmod u+w $out/lib/*.so.* @@ -440,9 +440,9 @@ for LIB in $out/lib/*.dylib; do chmod u+w $LIB install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost}/lib/ $LIB || true + install_name_tool -delete_rpath ${boost-nix}/lib/ $LIB || true done - install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib + install_name_tool -change ${boost-nix}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib ''} ''; @@ -470,9 +470,13 @@ ''} ${lib.optionalString currentStdenv.isDarwin '' install_name_tool \ - -change ${boost}/lib/libboost_context.dylib \ + -change ${boost-nix}/lib/libboost_context.dylib \ $out/lib/libboost_context.dylib \ $out/lib/libnixutil.dylib + install_name_tool \ + -change ${boost-nix}/lib/libboost_regex.dylib \ + $out/lib/libboost_regex.dylib \ + $out/lib/libnixexpr.dylib ''} ''; @@ -495,6 +499,12 @@ meta.mainProgram = "nix"; }); + boost-nix = final.boost.override { + # enableIcu arg is not yet supported + # but will be with next nixpkgs update + enableIcu = false; + }; + lowdown-nix = with final; currentStdenv.mkDerivation rec { name = "lowdown-0.9.0"; diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index ed7bf9490..946059339 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib libexpr_LIBS = libutil libstore libfetchers -libexpr_LDFLAGS += -lboost_context -pthread +libexpr_LDFLAGS += -lboost_context -lboost_regex -pthread ifdef HOST_LINUX libexpr_LDFLAGS += -ldl endif diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ba735b435..0c34cb6e8 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -16,6 +16,7 @@ #include "primops.hh" #include +#include #include #include @@ -24,7 +25,6 @@ #include #include -#include #include #include @@ -3885,19 +3885,30 @@ static RegisterPrimOp primop_convertHash({ .fun = prim_convertHash, }); +// regex aliases, switch between boost and std +using regex = boost::regex; +using regex_error = boost::regex_error; +using cmatch = boost::cmatch; +using cregex_iterator = boost::cregex_iterator; +namespace regex_constants = boost::regex_constants; +// overloaded function alias +constexpr auto regex_match = [] (auto &&...args) { + return boost::regex_match(std::forward(args)...); + }; + struct RegexCache { // TODO use C++20 transparent comparison when available - std::unordered_map cache; + std::unordered_map cache; std::list keys; - std::regex get(std::string_view re) + regex get(std::string_view re) { auto it = cache.find(re); if (it != cache.end()) return it->second; keys.emplace_back(re); - return cache.emplace(keys.back(), std::regex(keys.back(), std::regex::extended)).first->second; + return cache.emplace(keys.back(), regex(keys.back(), regex::extended)).first->second; } }; @@ -3917,8 +3928,8 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) NixStringContext context; const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match"); - std::cmatch match; - if (!std::regex_match(str.begin(), str.end(), match, regex)) { + cmatch match; + if (!regex_match(str.begin(), str.end(), match, regex)) { v.mkNull(); return; } @@ -3933,8 +3944,8 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) (v.listElems()[i] = state.allocValue())->mkString(match[i + 1].str()); } - } catch (std::regex_error & e) { - if (e.code() == std::regex_constants::error_space) { + } catch (regex_error & e) { + if (e.code() == regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ state.debugThrowLastTrace(EvalError({ .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), @@ -3997,8 +4008,8 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) NixStringContext context; const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split"); - auto begin = std::cregex_iterator(str.begin(), str.end(), regex); - auto end = std::cregex_iterator(); + auto begin = cregex_iterator(str.begin(), str.end(), regex); + auto end = cregex_iterator(); // Any matches results are surrounded by non-matching results. const size_t len = std::distance(begin, end); @@ -4037,8 +4048,8 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) assert(idx == 2 * len + 1); - } catch (std::regex_error & e) { - if (e.code() == std::regex_constants::error_space) { + } catch (regex_error & e) { + if (e.code() == regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ state.debugThrowLastTrace(EvalError({ .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), From 213594721ace2b2e2dcf3d57615178a2c1690aa4 Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Sat, 25 Nov 2023 17:21:49 +0100 Subject: [PATCH 674/909] gitignore: Also ignore .DS_Store This is a file that Finder on Mac OS loves to add into various folders. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 04d96ca2c..f7dcea422 100644 --- a/.gitignore +++ b/.gitignore @@ -144,3 +144,6 @@ result # clangd and possibly more .cache/ + +# Mac OS +.DS_Store From f25c06d7a3289343887d09761c82356a3b6b441b Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Sun, 26 Nov 2023 15:46:09 +0100 Subject: [PATCH 675/909] docs: Fix broken link Link target definitions need to be in a separate paragraph to be collected. Fixup for https://github.com/NixOS/nix/commit/217d863f7a251a4d8a08ff3294944b45146c61c9 --- doc/manual/src/language/values.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md index 0bb656746..aea68a441 100644 --- a/doc/manual/src/language/values.md +++ b/doc/manual/src/language/values.md @@ -116,6 +116,7 @@ [store path]: ../glossary.md#gloss-store-path Paths can include [string interpolation] and can themselves be [interpolated in other expressions]. + [interpolated in other expressions]: ./string-interpolation.md#interpolated-expressions At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. From d63f72197cec6ff95d9ffc83aa8076acd86a3fd1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 24 Nov 2023 15:48:41 +0100 Subject: [PATCH 676/909] Don't run changelog-d in the build This way we lose the preview of release notes on master, as well as on https://nixos.org/manual/nix/unstable/release-notes/rl-next but we can come back to this. --- doc/manual/local.mk | 8 +++++++- doc/manual/src/contributing/hacking.md | 2 +- flake.nix | 9 ++++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 265a4649d..f22dfa69e 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -146,7 +146,13 @@ $(d)/language.json: $(bindir)/nix # Generate "Upcoming release" notes (or clear it and remove from menu) $(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/* - $(trace-gen) changelog-d doc/manual/rl-next > $@ + @if type -p changelog-d > /dev/null; then \ + echo " GEN " $@; \ + changelog-d doc/manual/rl-next > $@; \ + else \ + echo " NULL " $@; \ + true > $@; \ + fi $(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md $(trace-gen) true diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 855900d7a..3291d5a20 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -257,4 +257,4 @@ See also the [format documentation](https://github.com/haskell/cabal/blob/master ### Build process Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. -Non-releases build the release notes on the fly. +Set `buildUnreleasedNotes = true;` in `flake.nix` to build the release notes on the fly. diff --git a/flake.nix b/flake.nix index ca37948a7..db913f062 100644 --- a/flake.nix +++ b/flake.nix @@ -14,6 +14,9 @@ officialRelease = false; + # Set to true to build the release notes for the next release. + buildUnreleasedNotes = false; + version = lib.fileContents ./.version + versionSuffix; versionSuffix = if officialRelease @@ -196,7 +199,7 @@ ] ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)] # Official releases don't have rl-next, so we don't need to compile a changelog - ++ lib.optional (!officialRelease) changelog-d + ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d ; buildDeps = @@ -734,8 +737,8 @@ ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools - # We want changelog-d in the shell even if it's an official release - ++ lib.optional officialRelease changelog-d + # We want changelog-d in the shell even if the current build doesn't need it + ++ lib.optional (officialRelease || ! buildUnreleasedNotes) changelog-d ; buildInputs = buildDeps ++ propagatedDeps From c5d49ec7ab7b9fb33f0336a909ac837e208be807 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 24 Nov 2023 16:18:27 +0100 Subject: [PATCH 677/909] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/decdf666c833a325cb4417041a90681499e06a41' (2023-11-18) → 'github:NixOS/nixpkgs/9ba29e2346bc542e9909d1021e8fd7d4b3f64db0' (2023-11-23) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 1b3c0b6d5..f120d3b5f 100644 --- a/flake.lock +++ b/flake.lock @@ -50,11 +50,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1700342017, - "narHash": "sha256-HaibwlWH5LuqsaibW3sIVjZQtEM/jWtOHX4Nk93abGE=", + "lastModified": 1700748986, + "narHash": "sha256-/nqLrNU297h3PCw4QyDpZKZEUHmialJdZW2ceYFobds=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "decdf666c833a325cb4417041a90681499e06a41", + "rev": "9ba29e2346bc542e9909d1021e8fd7d4b3f64db0", "type": "github" }, "original": { From e7e21aa0c839460a62456fa44f31339c187077ff Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 24 Nov 2023 16:27:10 +0100 Subject: [PATCH 678/909] flake.nix: Use top level changelog-d It is about 2 MB now, as only it and libffi (tiny) are new in the build or shell closures. --- flake.nix | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index db913f062..a239226c0 100644 --- a/flake.nix +++ b/flake.nix @@ -176,9 +176,7 @@ "--enable-internal-api-docs" ]; - # TODO: after backport of https://github.com/NixOS/nixpkgs/pull/268487, remove `haskellPackages.` - - # vastly improves output closure, and adds shell completions - changelog-d = pkgs.buildPackages.haskellPackages.changelog-d; + changelog-d = pkgs.buildPackages.changelog-d; nativeBuildDeps = [ From 2b7016cc56d12e67de9f1f25b18311866a26a5fe Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 27 Nov 2023 08:33:03 +0100 Subject: [PATCH 679/909] add path based redirects up to now, those were managed outside of this repo, which as unsurprisingly a real hassle to deal with if one wanted to prevent URLs from breaking when moving pages around. this change removes a large part of the friction involved in moving content in the Nix manual. possible next steps for further automation: - check for content that moved and warn if it's not reachable from links that were valid prior to a change - create redirect rules automatically based on this information --- doc/manual/_redirects | 30 ++++++++++++++++++++++++++++++ doc/manual/redirects.js | 8 +++++--- 2 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 doc/manual/_redirects diff --git a/doc/manual/_redirects b/doc/manual/_redirects new file mode 100644 index 000000000..4ea289d86 --- /dev/null +++ b/doc/manual/_redirects @@ -0,0 +1,30 @@ +# redirect rules for paths (server-side) to prevent link rot. +# see ./redirects.js for redirects based on URL fragments (client-side) +# +# concrete user story this supports: +# - user finds URL to the manual for Nix x.y +# - Nix x.z (z > y) is the most recent release +# - updating the version in the URL will show the right thing +# +# format documentation: +# - https://docs.netlify.com/routing/redirects/#syntax-for-the-redirects-file +# - https://docs.netlify.com/routing/redirects/redirect-options/ +# +# conventions: +# - always force (!) since this allows re-using file names +# - group related paths to ease readability +# - always append new redirects to the end of the file +# - redirects that should have been there but are missing can be inserted where they belong + +/expressions/expression-language /language/ 301! +/expressions/language-values /language/values 301! +/expressions/language-constructs /language/constructs 301! +/expressions/language-operators /language/operators 301! +/expressions/* /language/:splat 301! + +/package-management/basic-package-mgmt /command-ref/nix-env 301! + +/package-management/channels* /command-ref/nix-channel 301! + +/package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301! + diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index d1b10109d..3b507adf3 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -1,7 +1,9 @@ -// redirect rules for anchors ensure backwards compatibility of URLs. -// this must be done on the client side, as web servers do not see the anchor part of the URL. +// redirect rules for URL fragments (client-side) to prevent link rot. +// this must be done on the client side, as web servers do not see the fragment part of the URL. +// it will only work with JavaScript enabled in the browser, but this is the best we can do here. +// see ./_redirects for path redirects (client-side) -// redirections are declared as follows: +// redirects are declared as follows: // each entry has as its key a path matching the requested URL path, relative to the mdBook document root. // // IMPORTANT: it must specify the full path with file name and suffix From f56401a114cb5504c3d74b893ce270ed28fd03e3 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Sat, 25 Nov 2023 11:26:57 +0800 Subject: [PATCH 680/909] `nix flake update` add deprecation warnings. This builds on #8817, to add additional UX help for people with existing muscle memory (or shell history) with --update-input and tries to gently guide them towards the newly evolved CLI UI. Co-authored-by: Cole Helbling --- src/libcmd/installables.cc | 24 ++++++++++++++++++++++++ src/nix/flake.cc | 8 +++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 1c6103020..68287b445 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -47,6 +47,16 @@ MixFlakeOptions::MixFlakeOptions() { auto category = "Common flake-related options"; + addFlag({ + .longName = "recreate-lock-file", + .description = "Recreate the flake's lock file from scratch.", + .category = category, + .handler = {[&]() { + lockFlags.recreateLockFile = true; + warn("'--recreate-lock-file' is deprecated and will be removed in a future version; use 'nix flake update' instead."); + }} + }); + addFlag({ .longName = "no-update-lock-file", .description = "Do not allow any updates to the flake's lock file.", @@ -79,6 +89,20 @@ MixFlakeOptions::MixFlakeOptions() .handler = {&lockFlags.commitLockFile, true} }); + addFlag({ + .longName = "update-input", + .description = "Update a specific flake input (ignoring its previous entry in the lock file).", + .category = category, + .labels = {"input-path"}, + .handler = {[&](std::string s) { + warn("'--update-input' is a deprecated alias for 'flake update' and will be removed in a future version."); + lockFlags.inputUpdates.insert(flake::parseInputPath(s)); + }}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); + }} + }); + addFlag({ .longName = "override-input", .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.", diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 38938f09e..e0c67fdfa 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -89,7 +89,13 @@ public: .label="inputs", .optional=true, .handler={[&](std::string inputToUpdate){ - auto inputPath = flake::parseInputPath(inputToUpdate); + InputPath inputPath; + try { + inputPath = flake::parseInputPath(inputToUpdate); + } catch (Error & e) { + warn("Invalid flake input '%s'. To update a specific flake, use 'nix flake update --flake %s' instead.", inputToUpdate, inputToUpdate); + throw e; + } if (lockFlags.inputUpdates.contains(inputPath)) warn("Input '%s' was specified multiple times. You may have done this by accident."); lockFlags.inputUpdates.insert(inputPath); From 75134b7513eb781074969fc8d6d865cc95063444 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Mon, 27 Nov 2023 08:56:24 +0000 Subject: [PATCH 681/909] libexpr: add missing dependency on 'flake/call-flake.nix.gen.hh' Without the change build for `eval.o` fails occasionally as: $ make src/libexpr/eval.o GEN Makefile.config GEN src/libexpr/primops/derivation.nix.gen.hh GEN src/libexpr/fetchurl.nix.gen.hh GEN src/libexpr/parser-tab.cc GEN src/libexpr/lexer-tab.cc src/libexpr/lexer.l:314: warning, -s option given but default rule can be matched CXX src/libexpr/eval.o src/libexpr/eval.cc:519:18: fatal error: flake/call-flake.nix.gen.hh: No such file or directory 519 | #include "flake/call-flake.nix.gen.hh" | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~ compilation terminated. make: *** [mk/patterns.mk:3: src/libexpr/eval.o] Error 1 Noticed in https://github.com/NixOS/nixpkgs/pull/269439 --- src/libexpr/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index ed7bf9490..637f998b6 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -47,6 +47,6 @@ $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh -$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh +$(d)/eval.o: $(d)/flake/call-flake.nix.gen.hh src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = From e986d20bedfc054663632255388bbd33fec99114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Mon, 27 Nov 2023 10:25:18 +0100 Subject: [PATCH 682/909] Remove an obsolete comment --- flake.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/flake.nix b/flake.nix index 1f7e7b2af..887ef93b4 100644 --- a/flake.nix +++ b/flake.nix @@ -500,8 +500,6 @@ }); boost-nix = final.boost.override { - # enableIcu arg is not yet supported - # but will be with next nixpkgs update enableIcu = false; }; From 384ffb4443fd47d04f36c6bcc6ebf476274673ab Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 27 Nov 2023 09:18:56 +0100 Subject: [PATCH 683/909] add deprecation warnings in documentation this is hacky, but can serve as a stopgap until we can do it programmatically. --- src/libcmd/installables.cc | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 68287b445..6e670efea 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -49,7 +49,13 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "recreate-lock-file", - .description = "Recreate the flake's lock file from scratch.", + .description = R"( + Recreate the flake's lock file from scratch. + + > **DEPRECATED** + > + > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead. + )", .category = category, .handler = {[&]() { lockFlags.recreateLockFile = true; @@ -73,8 +79,13 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "no-registries", - .description = - "Don't allow lookups in the flake registries. This option is deprecated; use `--no-use-registries`.", + .description = R"( + Don't allow lookups in the flake registries. + + > **DEPRECATED** + > + > Use [`--no-use-registries`](#opt-no-use-registries) instead. + )", .category = category, .handler = {[&]() { lockFlags.useRegistries = false; @@ -91,7 +102,13 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "update-input", - .description = "Update a specific flake input (ignoring its previous entry in the lock file).", + .description = R"( + Update a specific flake input (ignoring its previous entry in the lock file). + + > **DEPRECATED** + > + > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead. + )", .category = category, .labels = {"input-path"}, .handler = {[&](std::string s) { From f7bfec2806708573798d610cda101f27a24d9218 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 27 Nov 2023 15:18:29 +0100 Subject: [PATCH 684/909] maintainers/release-notes: Improve DATE check Co-authored-by: Valentin Gagarin --- maintainers/release-notes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintainers/release-notes b/maintainers/release-notes index 2e1be64a4..34cd85a56 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -134,7 +134,7 @@ if [[ ! -n "${DATE:-}" ]]; then fi case "$DATE" in - [0-9]*-[0-9]*-[0-9]*) + [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ;; *) die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)" From 68c48756fece5aee77f9b44607afa9248d75e67c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 27 Nov 2023 15:50:45 +0100 Subject: [PATCH 685/909] libexpr/local.mk: Make eval compile deps regular Dependency is now entirely through the eval.cc rule. All gen.hh deps are now there. --- src/libexpr/local.mk | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 637f998b6..b37fe6f1d 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -45,8 +45,6 @@ $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh -$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh - -$(d)/eval.o: $(d)/flake/call-flake.nix.gen.hh +$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = From 20cd5eb2b3668f5b95b6f020e1c258011c18ea33 Mon Sep 17 00:00:00 2001 From: Alois Wohlschlager Date: Mon, 27 Nov 2023 19:12:15 +0100 Subject: [PATCH 686/909] nix repl: Only hide the progress bar while waiting for user input In commit 0d2163c6dcf03463fa91ec6d0d96c928ad907366, the progress bar was hidden in nix repl because of a regression that caused it to interfere with user input. Several users like(d) seeing the progress bar in the repl during builds. Only hiding it while waiting for user input gives us the best of both worlds, so do just that. --- src/libcmd/repl.cc | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index bf5643a5c..0986296ad 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -43,7 +43,6 @@ extern "C" { #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" -#include "progress-bar.hh" #include "print.hh" #if HAVE_BOEHMGC @@ -262,13 +261,11 @@ void NixRepl::mainLoop() rl_set_list_possib_func(listPossibleCallback); #endif - /* Stop the progress bar because it interferes with the display of - the repl. */ - stopProgressBar(); - std::string input; while (true) { + // Hide the progress bar while waiting for user input, so that it won't interfere. + logger->pause(); // When continuing input from previous lines, don't print a prompt, just align to the same // number of chars as the prompt. if (!getLine(input, input.empty() ? "nix-repl> " : " ")) { @@ -278,6 +275,7 @@ void NixRepl::mainLoop() logger->cout(""); break; } + logger->resume(); try { if (!removeWhitespace(input).empty() && !processLine(input)) return; } catch (ParseError & e) { From f300e11b056dea414d7d77bbc6e5a7dc5d9ddd41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 27 Nov 2023 19:41:30 +0100 Subject: [PATCH 687/909] Rename `nix show-config` to `nix config show` Part of #7672 --- doc/manual/local.mk | 4 ++-- doc/manual/rl-next/nix-config-show.md | 8 +++++++ src/nix/{show-config.cc => config.cc} | 27 ++++++++++++++++++++--- src/nix/main.cc | 1 + tests/functional/config.sh | 12 +++++----- tests/functional/experimental-features.sh | 8 +++---- 6 files changed, 45 insertions(+), 15 deletions(-) create mode 100644 doc/manual/rl-next/nix-config-show.md rename src/nix/{show-config.cc => config.cc} (67%) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index f22dfa69e..d568681d4 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -24,7 +24,7 @@ man-pages += $(foreach subcommand, \ clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8 # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox. -# Set cores to 0 because otherwise nix show-config resolves the cores based on the current machine +# Set cores to 0 because otherwise `nix config show` resolves the cores based on the current machine dummy-env = env -i \ HOME=/dummy \ NIX_CONF_DIR=/dummy \ @@ -111,7 +111,7 @@ $(d)/nix.json: $(bindir)/nix @mv $@.tmp $@ $(d)/conf-file.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp + $(trace-gen) $(dummy-env) $(bindir)/nix config show --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ $(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix diff --git a/doc/manual/rl-next/nix-config-show.md b/doc/manual/rl-next/nix-config-show.md new file mode 100644 index 000000000..08ad207cb --- /dev/null +++ b/doc/manual/rl-next/nix-config-show.md @@ -0,0 +1,8 @@ +synopsis: `nix config show` +issues: #7672 +prs: #9477 +description: { + +`nix show-config` was renamed to `nix config show` to be more consistent with the rest of the command-line interface. + +} diff --git a/src/nix/show-config.cc b/src/nix/config.cc similarity index 67% rename from src/nix/show-config.cc rename to src/nix/config.cc index 3530584f9..5b280d11d 100644 --- a/src/nix/show-config.cc +++ b/src/nix/config.cc @@ -7,11 +7,31 @@ using namespace nix; -struct CmdShowConfig : Command, MixJSON +struct CmdConfig : virtual NixMultiCommand +{ + CmdConfig() : MultiCommand(RegisterCommand::getCommandsFor({"config"})) + { } + + std::string description() override + { + return "manipulate the Nix configuration"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix config' requires a sub-command."); + command->second->run(); + } +}; + +struct CmdConfigShow : Command, MixJSON { std::optional name; - CmdShowConfig() { + CmdConfigShow() { expectArgs({ .label = {"name"}, .optional = true, @@ -56,4 +76,5 @@ struct CmdShowConfig : Command, MixJSON } }; -static auto rShowConfig = registerCommand("show-config"); +static auto rCmdConfig = registerCommand("config"); +static auto rShowConfig = registerCommand2({"config", "show"}); diff --git a/src/nix/main.cc b/src/nix/main.cc index 73641f6d2..2a6c2f478 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -134,6 +134,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs {"ping-store", {"store", "ping"}}, {"sign-paths", {"store", "sign"}}, {"show-derivation", {"derivation", "show"}}, + {"show-config", {"config", "show"}}, {"to-base16", {"hash", "to-base16"}}, {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, diff --git a/tests/functional/config.sh b/tests/functional/config.sh index 0780c55d0..324fe95bd 100644 --- a/tests/functional/config.sh +++ b/tests/functional/config.sh @@ -40,20 +40,20 @@ files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | x # Test that it's possible to load the config from a custom location here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") export NIX_USER_CONF_FILES=$here/config/nix-with-substituters.conf -var=$(nix show-config | grep '^substituters =' | cut -d '=' -f 2 | xargs) +var=$(nix config show | grep '^substituters =' | cut -d '=' -f 2 | xargs) [[ $var == https://example.com ]] # Test that it's possible to load config from the environment -prev=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) +prev=$(nix config show | grep '^cores' | cut -d '=' -f 2 | xargs) export NIX_CONFIG="cores = 4242"$'\n'"experimental-features = nix-command flakes" -exp_cores=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) -exp_features=$(nix show-config | grep '^experimental-features' | cut -d '=' -f 2 | xargs) +exp_cores=$(nix config show | grep '^cores' | cut -d '=' -f 2 | xargs) +exp_features=$(nix config show | grep '^experimental-features' | cut -d '=' -f 2 | xargs) [[ $prev != $exp_cores ]] [[ $exp_cores == "4242" ]] # flakes implies fetch-tree [[ $exp_features == "fetch-tree flakes nix-command" ]] # Test that it's possible to retrieve a single setting's value -val=$(nix show-config | grep '^warn-dirty' | cut -d '=' -f 2 | xargs) -val2=$(nix show-config warn-dirty) +val=$(nix config show | grep '^warn-dirty' | cut -d '=' -f 2 | xargs) +val2=$(nix config show warn-dirty) [[ $val == $val2 ]] diff --git a/tests/functional/experimental-features.sh b/tests/functional/experimental-features.sh index 607bf0a8e..9ee4a53d4 100644 --- a/tests/functional/experimental-features.sh +++ b/tests/functional/experimental-features.sh @@ -31,7 +31,7 @@ source common.sh NIX_CONFIG=' experimental-features = nix-command accept-flake-config = true -' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr grepQuiet "false" $TEST_ROOT/stdout grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr @@ -39,7 +39,7 @@ grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature ' NIX_CONFIG=' accept-flake-config = true experimental-features = nix-command -' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr grepQuiet "false" $TEST_ROOT/stdout grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr @@ -47,7 +47,7 @@ grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature ' NIX_CONFIG=' experimental-features = nix-command flakes accept-flake-config = true -' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr grepQuiet "true" $TEST_ROOT/stdout grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr @@ -55,7 +55,7 @@ grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr NIX_CONFIG=' accept-flake-config = true experimental-features = nix-command flakes -' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr grepQuiet "true" $TEST_ROOT/stdout grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr From 6d1605818c12461edc9f4ee17a7929fdc8fe916c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 27 Nov 2023 19:46:05 +0100 Subject: [PATCH 688/909] Rename `nix doctor` to `nix config check` Fix #7672 --- doc/manual/src/contributing/cli-guideline.md | 2 +- src/nix/{doctor.cc => config-check.cc} | 4 ++-- src/nix/main.cc | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) rename src/nix/{doctor.cc => config-check.cc} (97%) diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md index e53d2d178..8dbac45b0 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/contributing/cli-guideline.md @@ -87,7 +87,7 @@ impacted the most by bad user experience. and [aligning of text](#text-alignment). - [Autocomplete](#shell-completion) of options. - Examples of such commands: `nix doctor`, `nix edit`, `nix eval`, ... + Examples of such commands: `nix edit`, `nix eval`, ... - **Utility and scripting commands** diff --git a/src/nix/doctor.cc b/src/nix/config-check.cc similarity index 97% rename from src/nix/doctor.cc rename to src/nix/config-check.cc index 59f9e3e5d..410feca2f 100644 --- a/src/nix/doctor.cc +++ b/src/nix/config-check.cc @@ -38,7 +38,7 @@ void checkInfo(const std::string & msg) { } -struct CmdDoctor : StoreCommand +struct CmdConfigCheck : StoreCommand { bool success = true; @@ -152,4 +152,4 @@ struct CmdDoctor : StoreCommand } }; -static auto rCmdDoctor = registerCommand("doctor"); +static auto rCmdConfigCheck = registerCommand2({ "config", "check" }); diff --git a/src/nix/main.cc b/src/nix/main.cc index 2a6c2f478..d715d0400 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -139,6 +139,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, {"verify", {"store", "verify"}}, + {"doctor", {"config", "check"}}, }; bool aliasUsed = false; From 52e0911302b20336c1600b60a98894423e110d7d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 25 Nov 2023 00:33:21 -0500 Subject: [PATCH 689/909] Use `buildprefix` in a few more places `installcheck` doesn't yet work, but the rest of the build can now happen mostly inside a separate build directory. Progress on #9342 Co-authored-by: Valentin Gagarin --- Makefile | 6 ++++-- doc/manual/src/contributing/hacking.md | 25 +++++++++++++++++++++++++ mk/build-dir.mk | 10 ++++++++++ mk/install-dirs.mk | 11 +++++++++++ mk/lib.mk | 25 +++---------------------- mk/templates.mk | 8 ++++---- src/libcmd/local.mk | 2 +- src/libexpr/local.mk | 4 ++-- src/libmain/local.mk | 2 +- src/libstore/local.mk | 2 +- tests/functional/local.mk | 4 ++-- 11 files changed, 64 insertions(+), 35 deletions(-) create mode 100644 mk/build-dir.mk create mode 100644 mk/install-dirs.mk diff --git a/Makefile b/Makefile index 4f4ac0c6e..77974074c 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ --include Makefile.config -clean-files += Makefile.config +include mk/build-dir.mk + +-include $(buildprefix)Makefile.config +clean-files += $(buildprefix)Makefile.config ifeq ($(ENABLE_BUILD), yes) makefiles = \ diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 3291d5a20..0a95334f7 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -146,6 +146,31 @@ $ nix build .#packages.aarch64-linux.default Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`). Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms. +### Building for multiple platforms at once + +It is useful to perform multiple cross and native builds on the same source tree, +for example to ensure that better support for one platform doesn't break the build for another. +In order to facilitate this, Nix has some support for being built out of tree – that is, placing build artefacts in a different directory than the source code: + +1. Create a directory for the build, e.g. + + ```bash + mkdir build + ``` + +2. Run the configure script from that directory, e.g. + + ```bash + cd build + ../configure + ``` + +3. Run make from the source directory, but with the build directory specified, e.g. + + ```bash + make builddir=build + ``` + ## System type Nix uses a string with he following format to identify the *system type* or *platform* it runs on: diff --git a/mk/build-dir.mk b/mk/build-dir.mk new file mode 100644 index 000000000..02f4cae60 --- /dev/null +++ b/mk/build-dir.mk @@ -0,0 +1,10 @@ +# Initialise support for build directories. +builddir ?= + +ifdef builddir + buildprefix = $(builddir)/ + buildprefixrel = $(builddir) +else + buildprefix = + buildprefixrel = . +endif diff --git a/mk/install-dirs.mk b/mk/install-dirs.mk new file mode 100644 index 000000000..732b0d6fc --- /dev/null +++ b/mk/install-dirs.mk @@ -0,0 +1,11 @@ +# Default installation paths. +prefix ?= /usr/local +libdir ?= $(prefix)/lib +bindir ?= $(prefix)/bin +libexecdir ?= $(prefix)/libexec +datadir ?= $(prefix)/share +localstatedir ?= $(prefix)/var +sysconfdir ?= $(prefix)/etc +mandir ?= $(prefix)/share/man + +DESTDIR ?= diff --git a/mk/lib.mk b/mk/lib.mk index 49abe9862..3d503364f 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -43,27 +43,6 @@ define newline endef -# Default installation paths. -prefix ?= /usr/local -libdir ?= $(prefix)/lib -bindir ?= $(prefix)/bin -libexecdir ?= $(prefix)/libexec -datadir ?= $(prefix)/share -localstatedir ?= $(prefix)/var -sysconfdir ?= $(prefix)/etc -mandir ?= $(prefix)/share/man - - -# Initialise support for build directories. -builddir ?= - -ifdef builddir - buildprefix = $(builddir)/ -else - buildprefix = -endif - - # Pass -fPIC if we're building dynamic libraries. BUILD_SHARED_LIBS ?= 1 @@ -94,6 +73,8 @@ ifeq ($(BUILD_DEBUG), 1) endif +include mk/build-dir.mk +include mk/install-dirs.mk include mk/functions.mk include mk/tracing.mk include mk/clean.mk @@ -112,7 +93,7 @@ define include-sub-makefile include $(1) endef -$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile, $(mf)))) +$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile,$(mf)))) # Instantiate stuff. diff --git a/mk/templates.mk b/mk/templates.mk index c7ac7afbf..866bdc17f 100644 --- a/mk/templates.mk +++ b/mk/templates.mk @@ -10,10 +10,10 @@ endef ifneq ($(MAKECMDGOALS), clean) -%.h: %.h.in - $(trace-gen) rm -f $@ && ./config.status --quiet --header=$@ +$(buildprefix)%.h: %.h.in + $(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --header=$(@:$(buildprefix)%=%) -%: %.in - $(trace-gen) rm -f $@ && ./config.status --quiet --file=$@ +$(buildprefix)%: %.in + $(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --file=$(@:$(buildprefix)%=%) endif diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index 541a7d2ba..afd35af08 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -12,4 +12,4 @@ libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread libcmd_LIBS = libstore libutil libexpr libmain libfetchers -$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 18d1bc95c..c07a18bb5 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -36,7 +36,7 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh -$(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644)) $(foreach i, $(wildcard src/libexpr/value/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644))) @@ -47,4 +47,4 @@ $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh -src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = +$(buildprefix)src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = diff --git a/src/libmain/local.mk b/src/libmain/local.mk index 99da95e27..5c7061863 100644 --- a/src/libmain/local.mk +++ b/src/libmain/local.mk @@ -14,4 +14,4 @@ libmain_LIBS = libstore libutil libmain_ALLOW_UNDEFINED = 1 -$(eval $(call install-file-in, $(d)/nix-main.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-main.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 0be0bf310..68ccdc409 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -59,7 +59,7 @@ $(d)/build.cc: clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh -$(eval $(call install-file-in, $(d)/nix-store.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-store.pc, $(libdir)/pkgconfig, 0644)) $(foreach i, $(wildcard src/libstore/builtins/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644))) diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 8d584142a..10b399d75 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -140,9 +140,9 @@ ifeq ($(ENABLE_BUILD), yes) endif $(d)/test-libstoreconsumer.sh.test $(d)/test-libstoreconsumer.sh.test-debug: \ - $(d)/test-libstoreconsumer/test-libstoreconsumer + $(buildprefix)$(d)/test-libstoreconsumer/test-libstoreconsumer $(d)/plugins.sh.test $(d)/plugins.sh.test-debug: \ - $(d)/plugins/libplugintest.$(SO_EXT) + $(buildprefix)$(d)/plugins/libplugintest.$(SO_EXT) install-tests += $(foreach x, $(nix_tests), $(d)/$(x)) From 02bd821f2e71372d31bbe6700bd68086cc2ee70a Mon Sep 17 00:00:00 2001 From: Alex Ameen Date: Wed, 29 Nov 2023 19:26:39 -0600 Subject: [PATCH 690/909] fix: `nlohmann::adl_serializer` for `std::optional` (#9147) This allows templates such as `NLOHMANN_DEFINE_TYPE_*` templates and other generators with things like `std::vector>`. Co-authored-by: John Ericson --- src/libutil/json-utils.hh | 19 ++++++++--- src/libutil/tests/json-utils.cc | 58 +++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 src/libutil/tests/json-utils.cc diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh index 77c63595c..06dd80cf7 100644 --- a/src/libutil/json-utils.hh +++ b/src/libutil/json-utils.hh @@ -78,20 +78,29 @@ namespace nlohmann { */ template struct adl_serializer> { - static std::optional from_json(const json & json) { + /** + * @brief Convert a JSON type to an `optional` treating + * `null` as `std::nullopt`. + */ + static void from_json(const json & json, std::optional & t) { static_assert( nix::json_avoids_null::value, "null is already in use for underlying type's JSON"); - return json.is_null() + t = json.is_null() ? std::nullopt - : std::optional { adl_serializer::from_json(json) }; + : std::make_optional(json.template get()); } - static void to_json(json & json, std::optional t) { + + /** + * @brief Convert an optional type to a JSON type treating `std::nullopt` + * as `null`. + */ + static void to_json(json & json, const std::optional & t) { static_assert( nix::json_avoids_null::value, "null is already in use for underlying type's JSON"); if (t) - adl_serializer::to_json(json, *t); + json = *t; else json = nullptr; } diff --git a/src/libutil/tests/json-utils.cc b/src/libutil/tests/json-utils.cc new file mode 100644 index 000000000..f0ce15c93 --- /dev/null +++ b/src/libutil/tests/json-utils.cc @@ -0,0 +1,58 @@ +#include +#include + +#include + +#include "json-utils.hh" + +namespace nix { + +/* Test `to_json` and `from_json` with `std::optional` types. + * We are specifically interested in whether we can _nest_ optionals in STL + * containers so we that we can leverage existing adl_serializer templates. */ + +TEST(to_json, optionalInt) { + std::optional val = std::make_optional(420); + ASSERT_EQ(nlohmann::json(val), nlohmann::json(420)); + val = std::nullopt; + ASSERT_EQ(nlohmann::json(val), nlohmann::json(nullptr)); +} + +TEST(to_json, vectorOfOptionalInts) { + std::vector> vals = { + std::make_optional(420), + std::nullopt, + }; + ASSERT_EQ(nlohmann::json(vals), nlohmann::json::parse("[420,null]")); +} + +TEST(to_json, optionalVectorOfInts) { + std::optional> val = std::make_optional(std::vector { + -420, + 420, + }); + ASSERT_EQ(nlohmann::json(val), nlohmann::json::parse("[-420,420]")); + val = std::nullopt; + ASSERT_EQ(nlohmann::json(val), nlohmann::json(nullptr)); +} + +TEST(from_json, optionalInt) { + nlohmann::json json = 420; + std::optional val = json; + ASSERT_TRUE(val.has_value()); + ASSERT_EQ(*val, 420); + json = nullptr; + json.get_to(val); + ASSERT_FALSE(val.has_value()); +} + +TEST(from_json, vectorOfOptionalInts) { + nlohmann::json json = { 420, nullptr }; + std::vector> vals = json; + ASSERT_EQ(vals.size(), 2); + ASSERT_TRUE(vals.at(0).has_value()); + ASSERT_EQ(*vals.at(0), 420); + ASSERT_FALSE(vals.at(1).has_value()); +} + +} /* namespace nix */ From a7115a47ef0d83ea81b494f6bc5b11d8286e0672 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 29 Nov 2023 21:03:56 -0500 Subject: [PATCH 691/909] Improve ACL clearing support (fixing FreeBSD build) The problem was that f880469173061a07f0b2a24734932c5a9ad633c6 forgot that the `#include ` was guarded by an `#ifdef __linux__`. However, the build failure was only on FreeBSD --- turns out other platforms have this header too! The fix therefore uses a new configure check so we properly clear ACLs on more platforms. --- configure.ac | 2 ++ src/libstore/local-store.cc | 1 - src/libstore/posix-fs-canonicalise.cc | 6 ++++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 71e93feaa..f8b937eb5 100644 --- a/configure.ac +++ b/configure.ac @@ -282,6 +282,8 @@ case "$host_os" in esac AC_SUBST(HAVE_SECCOMP, [$have_seccomp]) +# Optional dependencies for better normalizing file system data +AC_CHECK_HEADERS[sys/xattr.h] # Look for aws-cpp-sdk-s3. AC_LANG_PUSH(C++) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 4ff75f528..9ed061b01 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -34,7 +34,6 @@ #include #include #include -#include #endif #ifdef __CYGWIN__ diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc index cc3ab0b74..f38fa8369 100644 --- a/src/libstore/posix-fs-canonicalise.cc +++ b/src/libstore/posix-fs-canonicalise.cc @@ -1,4 +1,6 @@ -#include +#if HAVE_SYS_XATTR_H +# include +#endif #include "posix-fs-canonicalise.hh" #include "file-system.hh" @@ -76,7 +78,7 @@ static void canonicalisePathMetaData_( if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) throw Error("file '%1%' has an unsupported type", path); -#if __linux__ +#ifdef HAVE_SYS_XATTR_H /* Remove extended attributes / ACLs. */ ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); From d536c57e878a04f795c1ef8ee3232a47035da2cf Mon Sep 17 00:00:00 2001 From: Federico Pellegrin Date: Tue, 17 Aug 2021 04:26:41 +0200 Subject: [PATCH 692/909] Docs build: depend on locally built nix executable and not installed one Previously many of the documentation targets were depending on `$(bindir)/nix` which is the installed version. This meant that its install rules would be triggered (which in chain would also trigger the install of libraries, as reported in #5140). Therefore a build of the documentation without an installation would not be possible (which apart from doing unwanted operations it may also generate permission problems for example). The fix makes the rules depend on `$(nix_PATH)` instead, which is the executable in the build tree. --- Makefile | 11 ++++++++--- doc/manual/local.mk | 35 ++++++++++++++++++++--------------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 4f4ac0c6e..957920686 100644 --- a/Makefile +++ b/Makefile @@ -19,9 +19,7 @@ makefiles = \ misc/zsh/local.mk \ misc/systemd/local.mk \ misc/launchd/local.mk \ - misc/upstart/local.mk \ - doc/manual/local.mk \ - doc/internal-api/local.mk + misc/upstart/local.mk endif ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes) @@ -55,4 +53,11 @@ endif include mk/lib.mk +# Must be included after `mk/lib.mk` so rules refer to variables defined +# by the library. Rules are not "lazy" like variables, unfortunately. +ifeq ($(ENABLE_BUILD), yes) +$(eval $(call include-sub-makefile, doc/manual/local.mk)) +$(eval $(call include-sub-makefile, doc/internal-api/local.mk)) +endif + GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src diff --git a/doc/manual/local.mk b/doc/manual/local.mk index d568681d4..fa9db9f02 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -1,5 +1,10 @@ ifeq ($(doc_generate),yes) +# The version of Nix used to generate the doc. Can also be +# `$(nix_INSTALL_PATH)` or just `nix` (to grap ambient from the `PATH`), +# if one prefers. +doc_nix = $(nix_PATH) + MANUAL_SRCS := \ $(call rwildcard, $(d)/src, *.md) \ $(call rwildcard, $(d)/src, */*.md) @@ -32,7 +37,7 @@ dummy-env = env -i \ NIX_STATE_DIR=/dummy \ NIX_CONFIG='cores = 0' -nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw +nix-eval = $(dummy-env) $(doc_nix) eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw # re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution define process-includes @@ -96,52 +101,52 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src @cp $< $@ @$(call process-includes,$@,$@) -$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(bindir)/nix +$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)' @mv $@.tmp $@ -$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix +$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(doc_nix) @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "conf"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ -$(d)/nix.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix __dump-cli > $@.tmp +$(d)/nix.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) __dump-cli > $@.tmp @mv $@.tmp $@ -$(d)/conf-file.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix config show --json --experimental-features nix-command > $@.tmp +$(d)/conf-file.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) config show --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ -$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix +$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ -$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(bindir)/nix +$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ -$(d)/xp-features.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix __dump-xp-features > $@.tmp +$(d)/xp-features.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) __dump-xp-features > $@.tmp @mv $@.tmp $@ -$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix +$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(doc_nix) @cat doc/manual/src/language/builtins-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<)).builtins' >> $@.tmp; @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ -$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(bindir)/nix +$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(doc_nix) @cat doc/manual/src/language/builtin-constants-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtin-constants.nix (builtins.fromJSON (builtins.readFile $<)).constants' >> $@.tmp; @cat doc/manual/src/language/builtin-constants-suffix.md >> $@.tmp @mv $@.tmp $@ -$(d)/language.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp +$(d)/language.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) __dump-language > $@.tmp @mv $@.tmp $@ # Generate "Upcoming release" notes (or clear it and remove from menu) From 743232bf04d8ade18cfa2c791ed466ce48519878 Mon Sep 17 00:00:00 2001 From: Greg Pfeil Date: Thu, 30 Nov 2023 00:17:25 -0700 Subject: [PATCH 693/909] =?UTF-8?q?Don=E2=80=99t=20use=20`execvp`=20when?= =?UTF-8?q?=20we=20know=20the=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/nix/develop.cc | 2 +- src/nix/fmt.cc | 2 +- src/nix/run.cc | 10 +++++++--- src/nix/run.hh | 1 + 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 38482ed42..ae9be79a3 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -662,7 +662,7 @@ struct CmdDevelop : Common, MixEnvironment } } - runProgramInStore(store, shell, args, buildEnvironment.getSystem()); + runProgramInStore(store, true, shell, args, buildEnvironment.getSystem()); } }; diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc index c85eacded..396c93dbb 100644 --- a/src/nix/fmt.cc +++ b/src/nix/fmt.cc @@ -49,7 +49,7 @@ struct CmdFmt : SourceExprCommand { } } - runProgramInStore(store, app.program, programArgs); + runProgramInStore(store, false, app.program, programArgs); }; }; diff --git a/src/nix/run.cc b/src/nix/run.cc index ea0a17897..d531f712d 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -25,6 +25,7 @@ std::string chrootHelperName = "__run_in_chroot"; namespace nix { void runProgramInStore(ref store, + bool search, const std::string & program, const Strings & args, std::optional system) @@ -58,7 +59,10 @@ void runProgramInStore(ref store, if (system) setPersonality(*system); - execvp(program.c_str(), stringsToCharPtrs(args).data()); + if (search) + execvp(program.c_str(), stringsToCharPtrs(args).data()); + else + execv(program.c_str(), stringsToCharPtrs(args).data()); throw SysError("unable to execute '%s'", program); } @@ -132,7 +136,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment Strings args; for (auto & arg : command) args.push_back(arg); - runProgramInStore(store, *command.begin(), args); + runProgramInStore(store, true, *command.begin(), args); } }; @@ -194,7 +198,7 @@ struct CmdRun : InstallableValueCommand Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); - runProgramInStore(store, app.program, allArgs); + runProgramInStore(store, false, app.program, allArgs); } }; diff --git a/src/nix/run.hh b/src/nix/run.hh index 97ddef19b..c62287e7e 100644 --- a/src/nix/run.hh +++ b/src/nix/run.hh @@ -6,6 +6,7 @@ namespace nix { void runProgramInStore(ref store, + bool search, const std::string & program, const Strings & args, std::optional system = std::nullopt); From f99e468640ab0eac7f07ac2b328222eb45dee8d8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 27 Nov 2023 08:48:11 -0500 Subject: [PATCH 694/909] Avoid `/` in documentation URLs They are redundant and look weird. --- doc/manual/_redirects | 18 ++++++++++++++---- doc/manual/src/SUMMARY.md.in | 14 +++++++------- .../{advanced-topics.md => index.md} | 0 doc/manual/src/architecture/architecture.md | 2 +- .../command-ref/{command-ref.md => index.md} | 0 doc/manual/src/contributing/hacking.md | 2 +- .../contributing/{contributing.md => index.md} | 0 .../installation/{installation.md => index.md} | 0 .../{package-management.md => index.md} | 0 .../src/protocols/{protocols.md => index.md} | 0 doc/manual/src/quick-start.md | 2 +- .../{release-notes.md => index.md} | 0 12 files changed, 24 insertions(+), 14 deletions(-) rename doc/manual/src/advanced-topics/{advanced-topics.md => index.md} (100%) rename doc/manual/src/command-ref/{command-ref.md => index.md} (100%) rename doc/manual/src/contributing/{contributing.md => index.md} (100%) rename doc/manual/src/installation/{installation.md => index.md} (100%) rename doc/manual/src/package-management/{package-management.md => index.md} (100%) rename doc/manual/src/protocols/{protocols.md => index.md} (100%) rename doc/manual/src/release-notes/{release-notes.md => index.md} (100%) diff --git a/doc/manual/_redirects b/doc/manual/_redirects index 4ea289d86..2038671d7 100644 --- a/doc/manual/_redirects +++ b/doc/manual/_redirects @@ -13,18 +13,28 @@ # conventions: # - always force (!) since this allows re-using file names # - group related paths to ease readability -# - always append new redirects to the end of the file +# - keep in alphabetical/wildcards-last order, which will reduce version control conflicts # - redirects that should have been there but are missing can be inserted where they belong +/advanced-topics/advanced-topics /advanced-topics 301! + +/command-ref/command-ref /command-ref 301! + +/contributing/contributing /contributing 301! + /expressions/expression-language /language/ 301! -/expressions/language-values /language/values 301! /expressions/language-constructs /language/constructs 301! /expressions/language-operators /language/operators 301! +/expressions/language-values /language/values 301! /expressions/* /language/:splat 301! +/installation/installation /installation 301! + /package-management/basic-package-mgmt /command-ref/nix-env 301! - /package-management/channels* /command-ref/nix-channel 301! - +/package-management/package-management /package-management 301! /package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301! +/protocols/protocols /protocols 301! + +/release-notes/release-notes /release-notes 301! diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 8e7b4eeab..686d3e8d7 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -2,7 +2,7 @@ - [Introduction](introduction.md) - [Quick Start](quick-start.md) -- [Installation](installation/installation.md) +- [Installation](installation/index.md) - [Supported Platforms](installation/supported-platforms.md) - [Installing a Binary Distribution](installation/installing-binary.md) - [Installing Nix from Source](installation/installing-source.md) @@ -31,11 +31,11 @@ - [Import From Derivation](language/import-from-derivation.md) - [Built-in Constants](language/builtin-constants.md) - [Built-in Functions](language/builtins.md) -- [Package Management](package-management/package-management.md) +- [Package Management](package-management/index.md) - [Profiles](package-management/profiles.md) - [Garbage Collection](package-management/garbage-collection.md) - [Garbage Collector Roots](package-management/garbage-collector-roots.md) -- [Advanced Topics](advanced-topics/advanced-topics.md) +- [Advanced Topics](advanced-topics/index.md) - [Sharing Packages Between Machines](package-management/sharing-packages.md) - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) - [Copying Closures via SSH](package-management/copy-closure.md) @@ -45,7 +45,7 @@ - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md) - [Verifying Build Reproducibility](advanced-topics/diff-hook.md) - [Using the `post-build-hook`](advanced-topics/post-build-hook.md) -- [Command Reference](command-ref/command-ref.md) +- [Command Reference](command-ref/index.md) - [Common Options](command-ref/opt-common.md) - [Common Environment Variables](command-ref/env-common.md) - [Main Commands](command-ref/main-commands.md) @@ -102,18 +102,18 @@ - [Channels](command-ref/files/channels.md) - [Default Nix expression](command-ref/files/default-nix-expression.md) - [Architecture and Design](architecture/architecture.md) -- [Protocols](protocols/protocols.md) +- [Protocols](protocols/index.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [Glossary](glossary.md) -- [Contributing](contributing/contributing.md) +- [Contributing](contributing/index.md) - [Hacking](contributing/hacking.md) - [Testing](contributing/testing.md) - [Documentation](contributing/documentation.md) - [Experimental Features](contributing/experimental-features.md) - [CLI guideline](contributing/cli-guideline.md) - [C++ style guide](contributing/cxx.md) -- [Release Notes](release-notes/release-notes.md) +- [Release Notes](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md) - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) diff --git a/doc/manual/src/advanced-topics/advanced-topics.md b/doc/manual/src/advanced-topics/index.md similarity index 100% rename from doc/manual/src/advanced-topics/advanced-topics.md rename to doc/manual/src/advanced-topics/index.md diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 79429508f..2fec4ed20 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -52,7 +52,7 @@ The following [concept map] shows its main components (rectangles), the objects '---------------' ``` -At the top is the [command line interface](../command-ref/command-ref.md) that drives the underlying layers. +At the top is the [command line interface](../command-ref/index.md) that drives the underlying layers. The [Nix language](../language/index.md) evaluator transforms Nix expressions into self-contained *build plans*, which are used to derive *build results* from referenced *build inputs*. diff --git a/doc/manual/src/command-ref/command-ref.md b/doc/manual/src/command-ref/index.md similarity index 100% rename from doc/manual/src/command-ref/command-ref.md rename to doc/manual/src/command-ref/index.md diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 0a95334f7..9de5ad39b 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -10,7 +10,7 @@ $ cd nix The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions]. -[installation instructions]: ../installation/installation.md +[installation instructions]: ../installation/index.md ## Building Nix with flakes diff --git a/doc/manual/src/contributing/contributing.md b/doc/manual/src/contributing/index.md similarity index 100% rename from doc/manual/src/contributing/contributing.md rename to doc/manual/src/contributing/index.md diff --git a/doc/manual/src/installation/installation.md b/doc/manual/src/installation/index.md similarity index 100% rename from doc/manual/src/installation/installation.md rename to doc/manual/src/installation/index.md diff --git a/doc/manual/src/package-management/package-management.md b/doc/manual/src/package-management/index.md similarity index 100% rename from doc/manual/src/package-management/package-management.md rename to doc/manual/src/package-management/index.md diff --git a/doc/manual/src/protocols/protocols.md b/doc/manual/src/protocols/index.md similarity index 100% rename from doc/manual/src/protocols/protocols.md rename to doc/manual/src/protocols/index.md diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 1d2688ede..5f54abbde 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -13,7 +13,7 @@ to subsequent chapters. The install script will use `sudo`, so make sure you have sufficient rights. On Linux, `--daemon` can be omitted for a single-user install. - For other installation methods, see [here](installation/installation.md). + For other installation methods, see [here](installation/index.md). 1. See what installable packages are currently available in the channel: diff --git a/doc/manual/src/release-notes/release-notes.md b/doc/manual/src/release-notes/index.md similarity index 100% rename from doc/manual/src/release-notes/release-notes.md rename to doc/manual/src/release-notes/index.md From ea95327e72f5781295417b0eae46a5e351bebebd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Nov 2023 16:16:17 +0100 Subject: [PATCH 695/909] Move restricted/pure-eval access control out of the evaluator and into the accessor --- src/libcmd/installables.cc | 7 +- src/libexpr/eval.cc | 103 +++++---------------- src/libexpr/eval.hh | 25 +++--- src/libexpr/parser.y | 19 +++- src/libexpr/primops.cc | 119 +++++++++++-------------- src/nix-build/nix-build.cc | 7 +- src/nix-instantiate/nix-instantiate.cc | 2 +- tests/functional/restricted.sh | 4 +- 8 files changed, 115 insertions(+), 171 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 6e670efea..6b3c82374 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -260,9 +260,10 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s evalSettings.pureEval = false; auto state = getEvalState(); - Expr *e = state->parseExprFromFile( - resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file))) - ); + auto e = + state->parseExprFromFile( + resolveExprPath( + lookupFileArg(*state, *file))); Value root; state->eval(e, root); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7e68e6f9b..23ac349fe 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -509,7 +509,18 @@ EvalState::EvalState( , sOutputSpecified(symbols.create("outputSpecified")) , repair(NoRepair) , emptyBindings(0) - , rootFS(makeFSInputAccessor(CanonPath::root)) + , rootFS( + makeFSInputAccessor( + CanonPath::root, + evalSettings.restrictEval || evalSettings.pureEval + ? std::optional>(std::set()) + : std::nullopt, + [](const CanonPath & path) -> RestrictedPathError { + auto modeInformation = evalSettings.pureEval + ? "in pure evaluation mode (use '--impure' to override)" + : "in restricted mode"; + throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); + })) , corepkgsFS(makeMemoryInputAccessor()) , internalFS(makeMemoryInputAccessor()) , derivationInternal{corepkgsFS->addFile( @@ -551,28 +562,10 @@ EvalState::EvalState( searchPath.elements.emplace_back(SearchPath::Elem::parse(i)); } - if (evalSettings.restrictEval || evalSettings.pureEval) { - allowedPaths = PathSet(); - - for (auto & i : searchPath.elements) { - auto r = resolveSearchPathPath(i.path); - if (!r) continue; - - auto path = std::move(*r); - - if (store->isInStore(path)) { - try { - StorePathSet closure; - store->computeFSClosure(store->toStorePath(path).first, closure); - for (auto & path : closure) - allowPath(path); - } catch (InvalidPath &) { - allowPath(path); - } - } else - allowPath(path); - } - } + /* Allow access to all paths in the search path. */ + if (rootFS->hasAccessControl()) + for (auto & i : searchPath.elements) + resolveSearchPathPath(i.path, true); corepkgsFS->addFile( CanonPath("fetchurl.nix"), @@ -590,14 +583,12 @@ EvalState::~EvalState() void EvalState::allowPath(const Path & path) { - if (allowedPaths) - allowedPaths->insert(path); + rootFS->allowPath(CanonPath(path)); } void EvalState::allowPath(const StorePath & storePath) { - if (allowedPaths) - allowedPaths->insert(store->toRealPath(storePath)); + rootFS->allowPath(CanonPath(store->toRealPath(storePath))); } void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v) @@ -607,54 +598,6 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & mkStorePathString(storePath, v); } -SourcePath EvalState::checkSourcePath(const SourcePath & path_) -{ - // Don't check non-rootFS accessors, they're in a different namespace. - if (path_.accessor != ref(rootFS)) return path_; - - if (!allowedPaths) return path_; - - auto i = resolvedPaths.find(path_.path.abs()); - if (i != resolvedPaths.end()) - return i->second; - - bool found = false; - - /* First canonicalize the path without symlinks, so we make sure an - * attacker can't append ../../... to a path that would be in allowedPaths - * and thus leak symlink targets. - */ - Path abspath = canonPath(path_.path.abs()); - - for (auto & i : *allowedPaths) { - if (isDirOrInDir(abspath, i)) { - found = true; - break; - } - } - - if (!found) { - auto modeInformation = evalSettings.pureEval - ? "in pure eval mode (use '--impure' to override)" - : "in restricted mode"; - throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", abspath, modeInformation); - } - - /* Resolve symlinks. */ - debug("checking access to '%s'", abspath); - SourcePath path = rootPath(CanonPath(canonPath(abspath, true))); - - for (auto & i : *allowedPaths) { - if (isDirOrInDir(path.path.abs(), i)) { - resolvedPaths.insert_or_assign(path_.path.abs(), path); - return path; - } - } - - throw RestrictedPathError("access to canonical path '%1%' is forbidden in restricted mode", path); -} - - void EvalState::checkURI(const std::string & uri) { if (!evalSettings.restrictEval) return; @@ -674,12 +617,12 @@ void EvalState::checkURI(const std::string & uri) /* If the URI is a path, then check it against allowedPaths as well. */ if (hasPrefix(uri, "/")) { - checkSourcePath(rootPath(CanonPath(uri))); + rootFS->checkAllowed(CanonPath(uri)); return; } if (hasPrefix(uri, "file://")) { - checkSourcePath(rootPath(CanonPath(std::string(uri, 7)))); + rootFS->checkAllowed(CanonPath(uri.substr(7))); return; } @@ -1181,10 +1124,8 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env) } -void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial) +void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) { - auto path = checkSourcePath(path_); - FileEvalCache::iterator i; if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) { v = i->second; @@ -1205,7 +1146,7 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial e = j->second; if (!e) - e = parseExprFromFile(checkSourcePath(resolvedPath)); + e = parseExprFromFile(resolvedPath); fileParseCache[resolvedPath] = e; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 9a92992c1..ee7bdda0d 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -217,12 +217,6 @@ public: */ RepairFlag repair; - /** - * The allowed filesystem paths in restricted or pure evaluation - * mode. - */ - std::optional allowedPaths; - Bindings emptyBindings; /** @@ -396,12 +390,6 @@ public: */ void allowAndSetStorePathString(const StorePath & storePath, Value & v); - /** - * Check whether access to a path is allowed and throw an error if - * not. Otherwise return the canonicalised path. - */ - SourcePath checkSourcePath(const SourcePath & path); - void checkURI(const std::string & uri); /** @@ -445,13 +433,15 @@ public: SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); /** - * Try to resolve a search path value (not the optional key part) + * Try to resolve a search path value (not the optional key part). * * If the specified search path element is a URI, download it. * * If it is not found, return `std::nullopt` */ - std::optional resolveSearchPathPath(const SearchPath::Path & path); + std::optional resolveSearchPathPath( + const SearchPath::Path & elem, + bool initAccessControl = false); /** * Evaluate an expression to normal form @@ -756,6 +746,13 @@ public: */ [[nodiscard]] StringMap realiseContext(const NixStringContext & context); + /* Call the binary path filter predicate used builtins.path etc. */ + bool callPathFilter( + Value * filterFun, + const SourcePath & path, + std::string_view pathArg, + PosIdx pos); + private: /** diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index f6cf1f689..58fc580fc 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -783,7 +783,7 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_ } -std::optional EvalState::resolveSearchPathPath(const SearchPath::Path & value0) +std::optional EvalState::resolveSearchPathPath(const SearchPath::Path & value0, bool initAccessControl) { auto & value = value0.s; auto i = searchPathResolved.find(value); @@ -800,7 +800,6 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa logWarning({ .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) }); - res = std::nullopt; } } @@ -814,6 +813,20 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa else { auto path = absPath(value); + + /* Allow access to paths in the search path. */ + if (initAccessControl) { + allowPath(path); + if (store->isInStore(path)) { + try { + StorePathSet closure; + store->computeFSClosure(store->toStorePath(path).first, closure); + for (auto & p : closure) + allowPath(p); + } catch (InvalidPath &) { } + } + } + if (pathExists(path)) res = { path }; else { @@ -829,7 +842,7 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa else debug("failed to resolve search path element '%s'", value); - searchPathResolved[value] = res; + searchPathResolved.emplace(value, res); return res; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ebf2549e4..0f7706563 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -15,6 +15,7 @@ #include "value-to-json.hh" #include "value-to-xml.hh" #include "primops.hh" +#include "fs-input-accessor.hh" #include #include @@ -90,8 +91,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context) for (auto & [outputName, outputPath] : outputs) { /* Add the output of this derivations to the allowed paths. */ - if (allowedPaths) { - allowPath(outputPath); + if (rootFS->hasAccessControl()) { + allowPath(store->toRealPath(outputPath)); } /* Get all the output paths corresponding to the placeholders we had */ if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { @@ -110,27 +111,19 @@ StringMap EvalState::realiseContext(const NixStringContext & context) return res; } -struct RealisePathFlags { - // Whether to check that the path is allowed in pure eval mode - bool checkForPureEval = true; -}; - -static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, const RealisePathFlags flags = {}) +static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v) { NixStringContext context; auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path"); try { - if (!context.empty()) { + if (!context.empty() && path.accessor == state.rootFS) { auto rewrites = state.realiseContext(context); auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context); return {path.accessor, CanonPath(realPath)}; - } - - return flags.checkForPureEval - ? state.checkSourcePath(path) - : path; + } else + return path; } catch (Error & e) { e.addTrace(state.positions[pos], "while realising the context of path '%s'", path); throw; @@ -1493,7 +1486,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, })); NixStringContext context; - auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'")).path; + auto path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'").path; /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so e.g. nix-push does the right thing. */ @@ -1535,12 +1528,7 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, { auto & arg = *args[0]; - /* We don’t check the path right now, because we don’t want to - throw if the path isn’t allowed, but just return false (and we - can’t just catch the exception here because we still want to - throw if something in the evaluation of `arg` tries to - access an unauthorized path). */ - auto path = realisePath(state, pos, arg, { .checkForPureEval = false }); + auto path = realisePath(state, pos, arg); /* SourcePath doesn't know about trailing slash. */ auto mustBeDir = arg.type() == nString @@ -1548,14 +1536,9 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, || arg.string_view().ends_with("/.")); try { - auto checked = state.checkSourcePath(path); - auto st = checked.maybeLstat(); + auto st = path.maybeLstat(); auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory); v.mkBool(exists); - } catch (SysError & e) { - /* Don't give away info from errors while canonicalising - ‘path’ in restricted mode. */ - v.mkBool(false); } catch (RestrictedPathError & e) { v.mkBool(false); } @@ -1699,7 +1682,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V auto path = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.findFile"); - v.mkPath(state.checkSourcePath(state.findFile(searchPath, path, pos))); + v.mkPath(state.findFile(searchPath, path, pos)); } static RegisterPrimOp primop_findFile(PrimOp { @@ -2178,11 +2161,35 @@ static RegisterPrimOp primop_toFile({ .fun = prim_toFile, }); +bool EvalState::callPathFilter( + Value * filterFun, + const SourcePath & path, + std::string_view pathArg, + PosIdx pos) +{ + auto st = path.lstat(); + + /* Call the filter function. The first argument is the path, the + second is a string indicating the type of the file. */ + Value arg1; + arg1.mkString(pathArg); + + Value arg2; + // assert that type is not "unknown" + arg2.mkString(fileTypeToString(st.type)); + + Value * args []{&arg1, &arg2}; + Value res; + callFunction(*filterFun, 2, args, res, pos); + + return forceBool(res, pos, "while evaluating the return value of the path filter function"); +} + static void addPath( EvalState & state, const PosIdx pos, std::string_view name, - Path path, + SourcePath path, Value * filterFun, FileIngestionMethod method, const std::optional expectedHash, @@ -2190,48 +2197,29 @@ static void addPath( const NixStringContext & context) { try { - // FIXME: handle CA derivation outputs (where path needs to - // be rewritten to the actual output). - auto rewrites = state.realiseContext(context); - path = state.toRealPath(rewriteStrings(path, rewrites), context); - StorePathSet refs; - if (state.store->isInStore(path)) { + if (path.accessor == state.rootFS && state.store->isInStore(path.path.abs())) { + // FIXME: handle CA derivation outputs (where path needs to + // be rewritten to the actual output). + auto rewrites = state.realiseContext(context); + path = {state.rootFS, CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context))}; + try { - auto [storePath, subPath] = state.store->toStorePath(path); + auto [storePath, subPath] = state.store->toStorePath(path.path.abs()); // FIXME: we should scanForReferences on the path before adding it refs = state.store->queryPathInfo(storePath)->references; - path = state.store->toRealPath(storePath) + subPath; + path = {state.rootFS, CanonPath(state.store->toRealPath(storePath) + subPath)}; } catch (Error &) { // FIXME: should be InvalidPathError } } - path = evalSettings.pureEval && expectedHash - ? path - : state.checkSourcePath(state.rootPath(CanonPath(path))).path.abs(); - - PathFilter filter = filterFun ? ([&](const Path & path) { - auto st = lstat(path); - - /* Call the filter function. The first argument is the path, - the second is a string indicating the type of the file. */ - Value arg1; - arg1.mkString(path); - - Value arg2; - arg2.mkString( - S_ISREG(st.st_mode) ? "regular" : - S_ISDIR(st.st_mode) ? "directory" : - S_ISLNK(st.st_mode) ? "symlink" : - "unknown" /* not supported, will fail! */); - - Value * args []{&arg1, &arg2}; - Value res; - state.callFunction(*filterFun, 2, args, res, pos); - - return state.forceBool(res, pos, "while evaluating the return value of the path filter function"); - }) : defaultPathFilter; + std::unique_ptr filter; + if (filterFun) + filter = std::make_unique([&](const Path & p) { + auto p2 = CanonPath(p); + return state.callPathFilter(filterFun, {path.accessor, p2}, p2.abs(), pos); + }); std::optional expectedStorePath; if (expectedHash) @@ -2242,7 +2230,7 @@ static void addPath( }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, method, &filter, state.repair); + auto dstPath = path.fetchToStore(state.store, name, method, filter.get(), state.repair); if (expectedHash && expectedStorePath != dstPath) state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); state.allowAndSetStorePathString(dstPath, v); @@ -2261,7 +2249,8 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg auto path = state.coerceToPath(pos, *args[1], context, "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); - addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); + + addPath(state, pos, path.baseName(), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); } static RegisterPrimOp primop_filterSource({ @@ -2356,7 +2345,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value if (name.empty()) name = path->baseName(); - addPath(state, pos, name, path->path.abs(), filterFun, method, expectedHash, v, context); + addPath(state, pos, name, *path, filterFun, method, expectedHash, v, context); } static RegisterPrimOp primop_path({ diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 75ce12a8c..e2986bfe0 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -310,8 +310,11 @@ static void main_nix_build(int argc, char * * argv) else /* If we're in a #! script, interpret filenames relative to the script. */ - exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, - inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))))); + exprs.push_back( + state->parseExprFromFile( + resolveExprPath( + lookupFileArg(*state, + inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i)))); } } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index c67409e89..86b9be17d 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -183,7 +183,7 @@ static int main_nix_instantiate(int argc, char * * argv) for (auto & i : files) { Expr * e = fromArgs ? state->parseExprFromString(i, state->rootPath(CanonPath::fromCwd())) - : state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, i)))); + : state->parseExprFromFile(resolveExprPath(lookupFileArg(*state, i))); processExpr(*state, attrPaths, parseOnly, strict, autoArgs, evalOnly, outputKind, xmlOutputSourceLocation, e); } diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index 197ae7a10..b8deceacc 100644 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -14,8 +14,8 @@ nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I sr (! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel') nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel' -I src=../../src -(! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ') -nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ' -I src=. +(! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ') +nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' -I src=. p=$(nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)") cmp $p restricted.sh From 305939655a6cd680997981ca6077d4ce7f957984 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Nov 2023 16:28:33 +0100 Subject: [PATCH 696/909] Remove superfluous use of hasAccessControl() --- src/libexpr/primops.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 0f7706563..c442de986 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -91,9 +91,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context) for (auto & [outputName, outputPath] : outputs) { /* Add the output of this derivations to the allowed paths. */ - if (rootFS->hasAccessControl()) { - allowPath(store->toRealPath(outputPath)); - } + allowPath(store->toRealPath(outputPath)); + /* Get all the output paths corresponding to the placeholders we had */ if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { res.insert_or_assign( From 43d9fb6cf180c421be17b4247f5dd032cf4843f5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Nov 2023 16:44:54 +0100 Subject: [PATCH 697/909] Remove InputAccessor::root() --- src/libexpr/value.hh | 7 +++---- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/input-accessor.cc | 7 +------ src/libfetchers/input-accessor.hh | 7 +++++-- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index bcff8ae55..72a3a2b32 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -424,10 +424,9 @@ public: SourcePath path() const { assert(internalType == tPath); - return SourcePath { - .accessor = ref(_path.accessor->shared_from_this()), - .path = CanonPath(CanonPath::unchecked_t(), _path.path) - }; + return SourcePath( + ref(_path.accessor->shared_from_this()), + CanonPath(CanonPath::unchecked_t(), _path.path)); } std::string_view string_view() const diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 60208619e..5fd9e069f 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -374,7 +374,7 @@ void InputScheme::clone(const Input & input, const Path & destDir) const std::pair InputScheme::fetch(ref store, const Input & input) { auto [accessor, input2] = getAccessor(store, input); - auto storePath = accessor->root().fetchToStore(store, input2.getName()); + auto storePath = SourcePath(accessor).fetchToStore(store, input2.getName()); return {storePath, input2}; } diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index 85dc4609f..f54a5a6fd 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -53,11 +53,6 @@ StorePath InputAccessor::fetchToStore( return storePath; } -SourcePath InputAccessor::root() -{ - return {ref(shared_from_this()), CanonPath::root}; -} - std::ostream & operator << (std::ostream & str, const SourcePath & path) { str << path.to_string(); @@ -88,7 +83,7 @@ SourcePath SourcePath::parent() const SourcePath SourcePath::resolveSymlinks() const { - auto res = accessor->root(); + auto res = SourcePath(accessor); int linksAllowed = 1024; diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 26d17f064..d5ac238b1 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -36,8 +36,6 @@ struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this accessor; CanonPath path; + SourcePath(ref accessor, CanonPath path = CanonPath::root) + : accessor(std::move(accessor)) + , path(std::move(path)) + { } + std::string_view baseName() const; /** From be30c2ea8de7f12d610b695bd7c6edf22b32fe55 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Wed, 8 Nov 2023 17:52:22 -0800 Subject: [PATCH 698/909] Don't attempt to `git add` ignored files This uses `git check-ignore` to determine if files are ignored before attempting to add them in `putFile`. We also add a condition to the `fetchFromWorkdir` filter to always add the `flake.lock` file, even if it's not tracked. This is necessary to resolve inputs. This fixes #8854 without `git add --force`. --- src/libfetchers/git.cc | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 8cd74057c..734c29258 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -313,15 +313,26 @@ struct GitInputScheme : InputScheme writeFile((CanonPath(repoInfo.url) + path).abs(), contents); - runProgram("git", true, - { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) }); + auto result = runProgram(RunOptions { + .program = "git", + .args = {"-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "check-ignore", "--quiet", std::string(path.rel())}, + }); + auto exitCode = WEXITSTATUS(result.first); - // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit` - logger->pause(); - Finally restoreLogger([]() { logger->resume(); }); - if (commitMsg) + if (exitCode != 0) { + // The path is not `.gitignore`d, we can add the file. runProgram("git", true, - { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-m", *commitMsg }); + { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) }); + + + if (commitMsg) { + // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit` + logger->pause(); + Finally restoreLogger([]() { logger->resume(); }); + runProgram("git", true, + { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-m", *commitMsg }); + } + } } struct RepoInfo From 44d21f6ef9783bed8812d39ff7b1a28a4883f84b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 30 Nov 2023 20:36:50 +0100 Subject: [PATCH 699/909] keep generated documentation in a separate directory - helps navigating the code as it highlights which files are generated - makes it less error prone when working incrementally (although this should be just fixed by building out of tree) --- .gitignore | 2 +- .../manual/src/store/types/index.md.in | 4 ---- src/nix/local.mk | 19 ++++++++++++++++--- src/nix/main.cc | 2 +- src/nix/profile.md | 2 +- 5 files changed, 19 insertions(+), 10 deletions(-) rename src/nix/help-stores.md => doc/manual/src/store/types/index.md.in (99%) diff --git a/.gitignore b/.gitignore index 13a4dfb75..bcf9c4a01 100644 --- a/.gitignore +++ b/.gitignore @@ -54,7 +54,7 @@ perl/Makefile.config /src/nix/nix -/src/nix/doc +/src/nix/generated-doc # /src/nix-env/ /src/nix-env/nix-env diff --git a/src/nix/help-stores.md b/doc/manual/src/store/types/index.md.in similarity index 99% rename from src/nix/help-stores.md rename to doc/manual/src/store/types/index.md.in index 47ba9b94d..bb166a1fc 100644 --- a/src/nix/help-stores.md +++ b/doc/manual/src/store/types/index.md.in @@ -1,5 +1,3 @@ -R"( - Nix supports different types of stores. These are described below. ## Store URL format @@ -42,5 +40,3 @@ store as follows: * Otherwise, use the [local store](#local-store) `/nix/store`. @stores@ - -)" diff --git a/src/nix/local.mk b/src/nix/local.mk index 57f8259c4..a21aa705f 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -31,10 +31,23 @@ src/nix/develop.cc: src/nix/get-env.sh.gen.hh src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh -src/nix/main.cc: doc/manual/generate-manpage.nix.gen.hh doc/manual/utils.nix.gen.hh doc/manual/generate-settings.nix.gen.hh doc/manual/generate-store-info.nix.gen.hh +src/nix/main.cc: \ + doc/manual/generate-manpage.nix.gen.hh \ + doc/manual/utils.nix.gen.hh doc/manual/generate-settings.nix.gen.hh \ + doc/manual/generate-store-info.nix.gen.hh \ + src/nix/generated-doc/help-stores.md -src/nix/doc/files/%.md: doc/manual/src/command-ref/files/%.md +src/nix/generated-doc/files/%.md: doc/manual/src/command-ref/files/%.md @mkdir -p $$(dirname $@) @cp $< $@ -src/nix/profile.cc: src/nix/profile.md src/nix/doc/files/profiles.md.gen.hh +src/nix/profile.cc: src/nix/profile.md src/nix/generated-doc/files/profiles.md.gen.hh + +src/nix/generated-doc/help-stores.md: doc/manual/src/store/types/index.md.in + @mkdir -p $$(dirname $@) + @echo 'R"(' >> $@.tmp + @echo >> $@.tmp + @cat $^ >> $@.tmp + @echo >> $@.tmp + @echo ')"' >> $@.tmp + @mv $@.tmp $@ diff --git a/src/nix/main.cc b/src/nix/main.cc index 2a6c2f478..49e637fb0 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -297,7 +297,7 @@ struct CmdHelpStores : Command std::string doc() override { return - #include "help-stores.md" + #include "generated-doc/help-stores.md" ; } diff --git a/src/nix/profile.md b/src/nix/profile.md index bd13f906f..9b2f86f4a 100644 --- a/src/nix/profile.md +++ b/src/nix/profile.md @@ -11,7 +11,7 @@ them to be rolled back easily. )"" -#include "doc/files/profiles.md.gen.hh" +#include "generated-doc/files/profiles.md.gen.hh" R""( From d5ffc94f336fc4032dd4009c14c148e390e10e16 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 30 Nov 2023 21:41:47 +0100 Subject: [PATCH 700/909] use lookup paths in helper expressions consistently this makes the files in question a bit more independent of source location. to find where the value is set and how it's wired up: rg nix=doc/manual --- doc/manual/generate-builtin-constants.nix | 2 +- doc/manual/generate-builtins.nix | 2 +- doc/manual/generate-manpage.nix | 30 +++++++++++++++---- doc/manual/generate-settings.nix | 2 +- doc/manual/generate-store-info.nix | 4 +-- doc/manual/generate-xp-features-shortlist.nix | 2 +- doc/manual/generate-xp-features.nix | 2 +- 7 files changed, 32 insertions(+), 12 deletions(-) diff --git a/doc/manual/generate-builtin-constants.nix b/doc/manual/generate-builtin-constants.nix index 8af80a02c..cccd1e279 100644 --- a/doc/manual/generate-builtin-constants.nix +++ b/doc/manual/generate-builtin-constants.nix @@ -1,6 +1,6 @@ let inherit (builtins) concatStringsSep attrValues mapAttrs; - inherit (import ./utils.nix) optionalString squash; + inherit (import ) optionalString squash; in builtinsInfo: diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 813a287f5..05cae1c46 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -1,6 +1,6 @@ let inherit (builtins) concatStringsSep attrValues mapAttrs; - inherit (import ./utils.nix) optionalString squash; + inherit (import ) optionalString squash; in builtinsInfo: diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 14136016d..c4b9d1335 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -1,9 +1,29 @@ let inherit (builtins) - attrNames attrValues fromJSON listToAttrs mapAttrs groupBy - concatStringsSep concatMap length lessThan replaceStrings sort; - inherit (import ) attrsToList concatStrings optionalString filterAttrs trim squash unique; - showStoreDocs = import ./generate-store-info.nix; + attrNames + attrValues + concatMap + concatStringsSep + fromJSON + groupBy + length + lessThan + listToAttrs + mapAttrs + match + replaceStrings + sort + ; + inherit (import ) + attrsToList + concatStrings + filterAttrs + optionalString + squash + trim + unique + ; + showStoreDocs = import ; in inlineHTML: commandDump: @@ -97,7 +117,7 @@ let ${optionalString (cat != "") "## ${cat}"} ${concatStringsSep "\n" (attrValues (mapAttrs showOption opts))} - ''; + ''; showOption = name: option: let result = trim '' diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 8736bb793..3add10075 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -1,6 +1,6 @@ let inherit (builtins) attrValues concatStringsSep isAttrs isBool mapAttrs; - inherit (import ./utils.nix) concatStrings indent optionalString squash; + inherit (import ) concatStrings indent optionalString squash; in # `inlineHTML` is a hack to accommodate inconsistent output from `lowdown` diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index 36215aadf..73defcb71 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -1,7 +1,7 @@ let inherit (builtins) attrValues mapAttrs; - inherit (import ./utils.nix) concatStrings optionalString; - showSettings = import ./generate-settings.nix; + inherit (import ) concatStrings optionalString; + showSettings = import ; in inlineHTML: storesInfo: diff --git a/doc/manual/generate-xp-features-shortlist.nix b/doc/manual/generate-xp-features-shortlist.nix index 30e211c96..ec09f4b75 100644 --- a/doc/manual/generate-xp-features-shortlist.nix +++ b/doc/manual/generate-xp-features-shortlist.nix @@ -1,5 +1,5 @@ with builtins; -with import ./utils.nix; +with import ; let showExperimentalFeature = name: doc: diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix index adb94355c..fc7c7d4cf 100644 --- a/doc/manual/generate-xp-features.nix +++ b/doc/manual/generate-xp-features.nix @@ -1,5 +1,5 @@ with builtins; -with import ./utils.nix; +with import ; let showExperimentalFeature = name: doc: From 8cafc754d845529a78595d1196769257ee23ca56 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Nov 2023 21:54:53 +0100 Subject: [PATCH 701/909] Move access control from FSInputAccessor to FilteringInputAccessor --- src/libexpr/eval.cc | 23 +++--- src/libexpr/eval.hh | 3 +- src/libfetchers/filtering-input-accessor.cc | 83 +++++++++++++++++++++ src/libfetchers/filtering-input-accessor.hh | 73 ++++++++++++++++++ src/libfetchers/fs-input-accessor.cc | 77 +++---------------- src/libfetchers/fs-input-accessor.hh | 22 +----- src/libfetchers/git.cc | 6 +- 7 files changed, 191 insertions(+), 96 deletions(-) create mode 100644 src/libfetchers/filtering-input-accessor.cc create mode 100644 src/libfetchers/filtering-input-accessor.hh diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 23ac349fe..841c223cd 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -14,6 +14,7 @@ #include "profiles.hh" #include "print.hh" #include "fs-input-accessor.hh" +#include "filtering-input-accessor.hh" #include "memory-input-accessor.hh" #include "signals.hh" #include "gc-small-vector.hh" @@ -510,17 +511,15 @@ EvalState::EvalState( , repair(NoRepair) , emptyBindings(0) , rootFS( - makeFSInputAccessor( - CanonPath::root, - evalSettings.restrictEval || evalSettings.pureEval - ? std::optional>(std::set()) - : std::nullopt, + evalSettings.restrictEval || evalSettings.pureEval + ? ref(AllowListInputAccessor::create(makeFSInputAccessor(CanonPath::root), {}, [](const CanonPath & path) -> RestrictedPathError { auto modeInformation = evalSettings.pureEval ? "in pure evaluation mode (use '--impure' to override)" : "in restricted mode"; throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); })) + : makeFSInputAccessor(CanonPath::root)) , corepkgsFS(makeMemoryInputAccessor()) , internalFS(makeMemoryInputAccessor()) , derivationInternal{corepkgsFS->addFile( @@ -563,7 +562,7 @@ EvalState::EvalState( } /* Allow access to all paths in the search path. */ - if (rootFS->hasAccessControl()) + if (rootFS.dynamic_pointer_cast()) for (auto & i : searchPath.elements) resolveSearchPathPath(i.path, true); @@ -583,12 +582,14 @@ EvalState::~EvalState() void EvalState::allowPath(const Path & path) { - rootFS->allowPath(CanonPath(path)); + if (auto rootFS2 = rootFS.dynamic_pointer_cast()) + rootFS2->allowPath(CanonPath(path)); } void EvalState::allowPath(const StorePath & storePath) { - rootFS->allowPath(CanonPath(store->toRealPath(storePath))); + if (auto rootFS2 = rootFS.dynamic_pointer_cast()) + rootFS2->allowPath(CanonPath(store->toRealPath(storePath))); } void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v) @@ -617,12 +618,14 @@ void EvalState::checkURI(const std::string & uri) /* If the URI is a path, then check it against allowedPaths as well. */ if (hasPrefix(uri, "/")) { - rootFS->checkAllowed(CanonPath(uri)); + if (auto rootFS2 = rootFS.dynamic_pointer_cast()) + rootFS2->checkAccess(CanonPath(uri)); return; } if (hasPrefix(uri, "file://")) { - rootFS->checkAllowed(CanonPath(uri.substr(7))); + if (auto rootFS2 = rootFS.dynamic_pointer_cast()) + rootFS2->checkAccess(CanonPath(uri.substr(7))); return; } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index ee7bdda0d..f3f6d35b9 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -30,7 +30,6 @@ class EvalState; class StorePath; struct SingleDerivedPath; enum RepairFlag : bool; -struct FSInputAccessor; struct MemoryInputAccessor; @@ -222,7 +221,7 @@ public: /** * The accessor for the root filesystem. */ - const ref rootFS; + const ref rootFS; /** * The in-memory filesystem for paths. diff --git a/src/libfetchers/filtering-input-accessor.cc b/src/libfetchers/filtering-input-accessor.cc new file mode 100644 index 000000000..5ae416fd3 --- /dev/null +++ b/src/libfetchers/filtering-input-accessor.cc @@ -0,0 +1,83 @@ +#include "filtering-input-accessor.hh" + +namespace nix { + +std::string FilteringInputAccessor::readFile(const CanonPath & path) +{ + checkAccess(path); + return next->readFile(prefix + path); +} + +bool FilteringInputAccessor::pathExists(const CanonPath & path) +{ + return isAllowed(path) && next->pathExists(prefix + path); +} + +std::optional FilteringInputAccessor::maybeLstat(const CanonPath & path) +{ + checkAccess(path); + return next->maybeLstat(prefix + path); +} + +InputAccessor::DirEntries FilteringInputAccessor::readDirectory(const CanonPath & path) +{ + checkAccess(path); + DirEntries entries; + for (auto & entry : next->readDirectory(prefix + path)) { + if (isAllowed(path + entry.first)) + entries.insert(std::move(entry)); + } + return entries; +} + +std::string FilteringInputAccessor::readLink(const CanonPath & path) +{ + checkAccess(path); + return next->readLink(prefix + path); +} + +std::string FilteringInputAccessor::showPath(const CanonPath & path) +{ + return next->showPath(prefix + path); +} + +void FilteringInputAccessor::checkAccess(const CanonPath & path) +{ + if (!isAllowed(path)) + throw makeNotAllowedError + ? makeNotAllowedError(path) + : RestrictedPathError("access to path '%s' is forbidden", showPath(path)); +} + +struct AllowListInputAccessorImpl : AllowListInputAccessor +{ + std::set allowedPaths; + + AllowListInputAccessorImpl( + ref next, + std::set && allowedPaths, + MakeNotAllowedError && makeNotAllowedError) + : AllowListInputAccessor(SourcePath(next), std::move(makeNotAllowedError)) + , allowedPaths(std::move(allowedPaths)) + { } + + bool isAllowed(const CanonPath & path) override + { + return path.isAllowed(allowedPaths); + } + + void allowPath(CanonPath path) override + { + allowedPaths.insert(std::move(path)); + } +}; + +ref AllowListInputAccessor::create( + ref next, + std::set && allowedPaths, + MakeNotAllowedError && makeNotAllowedError) +{ + return make_ref(next, std::move(allowedPaths), std::move(makeNotAllowedError)); +} + +} diff --git a/src/libfetchers/filtering-input-accessor.hh b/src/libfetchers/filtering-input-accessor.hh new file mode 100644 index 000000000..209d26974 --- /dev/null +++ b/src/libfetchers/filtering-input-accessor.hh @@ -0,0 +1,73 @@ +#pragma once + +#include "input-accessor.hh" + +namespace nix { + +/** + * A function that should throw an exception of type + * `RestrictedPathError` explaining that access to `path` is + * forbidden. + */ +typedef std::function MakeNotAllowedError; + +/** + * An abstract wrapping `InputAccessor` that performs access + * control. Subclasses should override `checkAccess()` to implement an + * access control policy. + */ +struct FilteringInputAccessor : InputAccessor +{ + ref next; + CanonPath prefix; + MakeNotAllowedError makeNotAllowedError; + + FilteringInputAccessor(const SourcePath & src, MakeNotAllowedError && makeNotAllowedError) + : next(src.accessor) + , prefix(src.path) + , makeNotAllowedError(std::move(makeNotAllowedError)) + { } + + std::string readFile(const CanonPath & path) override; + + bool pathExists(const CanonPath & path) override; + + std::optional maybeLstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::string showPath(const CanonPath & path) override; + + /** + * Call `makeNotAllowedError` to throw a `RestrictedPathError` + * exception if `isAllowed()` returns `false` for `path`. + */ + void checkAccess(const CanonPath & path); + + /** + * Return `true` iff access to path is allowed. + */ + virtual bool isAllowed(const CanonPath & path) = 0; +}; + +/** + * A wrapping `InputAccessor` that checks paths against an allow-list. + */ +struct AllowListInputAccessor : public FilteringInputAccessor +{ + /** + * Grant access to the specified path. + */ + virtual void allowPath(CanonPath path) = 0; + + static ref create( + ref next, + std::set && allowedPaths, + MakeNotAllowedError && makeNotAllowedError); + + using FilteringInputAccessor::FilteringInputAccessor; +}; + +} diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index 2efee932d..c3d8d273c 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -4,19 +4,12 @@ namespace nix { -struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor +struct FSInputAccessor : InputAccessor, PosixSourceAccessor { CanonPath root; - std::optional> allowedPaths; - MakeNotAllowedError makeNotAllowedError; - FSInputAccessorImpl( - const CanonPath & root, - std::optional> && allowedPaths, - MakeNotAllowedError && makeNotAllowedError) + FSInputAccessor(const CanonPath & root) : root(root) - , allowedPaths(std::move(allowedPaths)) - , makeNotAllowedError(std::move(makeNotAllowedError)) { displayPrefix = root.isRoot() ? "" : root.abs(); } @@ -27,39 +20,30 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor std::function sizeCallback) override { auto absPath = makeAbsPath(path); - checkAllowed(absPath); PosixSourceAccessor::readFile(absPath, sink, sizeCallback); } bool pathExists(const CanonPath & path) override { - auto absPath = makeAbsPath(path); - return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath); + return PosixSourceAccessor::pathExists(makeAbsPath(path)); } std::optional maybeLstat(const CanonPath & path) override { - auto absPath = makeAbsPath(path); - checkAllowed(absPath); - return PosixSourceAccessor::maybeLstat(absPath); + return PosixSourceAccessor::maybeLstat(makeAbsPath(path)); } DirEntries readDirectory(const CanonPath & path) override { - auto absPath = makeAbsPath(path); - checkAllowed(absPath); DirEntries res; - for (auto & entry : PosixSourceAccessor::readDirectory(absPath)) - if (isAllowed(absPath + entry.first)) - res.emplace(entry); + for (auto & entry : PosixSourceAccessor::readDirectory(makeAbsPath(path))) + res.emplace(entry); return res; } std::string readLink(const CanonPath & path) override { - auto absPath = makeAbsPath(path); - checkAllowed(absPath); - return PosixSourceAccessor::readLink(absPath); + return PosixSourceAccessor::readLink(makeAbsPath(path)); } CanonPath makeAbsPath(const CanonPath & path) @@ -67,59 +51,22 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor return root + path; } - void checkAllowed(const CanonPath & absPath) override - { - if (!isAllowed(absPath)) - throw makeNotAllowedError - ? makeNotAllowedError(absPath) - : RestrictedPathError("access to path '%s' is forbidden", absPath); - } - - bool isAllowed(const CanonPath & absPath) - { - if (!absPath.isWithin(root)) - return false; - - if (allowedPaths) { - auto p = absPath.removePrefix(root); - if (!p.isAllowed(*allowedPaths)) - return false; - } - - return true; - } - - void allowPath(CanonPath path) override - { - if (allowedPaths) - allowedPaths->insert(std::move(path)); - } - - bool hasAccessControl() override - { - return (bool) allowedPaths; - } - std::optional getPhysicalPath(const CanonPath & path) override { return makeAbsPath(path); } }; -ref makeFSInputAccessor( - const CanonPath & root, - std::optional> && allowedPaths, - MakeNotAllowedError && makeNotAllowedError) +ref makeFSInputAccessor(const CanonPath & root) { - return make_ref(root, std::move(allowedPaths), std::move(makeNotAllowedError)); + return make_ref(root); } -ref makeStorePathAccessor( +ref makeStorePathAccessor( ref store, - const StorePath & storePath, - MakeNotAllowedError && makeNotAllowedError) + const StorePath & storePath) { - return makeFSInputAccessor(CanonPath(store->toRealPath(storePath)), {}, std::move(makeNotAllowedError)); + return makeFSInputAccessor(CanonPath(store->toRealPath(storePath))); } SourcePath getUnfilteredRootPath(CanonPath path) diff --git a/src/libfetchers/fs-input-accessor.hh b/src/libfetchers/fs-input-accessor.hh index 19a5211c8..ba5af5887 100644 --- a/src/libfetchers/fs-input-accessor.hh +++ b/src/libfetchers/fs-input-accessor.hh @@ -7,26 +7,12 @@ namespace nix { class StorePath; class Store; -struct FSInputAccessor : InputAccessor -{ - virtual void checkAllowed(const CanonPath & absPath) = 0; +ref makeFSInputAccessor( + const CanonPath & root); - virtual void allowPath(CanonPath path) = 0; - - virtual bool hasAccessControl() = 0; -}; - -typedef std::function MakeNotAllowedError; - -ref makeFSInputAccessor( - const CanonPath & root, - std::optional> && allowedPaths = {}, - MakeNotAllowedError && makeNotAllowedError = {}); - -ref makeStorePathAccessor( +ref makeStorePathAccessor( ref store, - const StorePath & storePath, - MakeNotAllowedError && makeNotAllowedError = {}); + const StorePath & storePath); SourcePath getUnfilteredRootPath(CanonPath path); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 8cd74057c..ff4b1e823 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -9,6 +9,7 @@ #include "processes.hh" #include "git.hh" #include "fs-input-accessor.hh" +#include "filtering-input-accessor.hh" #include "mounted-input-accessor.hh" #include "git-utils.hh" #include "logging.hh" @@ -639,7 +640,10 @@ struct GitInputScheme : InputScheme repoInfo.workdirInfo.files.insert(submodule.path); ref accessor = - makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url)); + AllowListInputAccessor::create( + makeFSInputAccessor(CanonPath(repoInfo.url)), + std::move(repoInfo.workdirInfo.files), + makeNotAllowedError(repoInfo.url)); /* If the repo has submodules, return a mounted input accessor consisting of the accessor for the top-level repo and the From cab41025d85d3b02f5175cf7ca2611c7a44c2cdd Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 30 Nov 2023 23:04:05 +0100 Subject: [PATCH 702/909] mention renaming of `nix doctor` --- doc/manual/rl-next/nix-config-show.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/rl-next/nix-config-show.md b/doc/manual/rl-next/nix-config-show.md index 08ad207cb..b2ad3c666 100644 --- a/doc/manual/rl-next/nix-config-show.md +++ b/doc/manual/rl-next/nix-config-show.md @@ -3,6 +3,6 @@ issues: #7672 prs: #9477 description: { -`nix show-config` was renamed to `nix config show` to be more consistent with the rest of the command-line interface. +`nix show-config` was renamed to `nix config show`, and `nix doctor` was renamed to `nix config check`, to be more consistent with the rest of the command-line interface. } From 39de819edaef2dc3e308a490bbb2b1622a932771 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 9 Oct 2023 10:08:22 +0200 Subject: [PATCH 703/909] rename debugging helper environment variable --- src/libutil/error.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 8488e7e21..72c346cb5 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -159,11 +159,11 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR /** * A development aid for finding missing positions, to improve error messages. Example use: * - * NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test + * _NIX_EVAL_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test * git diff -U20 tests * */ -static bool printUnknownLocations = getEnv("_NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS").has_value(); +static bool printUnknownLocations = getEnv("_NIX_EVAL_SHOW_UNKNOWN_LOCATIONS").has_value(); /** * Print a position, if it is known. From 0301b8fc7354a94dc03b57f796bfa6e853758af8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 30 Nov 2023 21:59:08 +0100 Subject: [PATCH 704/909] reword the experimental feature notice - put the highlight box around all the relevant instructions - simplify the wording - make the link more prominent by using the whole phrase for the link text --- doc/manual/generate-settings.nix | 21 ++++++------- doc/manual/generate-store-info.nix | 48 +++++++++++++++--------------- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 3add10075..74446b70b 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -31,18 +31,19 @@ let experimentalFeatureNote = optionalString (experimentalFeature != null) '' > **Warning** + > > This setting is part of an > [experimental feature](@docroot@/contributing/experimental-features.md). - - To change this setting, you need to make sure the corresponding experimental feature, - [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), - is enabled. - For example, include the following in [`nix.conf`](#): - - ``` - extra-experimental-features = ${experimentalFeature} - ${setting} = ... - ``` + > + > To change this setting, make sure the + > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}) + > is enabled. + > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): + > + > ``` + > extra-experimental-features = ${experimentalFeature} + > ${setting} = ... + > ``` ''; showDefault = documentDefault: defaultValue: diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index 73defcb71..8e26edb65 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -1,6 +1,6 @@ let inherit (builtins) attrValues mapAttrs; - inherit (import ) concatStrings optionalString; + inherit (import ) concatStrings optionalString squash; showSettings = import ; in @@ -10,36 +10,36 @@ let showStore = name: { settings, doc, experimentalFeature }: let + result = squash '' + # ${name} - result = '' - ## ${name} + ${doc} - ${doc} + ${experimentalFeatureNote} - ${experimentalFeatureNote} + ## Settings - ### Settings - - ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings} - ''; + ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings} + ''; # markdown doesn't like spaces in URLs slug = builtins.replaceStrings [ " " ] [ "-" ] name; - experimentalFeatureNote = optionalString (experimentalFeature != null) '' - > **Warning** - > This store is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). - - To use this store, you need to make sure the corresponding experimental feature, - [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), - is enabled. - For example, include the following in [`nix.conf`](#): - - ``` - extra-experimental-features = ${experimentalFeature} - ``` - ''; - in result; + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > + > This store is part of an + > [experimental feature](@docroot@/contributing/experimental-features.md). + > + > To use this store, make sure the + > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}) + > is enabled. + > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): + > + > ``` + > extra-experimental-features = ${experimentalFeature} + > ``` + ''; + in result; in concatStrings (attrValues (mapAttrs showStore storesInfo)) From c982198485a995d40b01b8caf62df5458046614d Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Thu, 30 Nov 2023 22:48:44 +0000 Subject: [PATCH 705/909] First step --- binary-tarball.nix | 81 ++++++++++++++ flake.nix | 262 +++++++++++++-------------------------------- lowdown.nix | 22 ++++ package.nix | 239 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 415 insertions(+), 189 deletions(-) create mode 100644 binary-tarball.nix create mode 100644 lowdown.nix create mode 100644 package.nix diff --git a/binary-tarball.nix b/binary-tarball.nix new file mode 100644 index 000000000..1fa185519 --- /dev/null +++ b/binary-tarball.nix @@ -0,0 +1,81 @@ +{ runCommand +, version +, system +, nix +, cacert +}: + +let + + installerClosureInfo = buildPackages.closureInfo { + rootPaths = [ nix cacert ]; + }; + + env = { + meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; + }; + +in + +runCommand "nix-binary-tarball-${version}" env '' + cp ${installerClosureInfo}/registration $TMPDIR/reginfo + cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh + substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + + substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + + if type -p shellcheck; then + # SC1090: Don't worry about not being able to find + # $nix/etc/profile.d/nix.sh + shellcheck --exclude SC1090 $TMPDIR/install + shellcheck $TMPDIR/create-darwin-volume.sh + shellcheck $TMPDIR/install-darwin-multi-user.sh + shellcheck $TMPDIR/install-systemd-multi-user.sh + + # SC1091: Don't panic about not being able to source + # /etc/profile + # SC2002: Ignore "useless cat" "error", when loading + # .reginfo, as the cat is a much cleaner + # implementation, even though it is "useless" + # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving + # root's home directory + shellcheck --external-sources \ + --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user + fi + + chmod +x $TMPDIR/install + chmod +x $TMPDIR/create-darwin-volume.sh + chmod +x $TMPDIR/install-darwin-multi-user.sh + chmod +x $TMPDIR/install-systemd-multi-user.sh + chmod +x $TMPDIR/install-multi-user + dir=nix-${version}-${system} + fn=$out/$dir.tar.xz + mkdir -p $out/nix-support + echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products + tar cvfJ $fn \ + --owner=0 --group=0 --mode=u+rw,uga+r \ + --mtime='1970-01-01' \ + --absolute-names \ + --hard-dereference \ + --transform "s,$TMPDIR/install,$dir/install," \ + --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \ + --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ + --transform "s,$NIX_STORE,$dir/store,S" \ + $TMPDIR/install \ + $TMPDIR/create-darwin-volume.sh \ + $TMPDIR/install-darwin-multi-user.sh \ + $TMPDIR/install-systemd-multi-user.sh \ + $TMPDIR/install-multi-user \ + $TMPDIR/reginfo \ + $(cat ${installerClosureInfo}/store-paths) +'' diff --git a/flake.nix b/flake.nix index 33673575b..a1fc1cd1c 100644 --- a/flake.nix +++ b/flake.nix @@ -7,7 +7,7 @@ inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; - outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, flake-compat, libgit2 }: + outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, libgit2 }: let inherit (nixpkgs) lib; @@ -34,7 +34,14 @@ "x86_64-freebsd13" "x86_64-netbsd" ]; - stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ]; + stdenvs = [ + "ccacheStdenv" + "clang11Stdenv" + "clangStdenv" + "gccStdenv" + "libcxxStdenv" + "stdenv" + ]; forAllSystems = lib.genAttrs systems; @@ -326,82 +333,18 @@ ''; }; - binaryTarball = nix: pkgs: - let - inherit (pkgs) buildPackages; - inherit (pkgs) cacert; - installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; }; - in - - buildPackages.runCommand "nix-binary-tarball-${version}" - { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; - meta.description = "Distribution-independent Nix bootstrap binaries for ${pkgs.system}"; - } - '' - cp ${installerClosureInfo}/registration $TMPDIR/reginfo - cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh - substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - - substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - - if type -p shellcheck; then - # SC1090: Don't worry about not being able to find - # $nix/etc/profile.d/nix.sh - shellcheck --exclude SC1090 $TMPDIR/install - shellcheck $TMPDIR/create-darwin-volume.sh - shellcheck $TMPDIR/install-darwin-multi-user.sh - shellcheck $TMPDIR/install-systemd-multi-user.sh - - # SC1091: Don't panic about not being able to source - # /etc/profile - # SC2002: Ignore "useless cat" "error", when loading - # .reginfo, as the cat is a much cleaner - # implementation, even though it is "useless" - # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving - # root's home directory - shellcheck --external-sources \ - --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user - fi - - chmod +x $TMPDIR/install - chmod +x $TMPDIR/create-darwin-volume.sh - chmod +x $TMPDIR/install-darwin-multi-user.sh - chmod +x $TMPDIR/install-systemd-multi-user.sh - chmod +x $TMPDIR/install-multi-user - dir=nix-${version}-${pkgs.system} - fn=$out/$dir.tar.xz - mkdir -p $out/nix-support - echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products - tar cvfJ $fn \ - --owner=0 --group=0 --mode=u+rw,uga+r \ - --mtime='1970-01-01' \ - --absolute-names \ - --hard-dereference \ - --transform "s,$TMPDIR/install,$dir/install," \ - --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \ - --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ - --transform "s,$NIX_STORE,$dir/store,S" \ - $TMPDIR/install \ - $TMPDIR/create-darwin-volume.sh \ - $TMPDIR/install-darwin-multi-user.sh \ - $TMPDIR/install-systemd-multi-user.sh \ - $TMPDIR/install-multi-user \ - $TMPDIR/reginfo \ - $(cat ${installerClosureInfo}/store-paths) - ''; + binaryTarball = nix: pkgs: pkgs.callPackage ./binary-tarball.nix { + inherit nix; + }; overlayFor = getStdenv: final: prev: - let currentStdenv = getStdenv final; in + let + stdenv = getStdenv final; + + lowdown-nix = final.callPackage ./lowdown.nix { + inherit lowdown-src stdenv; + }; + in { nixStable = prev.nix; @@ -409,129 +352,70 @@ nixUnstable = prev.nixUnstable; nix = - with final; - with commonDeps { - inherit pkgs; - inherit (currentStdenv.hostPlatform) isStatic; - }; - let - canRunInstalled = currentStdenv.buildPlatform.canExecute currentStdenv.hostPlatform; - in currentStdenv.mkDerivation (finalAttrs: { - name = "nix-${version}"; - inherit version; + let + officialRelease = false; + versionSuffix = + if officialRelease + then "" + else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - src = nixSrc; - VERSION_SUFFIX = versionSuffix; + sh = final.busybox-sandbox-shell or (final.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y - outputs = [ "out" "dev" "doc" ] - ++ lib.optional (currentStdenv.hostPlatform != currentStdenv.buildPlatform) "check"; + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps - # There have been issues building these dependencies - ++ lib.optionals (currentStdenv.hostPlatform == currentStdenv.buildPlatform) awsDeps - ++ lib.optionals finalAttrs.doCheck checkDeps; + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }); - propagatedBuildInputs = propagatedDeps; + boehmgc = (final.boehmgc.override { + enableLargeConfig = true; + }).overrideAttrs(o: { + patches = (o.patches or []) ++ [ + ./boehmgc-coroutine-sp-fallback.diff - disallowedReferences = [ boost-nix ]; + # https://github.com/ivmai/bdwgc/pull/586 + ./boehmgc-traceable_allocator-public.diff + ]; + }); - preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic) - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost-nix}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib - rm -f $out/lib/*.a - ${lib.optionalString currentStdenv.hostPlatform.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - ''} - ${lib.optionalString currentStdenv.hostPlatform.isDarwin '' - for LIB in $out/lib/*.dylib; do - chmod u+w $LIB - install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost-nix}/lib/ $LIB || true - done - install_name_tool -change ${boost-nix}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - ''} - ''; - - configureFlags = configureFlags ++ - [ "--sysconfdir=/etc" ] ++ - lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" ++ - [ (lib.enableFeature finalAttrs.doCheck "tests") ] ++ - lib.optionals finalAttrs.doCheck testConfigureFlags ++ - lib.optional (!canRunInstalled) "--disable-doc-gen"; - - enableParallelBuilding = true; - - makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; - - doCheck = true; - - installFlags = "sysconfdir=$(out)/etc"; - - postInstall = '' - mkdir -p $doc/nix-support - echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products - ${lib.optionalString currentStdenv.hostPlatform.isStatic '' - mkdir -p $out/nix-support - echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - ''} - ${lib.optionalString currentStdenv.isDarwin '' - install_name_tool \ - -change ${boost-nix}/lib/libboost_context.dylib \ - $out/lib/libboost_context.dylib \ - $out/lib/libnixutil.dylib - install_name_tool \ - -change ${boost-nix}/lib/libboost_regex.dylib \ - $out/lib/libboost_regex.dylib \ - $out/lib/libnixexpr.dylib - ''} - ''; - - doInstallCheck = finalAttrs.doCheck; - installCheckFlags = "sysconfdir=$(out)/etc"; - installCheckTarget = "installcheck"; # work around buggy detection in stdenv - - separateDebugInfo = !currentStdenv.hostPlatform.isStatic; - - strictDeps = true; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - - passthru.perl-bindings = final.callPackage ./perl { - inherit fileset; - stdenv = currentStdenv; + in final.callPackage ./package.nix { + inherit + boehmgc + fileset + sh + stdenv + versionSuffix + ; + boost = final.boost.override { enableIcu = false; }; + libgit2 = final.libgit2.overrideAttrs (attrs: { + src = libgit2; + version = libgit2.lastModifiedDate; + cmakeFlags = attrs.cmakeFlags or [] + ++ [ "-DUSE_SSH=exec" ]; + }); + lowdown = lowdown-nix; + officialRelease = false; }; - meta.platforms = lib.platforms.unix; - meta.mainProgram = "nix"; - }); - - boost-nix = final.boost.override { - enableIcu = false; + inherit lowdown-nix; }; - lowdown-nix = with final; currentStdenv.mkDerivation rec { - name = "lowdown-0.9.0"; - - src = lowdown-src; - - outputs = [ "out" "bin" "dev" ]; - - nativeBuildInputs = [ buildPackages.which ]; - - configurePhase = '' - ${if (currentStdenv.isDarwin && currentStdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""} - ./configure \ - PREFIX=${placeholder "dev"} \ - BINDIR=${placeholder "bin"}/bin - ''; - }; - }; - in { # A Nixpkgs overlay that overrides the 'nix' and # 'nix.perl-bindings' packages. diff --git a/lowdown.nix b/lowdown.nix new file mode 100644 index 000000000..5f469fad5 --- /dev/null +++ b/lowdown.nix @@ -0,0 +1,22 @@ +{ lib +, stdenv +, which +, lowdown-src +}: + +stdenv.mkDerivation rec { + name = "lowdown-0.9.0"; + + src = lowdown-src; + + outputs = [ "out" "bin" "dev" ]; + + nativeBuildInputs = [ which ]; + + configurePhase = '' + ${lib.optionalString (stdenv.isDarwin && stdenv.isAarch64) "echo \"HAVE_SANDBOX_INIT=false\" > configure.local"} + ./configure \ + PREFIX=${placeholder "dev"} \ + BINDIR=${placeholder "bin"}/bin + ''; +} diff --git a/package.nix b/package.nix new file mode 100644 index 000000000..ae075acf7 --- /dev/null +++ b/package.nix @@ -0,0 +1,239 @@ +{ lib +, callPackage +, stdenv +, versionSuffix ? "" +, officialRelease ? false +, buildUnreleasedNotes ? false +, autoconf-archive +, autoreconfHook +, aws-sdk-cpp +, boehmgc +, nlohmann_json +, bison +, boost +, brotli +, bzip2 +, changelog-d +, curl +, editline +, fileset +, flex +, git +, gtest +, jq +, libarchive +, libcpuid +, libgit2 +, libseccomp +, libsodium +, lowdown +, mdbook +, mdbook-linkcheck +, mercurial +, openssh +, openssl +, pkg-config +, rapidcheck +, sh +, sqlite +, util-linux +, xz +}: + +let + + version = lib.fileContents ./.version + versionSuffix; + + inherit (stdenv.hostPlatform) isStatic; + + canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; +in + +stdenv.mkDerivation (finalAttrs: { + name = "nix-${version}"; + inherit version; + + src = + let + baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.; + configureFiles = fileset.unions [ + ./.version + ./configure.ac + ./m4 + # TODO: do we really need README.md? It doesn't seem used in the build. + ./README.md + ]; + + topLevelBuildFiles = fileset.unions [ + ./local.mk + ./Makefile + ./Makefile.config.in + ./mk + ]; + + functionalTestFiles = fileset.unions [ + ./tests/functional + (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) + ]; + + in + fileset.toSource { + root = ./.; + fileset = fileset.intersect baseFiles (fileset.unions [ + configureFiles + topLevelBuildFiles + ./boehmgc-coroutine-sp-fallback.diff + ./doc + ./misc + ./precompiled-headers.h + ./src + ./unit-test-data + ./COPYING + ./scripts/local.mk + functionalTestFiles + ]); + }; + + VERSION_SUFFIX = versionSuffix; + + outputs = [ "out" "dev" "doc" ] + ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "check"; + + nativeBuildInputs = [ + bison + flex + (lib.getBin lowdown) + mdbook + mdbook-linkcheck + autoconf-archive + autoreconfHook + pkg-config + + # Tests + git + mercurial # FIXME: remove? only needed for tests + jq # Also for custom mdBook preprocessor. + openssh # only needed for tests (ssh-keygen) + ] + ++ lib.optional stdenv.hostPlatform.isLinux util-linux + # Official releases don't have rl-next, so we don't need to compile a changelog + ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d; + + buildInputs = [ + boost + brotli + bzip2 + curl + editline + libarchive + libgit2 + libsodium + lowdown + openssl + sqlite + xz + ] + ++ lib.optionals stdenv.isLinux [libseccomp] + ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid + # There have been issues building these dependencies + ++ lib.optionals (stdenv.hostPlatform == stdenv.buildPlatform) (lib.optional (stdenv.isLinux || stdenv.isDarwin) + (aws-sdk-cpp.override { + apis = ["s3" "transfer"]; + customMemoryManagement = false; + })) + ++ lib.optionals finalAttrs.doCheck ([ + gtest + rapidcheck + ]); + + propagatedBuildInputs = [ + boehmgc + nlohmann_json + ]; + + disallowedReferences = [ boost ]; + + preConfigure = lib.optionalString (! stdenv.hostPlatform.isStatic) + '' + # Copy libboost_context so we don't get all of Boost in our closure. + # https://github.com/NixOS/nixpkgs/issues/45462 + mkdir -p $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib + rm -f $out/lib/*.a + ${lib.optionalString stdenv.hostPlatform.isLinux '' + chmod u+w $out/lib/*.so.* + patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + ''} + ${lib.optionalString stdenv.hostPlatform.isDarwin '' + for LIB in $out/lib/*.dylib; do + chmod u+w $LIB + install_name_tool -id $LIB $LIB + install_name_tool -delete_rpath ${boost}/lib/ $LIB || true + done + install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib + ''} + ''; + + configureFlags = + lib.optionals stdenv.isLinux [ + "--with-boost=${boost}/lib" + "--with-sandbox-shell=${sh}/bin/busybox" + ] + ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [ + "LDFLAGS=-fuse-ld=gold" + ] + ++ [ "--sysconfdir=/etc" ] + ++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" + ++ [ (lib.enableFeature finalAttrs.doCheck "tests") ] + ++ lib.optionals finalAttrs.doCheck ([ "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" ] + ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ + "--enable-install-unit-tests" + "--with-check-bin-dir=${builtins.placeholder "check"}/bin" + "--with-check-lib-dir=${builtins.placeholder "check"}/lib" + ]) + ++ lib.optional (!canRunInstalled) "--disable-doc-gen"; + + enableParallelBuilding = true; + + makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; + + doCheck = true; + + installFlags = "sysconfdir=$(out)/etc"; + + postInstall = '' + mkdir -p $doc/nix-support + echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products + ${lib.optionalString stdenv.hostPlatform.isStatic '' + mkdir -p $out/nix-support + echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products + ''} + ${lib.optionalString stdenv.isDarwin '' + install_name_tool \ + -change ${boost}/lib/libboost_context.dylib \ + $out/lib/libboost_context.dylib \ + $out/lib/libnixutil.dylib + install_name_tool \ + -change ${boost}/lib/libboost_regex.dylib \ + $out/lib/libboost_regex.dylib \ + $out/lib/libnixexpr.dylib + ''} + ''; + + doInstallCheck = finalAttrs.doCheck; + installCheckFlags = "sysconfdir=$(out)/etc"; + installCheckTarget = "installcheck"; # work around buggy detection in stdenv + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + passthru.perl-bindings = callPackage ./perl { + inherit fileset stdenv; + }; + + meta.platforms = lib.platforms.unix; + meta.mainProgram = "nix"; +}) From c64190e65048547712fcf7a0ae09fbfd0a709474 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Thu, 30 Nov 2023 22:49:02 +0000 Subject: [PATCH 706/909] Run statix --- flake.nix | 6 +++--- package.nix | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index a1fc1cd1c..e32a84ae5 100644 --- a/flake.nix +++ b/flake.nix @@ -7,7 +7,7 @@ inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; - outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, libgit2 }: + outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, libgit2, ... }: let inherit (nixpkgs) lib; @@ -183,7 +183,7 @@ "--enable-internal-api-docs" ]; - changelog-d = pkgs.buildPackages.changelog-d; + inherit (pkgs.buildPackages) changelog-d; nativeBuildDeps = [ @@ -349,7 +349,7 @@ nixStable = prev.nix; # Forward from the previous stage as we don’t want it to pick the lowdown override - nixUnstable = prev.nixUnstable; + inherit (prev) nixUnstable; nix = let diff --git a/package.nix b/package.nix index ae075acf7..8d62120fb 100644 --- a/package.nix +++ b/package.nix @@ -141,10 +141,10 @@ stdenv.mkDerivation (finalAttrs: { apis = ["s3" "transfer"]; customMemoryManagement = false; })) - ++ lib.optionals finalAttrs.doCheck ([ + ++ lib.optionals finalAttrs.doCheck [ gtest rapidcheck - ]); + ]; propagatedBuildInputs = [ boehmgc From f55ee7cf7753caee7a27052fab679d8c8fe27cc4 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Thu, 30 Nov 2023 22:53:07 +0000 Subject: [PATCH 707/909] little refactoring --- flake.nix | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index e32a84ae5..544a07ba6 100644 --- a/flake.nix +++ b/flake.nix @@ -459,8 +459,21 @@ # to https://nixos.org/nix/install. It downloads the binary # tarball for the user's system and calls the second half of the # installation script. - installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ]; - installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"]; + installerScript = installScriptFor [ + "aarch64-linux" + "armv6l-linux" + "armv7l-linux" + "i686-linux" + "x86_64-linux" + "aarch64-darwin" + "x86_64-darwin" + ]; + installerScriptForGHA = installScriptFor [ + "armv6l-linux" + "armv7l-linux" + "x86_64-linux" + "x86_64-darwin" + ]; # docker image with Nix inside dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); From 02d9cf2d303e4e7e283dba2f3181f3e40843c354 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 1 Dec 2023 00:41:19 +0100 Subject: [PATCH 708/909] shorten the quick start chapter this focuses on `nix-shell -p` and refers to search.nixos.org for package search, which is currently the easiest and most effective way to find program names. --- doc/manual/src/quick-start.md | 87 +++++++---------------------------- 1 file changed, 16 insertions(+), 71 deletions(-) diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 5f54abbde..04a0b7c96 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -1,10 +1,9 @@ # Quick Start -This chapter is for impatient people who don't like reading -documentation. For more in-depth information you are kindly referred -to subsequent chapters. +This chapter is for impatient people who don't like reading documentation. +For more in-depth information you are kindly referred to subsequent chapters. -1. Install Nix by running the following: +1. Install Nix: ```console $ curl -L https://nixos.org/nix/install | sh @@ -13,87 +12,33 @@ to subsequent chapters. The install script will use `sudo`, so make sure you have sufficient rights. On Linux, `--daemon` can be omitted for a single-user install. - For other installation methods, see [here](installation/index.md). + For other installation methods, see the detailed [installation instructions](installation/index.md). -1. See what installable packages are currently available in the - channel: +1. Run software without installing it permanently: ```console - $ nix-env --query --available --attr-path - nixpkgs.docbook_xml_dtd_43 docbook-xml-4.3 - nixpkgs.docbook_xml_dtd_45 docbook-xml-4.5 - nixpkgs.firefox firefox-33.0.2 - nixpkgs.hello hello-2.9 - nixpkgs.libxslt libxslt-1.1.28 - … + $ nix-shell --packages cowsay lolcat ``` -1. Install some packages from the channel: + This downloads the specified packages with all their dependencies, and drops you into a Bash shell where the commands provided by those packages are present. + This will not affect your normal environment: ```console - $ nix-env --install --attr nixpkgs.hello + [nix-shell:~]$ cowsay Hello, Nix! | lolcat ``` - This should download pre-built packages; it should not build them - locally (if it does, something went wrong). - -1. Test that they work: + Exiting the shell will make the programs disappear again: ```console - $ which hello - /home/eelco/.nix-profile/bin/hello - $ hello - Hello, world! - ``` - -1. Uninstall a package: - - ```console - $ nix-env --uninstall hello - ``` - -1. You can also test a package without installing it: - - ```console - $ nix-shell --packages hello - ``` - - This builds or downloads GNU Hello and its dependencies, then drops - you into a Bash shell where the `hello` command is present, all - without affecting your normal environment: - - ```console - [nix-shell:~]$ hello - Hello, world! - [nix-shell:~]$ exit - - $ hello - hello: command not found + $ lolcat + lolcat: command not found ``` -1. To keep up-to-date with the channel, do: +1. Search for more packages on to try them out. + +1. Free up storage space: ```console - $ nix-channel --update nixpkgs - $ nix-env --upgrade '*' - ``` - - The latter command will upgrade each installed package for which - there is a “newer” version (as determined by comparing the version - numbers). - -1. If you're unhappy with the result of a `nix-env` action (e.g., an - upgraded package turned out not to work properly), you can go back: - - ```console - $ nix-env --rollback - ``` - -1. You should periodically run the Nix garbage collector to get rid of - unused packages, since uninstalls or upgrades don't actually delete - them: - - ```console - $ nix-collect-garbage --delete-old + $ nix-collect-garbage ``` From 908a011a4a2fe4e494e5b6e4c94f013f159f3616 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 1 Dec 2023 00:50:20 +0100 Subject: [PATCH 709/909] Revert "Switch from std::regex to boost::regex" --- flake.nix | 22 +++++++--------------- src/libexpr/local.mk | 2 +- src/libexpr/primops.cc | 35 ++++++++++++----------------------- 3 files changed, 20 insertions(+), 39 deletions(-) diff --git a/flake.nix b/flake.nix index 33673575b..822b3d31e 100644 --- a/flake.nix +++ b/flake.nix @@ -157,7 +157,7 @@ configureFlags = lib.optionals stdenv.isLinux [ - "--with-boost=${boost-nix}/lib" + "--with-boost=${boost}/lib" "--with-sandbox-shell=${sh}/bin/busybox" ] ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [ @@ -210,7 +210,7 @@ version = libgit2.lastModifiedDate; cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"]; })) - boost-nix + boost lowdown-nix libsodium ] @@ -434,14 +434,14 @@ propagatedBuildInputs = propagatedDeps; - disallowedReferences = [ boost-nix ]; + disallowedReferences = [ boost ]; preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic) '' # Copy libboost_context so we don't get all of Boost in our closure. # https://github.com/NixOS/nixpkgs/issues/45462 mkdir -p $out/lib - cp -pd ${boost-nix}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib rm -f $out/lib/*.a ${lib.optionalString currentStdenv.hostPlatform.isLinux '' chmod u+w $out/lib/*.so.* @@ -451,9 +451,9 @@ for LIB in $out/lib/*.dylib; do chmod u+w $LIB install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost-nix}/lib/ $LIB || true + install_name_tool -delete_rpath ${boost}/lib/ $LIB || true done - install_name_tool -change ${boost-nix}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib + install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib ''} ''; @@ -481,13 +481,9 @@ ''} ${lib.optionalString currentStdenv.isDarwin '' install_name_tool \ - -change ${boost-nix}/lib/libboost_context.dylib \ + -change ${boost}/lib/libboost_context.dylib \ $out/lib/libboost_context.dylib \ $out/lib/libnixutil.dylib - install_name_tool \ - -change ${boost-nix}/lib/libboost_regex.dylib \ - $out/lib/libboost_regex.dylib \ - $out/lib/libnixexpr.dylib ''} ''; @@ -510,10 +506,6 @@ meta.mainProgram = "nix"; }); - boost-nix = final.boost.override { - enableIcu = false; - }; - lowdown-nix = with final; currentStdenv.mkDerivation rec { name = "lowdown-0.9.0"; diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index c07a18bb5..ed6bc761a 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib libexpr_LIBS = libutil libstore libfetchers -libexpr_LDFLAGS += -lboost_context -lboost_regex -pthread +libexpr_LDFLAGS += -lboost_context -pthread ifdef HOST_LINUX libexpr_LDFLAGS += -ldl endif diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ebf2549e4..146a7603c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -17,7 +17,6 @@ #include "primops.hh" #include -#include #include #include @@ -26,6 +25,7 @@ #include #include +#include #include #include @@ -3886,30 +3886,19 @@ static RegisterPrimOp primop_convertHash({ .fun = prim_convertHash, }); -// regex aliases, switch between boost and std -using regex = boost::regex; -using regex_error = boost::regex_error; -using cmatch = boost::cmatch; -using cregex_iterator = boost::cregex_iterator; -namespace regex_constants = boost::regex_constants; -// overloaded function alias -constexpr auto regex_match = [] (auto &&...args) { - return boost::regex_match(std::forward(args)...); - }; - struct RegexCache { // TODO use C++20 transparent comparison when available - std::unordered_map cache; + std::unordered_map cache; std::list keys; - regex get(std::string_view re) + std::regex get(std::string_view re) { auto it = cache.find(re); if (it != cache.end()) return it->second; keys.emplace_back(re); - return cache.emplace(keys.back(), regex(keys.back(), regex::extended)).first->second; + return cache.emplace(keys.back(), std::regex(keys.back(), std::regex::extended)).first->second; } }; @@ -3929,8 +3918,8 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) NixStringContext context; const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match"); - cmatch match; - if (!regex_match(str.begin(), str.end(), match, regex)) { + std::cmatch match; + if (!std::regex_match(str.begin(), str.end(), match, regex)) { v.mkNull(); return; } @@ -3945,8 +3934,8 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) (v.listElems()[i] = state.allocValue())->mkString(match[i + 1].str()); } - } catch (regex_error & e) { - if (e.code() == regex_constants::error_space) { + } catch (std::regex_error & e) { + if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ state.debugThrowLastTrace(EvalError({ .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), @@ -4009,8 +3998,8 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) NixStringContext context; const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split"); - auto begin = cregex_iterator(str.begin(), str.end(), regex); - auto end = cregex_iterator(); + auto begin = std::cregex_iterator(str.begin(), str.end(), regex); + auto end = std::cregex_iterator(); // Any matches results are surrounded by non-matching results. const size_t len = std::distance(begin, end); @@ -4049,8 +4038,8 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) assert(idx == 2 * len + 1); - } catch (regex_error & e) { - if (e.code() == regex_constants::error_space) { + } catch (std::regex_error & e) { + if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ state.debugThrowLastTrace(EvalError({ .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), From 4781e7fa7048d2861172baaaa04a7be4b8a2b631 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 30 Nov 2023 23:07:09 +0100 Subject: [PATCH 710/909] Document each store type on its own page This makes for more useful manual table of contents, that displays the information at a glance. The `nix help-stores` command is kept as-is, even though it will show up in the manual with the same information as these pages due to the way it is written as a "`--help`-style" command. Deciding what to do with that command is left for a later PR. This change also lists all store types at the top of the respective overview page. Co-authored-by: John Ericson `. + let + help-stores = '' + ${index} - maybeOptions = let - allVisibleOptions = filterAttrs - (_: o: ! o.hiddenCategory) - (details.flags // toplevel.flags); - in optionalString (allVisibleOptions != {}) '' - # Options + ${allStores} + ''; + index = replaceStrings + [ "@store-types@" ] [ storesOverview ] + details.doc; + storesOverview = + let + showEntry = store: + "- [${store.name}](#${store.slug})"; + in + concatStringsSep "\n" (map showEntry storesList) + "\n"; + allStores = concatStringsSep "\n" (attrValues storePages); + storePages = listToAttrs + (map (s: { name = s.filename; value = s.page; }) storesList); + storesList = showStoreDocs { + storeInfo = commandInfo.stores; + inherit inlineHTML; + }; + in + optionalString (details ? doc) ( + if match "@store-types@" details.doc != [ ] + then help-stores + else details.doc + ); - ${showOptions inlineHTML allVisibleOptions} + maybeOptions = + let + allVisibleOptions = filterAttrs + (_: o: ! o.hiddenCategory) + (details.flags // toplevel.flags); + in + optionalString (allVisibleOptions != { }) '' + # Options - > **Note** - > - > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. - ''; + ${showOptions inlineHTML allVisibleOptions} + + > **Note** + > + > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. + ''; showOptions = inlineHTML: allOptions: let diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index 8e26edb65..57247a181 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -1,14 +1,20 @@ let - inherit (builtins) attrValues mapAttrs; - inherit (import ) concatStrings optionalString squash; + inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings; + inherit (import ) optionalString filterAttrs trim squash toLower unique indent; showSettings = import ; in -inlineHTML: storesInfo: +{ + # data structure describing all stores and their parameters + storeInfo, + # whether to add inline HTML tags + # `lowdown` does not eat those for one of the output modes + inlineHTML, +}: let - showStore = name: { settings, doc, experimentalFeature }: + showStore = { name, slug }: { settings, doc, experimentalFeature }: let result = squash '' # ${name} @@ -22,9 +28,6 @@ let ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings} ''; - # markdown doesn't like spaces in URLs - slug = builtins.replaceStrings [ " " ] [ "-" ] name; - experimentalFeatureNote = optionalString (experimentalFeature != null) '' > **Warning** > @@ -42,4 +45,13 @@ let ''; in result; -in concatStrings (attrValues (mapAttrs showStore storesInfo)) + storesList = map + (name: rec { + inherit name; + slug = replaceStrings [ " " ] [ "-" ] (toLower name); + filename = "${slug}.md"; + page = showStore { inherit name slug; } storeInfo.${name}; + }) + (attrNames storeInfo); + +in storesList diff --git a/doc/manual/generate-store-types.nix b/doc/manual/generate-store-types.nix new file mode 100644 index 000000000..3b78a0e1b --- /dev/null +++ b/doc/manual/generate-store-types.nix @@ -0,0 +1,39 @@ +let + inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings; + showSettings = import ; + showStoreDocs = import ; +in + +storeInfo: + +let + storesList = showStoreDocs { + inherit storeInfo; + inlineHTML = true; + }; + + index = + let + showEntry = store: + "- [${store.name}](./${store.filename})"; + in + concatStringsSep "\n" (map showEntry storesList); + + "index.md" = replaceStrings + [ "@store-types@" ] [ index ] + (readFile ./src/store/types/index.md.in); + + tableOfContents = + let + showEntry = store: + " - [${store.name}](store/types/${store.filename})"; + in + concatStringsSep "\n" (map showEntry storesList) + "\n"; + + "SUMMARY.md" = tableOfContents; + + storePages = listToAttrs + (map (s: { name = s.filename; value = s.page; }) storesList); + +in +storePages // { inherit "index.md" "SUMMARY.md"; } diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix index fc7c7d4cf..0eec0e1da 100644 --- a/doc/manual/generate-xp-features.nix +++ b/doc/manual/generate-xp-features.nix @@ -8,4 +8,6 @@ let ${doc} ''; -in xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps))) +in + +xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index fa9db9f02..456000d3d 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -97,10 +97,17 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md @cp $< $@ @$(call process-includes,$@,$@) +$(d)/src/store/types: $(d)/nix.json $(d)/utils.nix $(d)/generate-store-info.nix $(d)/generate-store-types.nix $(d)/src/store/types/index.md.in $(doc_nix) + @# FIXME: build out of tree! + @rm -rf $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-store-types.nix (builtins.fromJSON (builtins.readFile $<)).stores' + @# do not destroy existing contents + @mv $@.tmp/* $@/ + $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)' @@ -200,7 +207,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli # `@docroot@` is to be preserved for documenting the mechanism # FIXME: maybe contributing guides should live right next to the code # instead of in the manual -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 686d3e8d7..c67ddc6cb 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -20,6 +20,8 @@ - [File System Object](store/file-system-object.md) - [Store Object](store/store-object.md) - [Store Path](store/store-path.md) + - [Store Types](store/types/index.md) +{{#include ./store/types/SUMMARY.md}} - [Nix Language](language/index.md) - [Data Types](language/values.md) - [Language Constructs](language/constructs.md) diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md index e53d2d178..f7e24d96b 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/contributing/cli-guideline.md @@ -426,7 +426,7 @@ This leads to the following guidelines: ### Examples -This is bad, because all keys must be assumed to be store implementations: +This is bad, because all keys must be assumed to be store types: ```json { diff --git a/doc/manual/src/store/index.md b/doc/manual/src/store/index.md index 8a5305062..f1e8f1402 100644 --- a/doc/manual/src/store/index.md +++ b/doc/manual/src/store/index.md @@ -2,4 +2,4 @@ The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data. -There are multiple implementations of Nix stores with different capabilities, such as the actual filesystem (`/nix/store`) or binary caches. +There are [multiple types of Nix stores](./types/index.md) with different capabilities, such as the default one on the [local filesystem](./types/local-store.md) (`/nix/store`) or [binary caches](./types/http-binary-cache-store.md). diff --git a/doc/manual/src/store/types/index.md.in b/doc/manual/src/store/types/index.md.in index bb166a1fc..b4db553a2 100644 --- a/doc/manual/src/store/types/index.md.in +++ b/doc/manual/src/store/types/index.md.in @@ -1,4 +1,6 @@ -Nix supports different types of stores. These are described below. +Nix supports different types of stores: + +@store-types@ ## Store URL format @@ -39,4 +41,3 @@ store as follows: * Otherwise, use the [local store](#local-store) `/nix/store`. -@stores@ diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index 849832b2c..19ff49b64 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -1,5 +1,11 @@ with builtins; +let + lowerChars = stringToCharacters "abcdefghijklmnopqrstuvwxyz"; + upperChars = stringToCharacters "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + stringToCharacters = s: genList (p: substring p 1 s) (stringLength s); +in + rec { splitLines = s: filter (x: !isList x) (split "\n" s); @@ -18,6 +24,8 @@ rec { in if replaced == string then string else replaceStringsRec from to replaced; + toLower = replaceStrings upperChars lowerChars; + squash = replaceStringsRec "\n\n\n" "\n\n"; trim = string: diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 401acc38e..193972272 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -141,7 +141,7 @@ MixEvalArgs::MixEvalArgs() .longName = "eval-store", .description = R"( - The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format) to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them. )", .category = category, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ebf2549e4..4ecdda55c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4460,7 +4460,7 @@ void EvalState::createBaseEnv() .doc = R"( Logical file system location of the [Nix store](@docroot@/glossary.md#gloss-store) currently in use. - This value is determined by the `store` parameter in [Store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md): + This value is determined by the `store` parameter in [Store URLs](@docroot@/store/types/index.md#store-url-format): ```shell-session $ nix-instantiate --store 'dummy://?store=/blah' --eval --expr builtins.storeDir diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 838d2aba2..38b0d516c 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -117,10 +117,11 @@ public: Setting storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store", R"( - The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format) to use for most operations. - See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) - for supported store types and settings. + See the + [Store Types](@docroot@/store/types/index.md) + section of the manual for supported store types and settings. )"}; Setting keepFailed{this, false, "keep-failed", @@ -759,7 +760,7 @@ public: Strings{"https://cache.nixos.org/"}, "substituters", R"( - A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) to be used as substituters, separated by whitespace. + A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace. A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them. Substituters are tried based on their priority value, which each substituter can set independently. @@ -778,7 +779,7 @@ public: Setting trustedSubstituters{ this, {}, "trusted-substituters", R"( - A list of [Nix store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace. + A list of [Nix store URLs](@docroot@/store/types/index.md#store-url-format), separated by whitespace. These are not used by default, but users of the Nix daemon can enable them by specifying [`substituters`](#conf-substituters). Unprivileged users (those set in only [`allowed-users`](#conf-allowed-users) but not [`trusted-users`](#conf-trusted-users)) can pass as `substituters` only those URLs listed in `trusted-substituters`. diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f16949f42..3d3919882 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -20,7 +20,7 @@ namespace nix { -/* TODO: Separate these store impls into different files, give them better names */ +/* TODO: Separate these store types into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) : RemoteStoreConfig(params) , Store(params) diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/ssh-store-config.hh index bf55d20cf..4ce4ffc4c 100644 --- a/src/libstore/ssh-store-config.hh +++ b/src/libstore/ssh-store-config.hh @@ -20,7 +20,7 @@ struct CommonSSHStoreConfig : virtual StoreConfig const Setting remoteStore{this, "", "remote-store", R"( - [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + [Store URL](@docroot@/store/types/index.md#store-url-format) to be used on the remote machine. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). )"}; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 8b6bf9aed..e28baf34e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -30,7 +30,7 @@ namespace nix { /** - * About the class hierarchy of the store implementations: + * About the class hierarchy of the store types: * * Each store type `Foo` consists of two classes: * @@ -962,7 +962,7 @@ OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * ev * - ‘ssh://[user@]’: A remote Nix store accessed by running * ‘nix-store --serve’ via SSH. * - * You can pass parameters to the store implementation by appending + * You can pass parameters to the store type by appending * ‘?key=value&key=value&...’ to the URI. */ ref openStore(const std::string & uri = settings.storeUri.get(), diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 2418e3f4c..e4bdb8cb3 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -252,7 +252,7 @@ constexpr std::array xpFeatureDetails .tag = Xp::ReadOnlyLocalStore, .name = "read-only-local-store", .description = R"( - Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs. + Allow the use of the `read-only` parameter in [local store](@docroot@/store/types/local-store.md) URIs. )", }, { diff --git a/src/nix/nix.md b/src/nix/nix.md index eb150f03b..749456014 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -235,10 +235,14 @@ operate are determined as follows: # Nix stores -Most `nix` subcommands operate on a *Nix store*. These are documented -in [`nix help-stores`](./nix3-help-stores.md). +Most `nix` subcommands operate on a *Nix store*. +The various store types are documented in the +[Store Types](@docroot@/store/types/index.md) +section of the manual. -# Shebang interpreter +The same information is also available from the [`nix help-stores`](./nix3-help-stores.md) command. + +# Shebang interpreter The `nix` command can be used as a `#!` interpreter. Arguments to Nix can be passed on subsequent lines in the script. From 333ea684b065318aa49aec367c995b3d8c5d65ed Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 1 Dec 2023 01:39:52 +0100 Subject: [PATCH 711/909] Add boost::regex regression test --- src/libexpr/tests/primops.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libexpr/tests/primops.cc b/src/libexpr/tests/primops.cc index d820b860e..7485fa0d0 100644 --- a/src/libexpr/tests/primops.cc +++ b/src/libexpr/tests/primops.cc @@ -814,6 +814,14 @@ namespace nix { ASSERT_THAT(*v.listElems()[0], IsStringEq("FOO")); } + TEST_F(PrimOpTest, match5) { + // The regex "\\{}" is valid and matches the string "{}". + // Caused a regression before when trying to switch from std::regex to boost::regex. + // See https://github.com/NixOS/nix/pull/7762#issuecomment-1834303659 + auto v = eval("builtins.match \"\\\\{}\" \"{}\""); + ASSERT_THAT(v, IsListOfSize(0)); + } + TEST_F(PrimOpTest, attrNames) { auto v = eval("builtins.attrNames { x = 1; y = 2; z = 3; a = 2; }"); ASSERT_THAT(v, IsListOfSize(4)); From d5e934fb73496a2509755be5945a8bcf1730d59d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 1 Dec 2023 01:54:48 +0100 Subject: [PATCH 712/909] add redirect to new store page --- doc/manual/_redirects | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/_redirects b/doc/manual/_redirects index 2038671d7..62c693c97 100644 --- a/doc/manual/_redirects +++ b/doc/manual/_redirects @@ -31,9 +31,9 @@ /installation/installation /installation 301! /package-management/basic-package-mgmt /command-ref/nix-env 301! -/package-management/channels* /command-ref/nix-channel 301! +/package-management/channels /command-ref/nix-channel 301! /package-management/package-management /package-management 301! -/package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301! +/package-management/s3-substituter /store/types/s3-binary-cache-store 301! /protocols/protocols /protocols 301! From eff9b12bc296213c3ba824e90869bcafc4103e1c Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Fri, 1 Dec 2023 11:25:22 +0000 Subject: [PATCH 713/909] Further changes --- binary-tarball.nix | 6 ++- coverage.nix | 35 ++++++++++++++ flake.nix | 115 ++++++++++----------------------------------- package.nix | 88 +++++++++++++++++----------------- 4 files changed, 108 insertions(+), 136 deletions(-) create mode 100644 coverage.nix diff --git a/binary-tarball.nix b/binary-tarball.nix index 1fa185519..0053abbca 100644 --- a/binary-tarball.nix +++ b/binary-tarball.nix @@ -1,8 +1,8 @@ { runCommand -, version , system -, nix +, buildPackages , cacert +, nix }: let @@ -11,6 +11,8 @@ let rootPaths = [ nix cacert ]; }; + inherit (nix) version; + env = { meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; }; diff --git a/coverage.nix b/coverage.nix new file mode 100644 index 000000000..2390ef52d --- /dev/null +++ b/coverage.nix @@ -0,0 +1,35 @@ +{ lib +, releaseTools +, nix +, stdenv +}: + +let + inherit (nix) version; + +in + +releaseTools.coverageAnalysis { + name = "nix-coverage-${version}"; + + inherit (nix) + src + configureFlags + nativeBuildInputs + buildInputs + #checkInputs + ; + + enableParallelBuilding = true; + + dontInstall = false; + + doInstallCheck = true; + installCheckTarget = "installcheck"; # work around buggy detection in stdenv + + lcovFilter = [ "*/boost/*" "*-tab.*" ]; + + hardeningDisable = ["fortify"]; + + NIX_CFLAGS_COMPILE = "-DCOVERAGE=1"; +} diff --git a/flake.nix b/flake.nix index 544a07ba6..c0841a76d 100644 --- a/flake.nix +++ b/flake.nix @@ -479,60 +479,25 @@ dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); # Line coverage analysis. - coverage = - with nixpkgsFor.x86_64-linux.native; - with commonDeps { inherit pkgs; }; - - releaseTools.coverageAnalysis { - name = "nix-coverage-${version}"; - - src = nixSrc; - - configureFlags = testConfigureFlags; - - enableParallelBuilding = true; - - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ propagatedDeps ++ awsDeps ++ checkDeps; - - dontInstall = false; - - doInstallCheck = true; - installCheckTarget = "installcheck"; # work around buggy detection in stdenv - - lcovFilter = [ "*/boost/*" "*-tab.*" ]; - - hardeningDisable = ["fortify"]; - - NIX_CFLAGS_COMPILE = "-DCOVERAGE=1"; - }; + coverage = nixpkgsFor.x86_64-linux.native.callPackage ./coverage.nix {}; # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = - with nixpkgsFor.x86_64-linux.native; - with commonDeps { inherit pkgs; }; + internal-api-docs = nixpkgsFor.x86_64-linux.native.nix.overrideAttrs (old: { + pname = "nix-internal-api-docs"; - stdenv.mkDerivation { - pname = "nix-internal-api-docs"; - inherit version; + configureFlags = old.configureFlags ++ [ "--enable-internal-api-docs" ]; + nativeBuildInputs = old.nativeBuildInputs ++ [ nixpkgsFor.x86_64-linux.native.doxygen ]; - src = nixSrc; + dontBuild = true; + doCheck = false; - configureFlags = testConfigureFlags ++ internalApiDocsConfigureFlags; + installTargets = [ "internal-api-html" ]; - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ propagatedDeps - ++ awsDeps ++ checkDeps ++ internalApiDocsDeps; - - dontBuild = true; - - installTargets = [ "internal-api-html" ]; - - postInstall = '' - mkdir -p $out/nix-support - echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products - ''; - }; + postInstall = '' + mkdir -p $out/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products + ''; + }); # System tests. tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { @@ -540,7 +505,9 @@ # Make sure that nix-env still produces the exact same result # on a particular version of Nixpkgs. evalNixpkgs = - with nixpkgsFor.x86_64-linux.native; + let + inherit (nixpkgsFor.x86_64-linux.native) runCommand nix nixpkgs-regression; + in runCommand "eval-nixos" { buildInputs = [ nix ]; } '' type -p nix-env @@ -627,47 +594,17 @@ stdenvs))); devShells = let - makeShell = pkgs: stdenv: - let - canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; - in - with commonDeps { inherit pkgs; }; - stdenv.mkDerivation { - name = "nix"; + makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; }).overrideAttrs (_: { + installFlags = "sysconfdir=$(out)/etc"; + shellHook = '' + PATH=$prefix/bin:$PATH + unset PYTHONPATH + export MANPATH=$out/share/man:$MANPATH - outputs = [ "out" "dev" "doc" ] - ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "check"; - - nativeBuildInputs = nativeBuildDeps - ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear - ++ lib.optional - (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) - pkgs.buildPackages.clang-tools - # We want changelog-d in the shell even if the current build doesn't need it - ++ lib.optional (officialRelease || ! buildUnreleasedNotes) changelog-d - ; - - buildInputs = buildDeps ++ propagatedDeps - ++ awsDeps ++ checkDeps ++ internalApiDocsDeps; - - configureFlags = configureFlags - ++ testConfigureFlags ++ internalApiDocsConfigureFlags - ++ lib.optional (!canRunInstalled) "--disable-doc-gen"; - - enableParallelBuilding = true; - - installFlags = "sysconfdir=$(out)/etc"; - - shellHook = - '' - PATH=$prefix/bin:$PATH - unset PYTHONPATH - export MANPATH=$out/share/man:$MANPATH - - # Make bash completion work. - XDG_DATA_DIRS+=:$out/share - ''; - }; + # Make bash completion work. + XDG_DATA_DIRS+=:$out/share + ''; + }); in forAllSystems (system: let diff --git a/package.nix b/package.nix index 8d62120fb..bed77ba3b 100644 --- a/package.nix +++ b/package.nix @@ -41,16 +41,12 @@ }: let - version = lib.fileContents ./.version + versionSuffix; - - inherit (stdenv.hostPlatform) isStatic; - canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; in stdenv.mkDerivation (finalAttrs: { - name = "nix-${version}"; + pname = "nix"; inherit version; src = @@ -103,17 +99,12 @@ stdenv.mkDerivation (finalAttrs: { bison flex (lib.getBin lowdown) + jq # Also for custom mdBook preprocessor. mdbook mdbook-linkcheck autoconf-archive autoreconfHook pkg-config - - # Tests - git - mercurial # FIXME: remove? only needed for tests - jq # Also for custom mdBook preprocessor. - openssh # only needed for tests (ssh-keygen) ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux # Official releases don't have rl-next, so we don't need to compile a changelog @@ -133,19 +124,29 @@ stdenv.mkDerivation (finalAttrs: { sqlite xz ] - ++ lib.optionals stdenv.isLinux [libseccomp] + ++ lib.optional stdenv.isLinux libseccomp ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies - ++ lib.optionals (stdenv.hostPlatform == stdenv.buildPlatform) (lib.optional (stdenv.isLinux || stdenv.isDarwin) + ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) (aws-sdk-cpp.override { apis = ["s3" "transfer"]; customMemoryManagement = false; - })) - ++ lib.optionals finalAttrs.doCheck [ + }) + ; + + doCheck = true; + + checkInputs = [ gtest rapidcheck ]; + nativeCheckInputs = [ + git + mercurial # FIXME: remove? only needed for tests + openssh # only needed for tests (ssh-keygen) + ]; + propagatedBuildInputs = [ boehmgc nlohmann_json @@ -153,52 +154,49 @@ stdenv.mkDerivation (finalAttrs: { disallowedReferences = [ boost ]; - preConfigure = lib.optionalString (! stdenv.hostPlatform.isStatic) - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib - rm -f $out/lib/*.a - ${lib.optionalString stdenv.hostPlatform.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - ''} - ${lib.optionalString stdenv.hostPlatform.isDarwin '' - for LIB in $out/lib/*.dylib; do - chmod u+w $LIB - install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost}/lib/ $LIB || true - done - install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - ''} - ''; + preConfigure = lib.optionalString (! stdenv.hostPlatform.isStatic) '' + # Copy libboost_context so we don't get all of Boost in our closure. + # https://github.com/NixOS/nixpkgs/issues/45462 + mkdir -p $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib + rm -f $out/lib/*.a + ${lib.optionalString stdenv.hostPlatform.isLinux '' + chmod u+w $out/lib/*.so.* + patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + ''} + ${lib.optionalString stdenv.hostPlatform.isDarwin '' + for LIB in $out/lib/*.dylib; do + chmod u+w $LIB + install_name_tool -id $LIB $LIB + install_name_tool -delete_rpath ${boost}/lib/ $LIB || true + done + install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib + ''} + ''; configureFlags = lib.optionals stdenv.isLinux [ "--with-boost=${boost}/lib" "--with-sandbox-shell=${sh}/bin/busybox" ] - ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [ + ++ lib.optional (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) "LDFLAGS=-fuse-ld=gold" - ] ++ [ "--sysconfdir=/etc" ] ++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" ++ [ (lib.enableFeature finalAttrs.doCheck "tests") ] - ++ lib.optionals finalAttrs.doCheck ([ "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" ] - ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ - "--enable-install-unit-tests" - "--with-check-bin-dir=${builtins.placeholder "check"}/bin" - "--with-check-lib-dir=${builtins.placeholder "check"}/lib" - ]) + ++ lib.optionals finalAttrs.doCheck ( + [ "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" ] + ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ + "--enable-install-unit-tests" + "--with-check-bin-dir=${builtins.placeholder "check"}/bin" + "--with-check-lib-dir=${builtins.placeholder "check"}/lib" + ]) ++ lib.optional (!canRunInstalled) "--disable-doc-gen"; enableParallelBuilding = true; makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; - doCheck = true; - installFlags = "sysconfdir=$(out)/etc"; postInstall = '' From ea2dd166235e049699cf7f70c243c2b83089f824 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 1 Dec 2023 15:35:21 +0100 Subject: [PATCH 714/909] Use a proper enum rather than a boolean in runProgramInStore Makes the call-site much easier to understand. --- src/nix/develop.cc | 2 +- src/nix/fmt.cc | 2 +- src/nix/run.cc | 8 ++++---- src/nix/run.hh | 7 ++++++- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index ae9be79a3..606b044b0 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -662,7 +662,7 @@ struct CmdDevelop : Common, MixEnvironment } } - runProgramInStore(store, true, shell, args, buildEnvironment.getSystem()); + runProgramInStore(store, UseSearchPath::Use, shell, args, buildEnvironment.getSystem()); } }; diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc index 396c93dbb..059904150 100644 --- a/src/nix/fmt.cc +++ b/src/nix/fmt.cc @@ -49,7 +49,7 @@ struct CmdFmt : SourceExprCommand { } } - runProgramInStore(store, false, app.program, programArgs); + runProgramInStore(store, UseSearchPath::DontUse, app.program, programArgs); }; }; diff --git a/src/nix/run.cc b/src/nix/run.cc index d531f712d..efc0c56a1 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -25,7 +25,7 @@ std::string chrootHelperName = "__run_in_chroot"; namespace nix { void runProgramInStore(ref store, - bool search, + UseSearchPath useSearchPath, const std::string & program, const Strings & args, std::optional system) @@ -59,7 +59,7 @@ void runProgramInStore(ref store, if (system) setPersonality(*system); - if (search) + if (useSearchPath == UseSearchPath::Use) execvp(program.c_str(), stringsToCharPtrs(args).data()); else execv(program.c_str(), stringsToCharPtrs(args).data()); @@ -136,7 +136,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment Strings args; for (auto & arg : command) args.push_back(arg); - runProgramInStore(store, true, *command.begin(), args); + runProgramInStore(store, UseSearchPath::Use, *command.begin(), args); } }; @@ -198,7 +198,7 @@ struct CmdRun : InstallableValueCommand Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); - runProgramInStore(store, false, app.program, allArgs); + runProgramInStore(store, UseSearchPath::DontUse, app.program, allArgs); } }; diff --git a/src/nix/run.hh b/src/nix/run.hh index c62287e7e..a55917b06 100644 --- a/src/nix/run.hh +++ b/src/nix/run.hh @@ -5,8 +5,13 @@ namespace nix { +enum struct UseSearchPath { + Use, + DontUse +}; + void runProgramInStore(ref store, - bool search, + UseSearchPath useSearchPath, const std::string & program, const Strings & args, std::optional system = std::nullopt); From d59bdbe4fd757d99b6625db1d3560a39a371d9e9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 1 Dec 2023 10:20:19 -0500 Subject: [PATCH 715/909] Add two missing `#include "nar-info.hh"` GitHub's racy CI caused this oversight to sneak through. --- src/libstore/tests/nar-info.cc | 1 + src/nix/path-info.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/src/libstore/tests/nar-info.cc b/src/libstore/tests/nar-info.cc index c5b21d56b..88e6e1add 100644 --- a/src/libstore/tests/nar-info.cc +++ b/src/libstore/tests/nar-info.cc @@ -2,6 +2,7 @@ #include #include "path-info.hh" +#include "nar-info.hh" #include "tests/characterization.hh" #include "tests/libstore.hh" diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 080d6bbf1..5f10cfb61 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -2,6 +2,7 @@ #include "shared.hh" #include "store-api.hh" #include "common-args.hh" +#include "nar-info.hh" #include #include From 91b6833686a6a6d9eac7f3f66393ec89ef1d3b57 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Aug 2023 10:20:28 -0400 Subject: [PATCH 716/909] Move tests to separate directories, and document Today, with the tests inside a `tests` intermingled with the corresponding library's source code, we have a few problems: - We have to be careful that wildcards don't end up with tests being built as part of Nix proper, or test headers being installed as part of Nix proper. - Tests in libraries but not executables is not right: - It means each executable runs the previous unit tests again, because it needs the libraries. - It doesn't work right on Windows, which doesn't want you to load a DLL just for the side global variable . It could be made to work with the dlopen equivalent, but that's gross! This reorg solves these problems. There is a remaining problem which is that sibbling headers (like `hash.hh` the test header vs `hash.hh` the main `libnixutil` header) end up shadowing each other. This PR doesn't solve that. That is left as future work for a future PR. Co-authored-by: Valentin Gagarin --- .gitignore | 6 +- Makefile | 10 ++- doc/internal-api/doxygen.cfg.in | 12 ++- doc/manual/src/contributing/testing.md | 67 ++++++++++---- flake.nix | 2 +- mk/common-test.sh | 2 +- mk/programs.mk | 2 +- src/libexpr/tests/local.mk | 23 ----- src/libstore/tests/local.mk | 37 -------- src/libutil/tests/local.mk | 41 --------- tests/unit/libexpr-support/local.mk | 23 +++++ .../unit/libexpr-support}/tests/libexpr.hh | 0 .../libexpr-support/tests/value/context.cc | 30 +++++++ .../libexpr-support}/tests/value/context.hh | 2 +- .../unit/libexpr}/derived-path.cc | 0 .../unit/libexpr}/error_traces.cc | 0 .../tests => tests/unit/libexpr}/flakeref.cc | 0 .../tests => tests/unit/libexpr}/json.cc | 0 tests/unit/libexpr/local.mk | 36 ++++++++ .../tests => tests/unit/libexpr}/primops.cc | 0 .../unit/libexpr}/search-path.cc | 0 .../tests => tests/unit/libexpr}/trivial.cc | 0 .../unit/libexpr}/value/context.cc | 30 ------- .../unit/libexpr}/value/print.cc | 0 tests/unit/libstore-support/local.mk | 21 +++++ .../libstore-support/tests/derived-path.cc | 57 ++++++++++++ .../libstore-support}/tests/derived-path.hh | 0 .../unit/libstore-support}/tests/libstore.hh | 0 .../libstore-support/tests/outputs-spec.cc | 24 +++++ .../libstore-support}/tests/outputs-spec.hh | 2 +- tests/unit/libstore-support/tests/path.cc | 82 ++++++++++++++++++ .../unit/libstore-support}/tests/path.hh | 3 + .../unit/libstore-support}/tests/protocol.hh | 2 +- .../unit/libstore}/common-protocol.cc | 0 .../data}/common-protocol/content-address.bin | Bin .../data}/common-protocol/drv-output.bin | Bin .../optional-content-address.bin | Bin .../common-protocol/optional-store-path.bin | Bin .../data}/common-protocol/realisation.bin | Bin .../libstore/data}/common-protocol/set.bin | Bin .../data}/common-protocol/store-path.bin | Bin .../libstore/data}/common-protocol/string.bin | Bin .../libstore/data}/common-protocol/vector.bin | Bin .../derivation/bad-old-version-dyn-deps.drv | 0 .../libstore/data}/derivation/bad-version.drv | 0 .../data}/derivation/dynDerivationDeps.drv | 0 .../data}/derivation/dynDerivationDeps.json | 0 .../data}/derivation/output-caFixedFlat.json | 0 .../data}/derivation/output-caFixedNAR.json | 0 .../data}/derivation/output-caFixedText.json | 0 .../data}/derivation/output-caFloating.json | 0 .../data}/derivation/output-deferred.json | 0 .../data}/derivation/output-impure.json | 0 .../derivation/output-inputAddressed.json | 0 .../unit/libstore/data}/derivation/simple.drv | 0 .../libstore/data}/derivation/simple.json | 0 .../unit/libstore/data}/nar-info/impure.json | 0 .../unit/libstore/data}/nar-info/pure.json | 0 .../unit/libstore/data}/path-info/impure.json | 0 .../unit/libstore/data}/path-info/pure.json | 0 .../data}/serve-protocol/build-result-2.2.bin | Bin .../data}/serve-protocol/build-result-2.3.bin | Bin .../data}/serve-protocol/build-result-2.6.bin | Bin .../data}/serve-protocol/content-address.bin | Bin .../data}/serve-protocol/drv-output.bin | Bin .../optional-content-address.bin | Bin .../serve-protocol/optional-store-path.bin | Bin .../data}/serve-protocol/realisation.bin | Bin .../libstore/data}/serve-protocol/set.bin | Bin .../data}/serve-protocol/store-path.bin | Bin .../libstore/data}/serve-protocol/string.bin | Bin .../libstore/data}/serve-protocol/vector.bin | Bin .../worker-protocol/build-result-1.27.bin | Bin .../worker-protocol/build-result-1.28.bin | Bin .../worker-protocol/build-result-1.29.bin | Bin .../data}/worker-protocol/content-address.bin | Bin .../worker-protocol/derived-path-1.29.bin | Bin .../worker-protocol/derived-path-1.30.bin | Bin .../data}/worker-protocol/drv-output.bin | Bin .../keyed-build-result-1.29.bin | Bin .../optional-content-address.bin | Bin .../worker-protocol/optional-store-path.bin | Bin .../worker-protocol/optional-trusted-flag.bin | Bin .../data}/worker-protocol/realisation.bin | Bin .../libstore/data}/worker-protocol/set.bin | Bin .../data}/worker-protocol/store-path.bin | Bin .../libstore/data}/worker-protocol/string.bin | Bin .../unkeyed-valid-path-info-1.15.bin | Bin .../worker-protocol/valid-path-info-1.15.bin | Bin .../worker-protocol/valid-path-info-1.16.bin | Bin .../libstore/data}/worker-protocol/vector.bin | Bin .../unit/libstore}/derivation.cc | 2 +- .../unit/libstore}/derived-path.cc | 53 ----------- .../unit/libstore}/downstream-placeholder.cc | 0 tests/unit/libstore/local.mk | 31 +++++++ .../tests => tests/unit/libstore}/machines.cc | 4 +- .../unit/libstore}/nar-info-disk-cache.cc | 0 .../tests => tests/unit/libstore}/nar-info.cc | 2 +- .../unit/libstore}/outputs-spec.cc | 27 +----- .../unit/libstore}/path-info.cc | 2 +- .../tests => tests/unit/libstore}/path.cc | 73 ---------------- .../unit/libstore}/references.cc | 0 .../unit/libstore}/serve-protocol.cc | 0 .../libstore}/test-data/machines.bad_format | 0 .../unit/libstore}/test-data/machines.valid | 0 .../unit/libstore}/worker-protocol.cc | 0 tests/unit/libutil-support/local.mk | 19 ++++ .../tests/characterization.hh | 4 +- tests/unit/libutil-support/tests/hash.cc | 20 +++++ .../unit/libutil-support}/tests/hash.hh | 0 .../tests => tests/unit/libutil}/args.cc | 6 +- .../unit/libutil}/canon-path.cc | 0 .../unit/libutil}/chunked-vector.cc | 0 .../tests => tests/unit/libutil}/closure.cc | 0 .../unit/libutil}/compression.cc | 0 .../tests => tests/unit/libutil}/config.cc | 0 .../unit/libutil/data}/git/check-data.sh | 2 +- .../libutil/data}/git/hello-world-blob.bin | Bin .../unit/libutil/data}/git/hello-world.bin | Bin .../unit/libutil/data}/git/tree.bin | Bin .../unit/libutil/data}/git/tree.txt | 0 .../tests => tests/unit/libutil}/git.cc | 6 +- .../tests => tests/unit/libutil}/hash.cc | 20 +---- .../tests => tests/unit/libutil}/hilite.cc | 0 .../unit/libutil}/json-utils.cc | 0 tests/unit/libutil/local.mk | 31 +++++++ .../tests => tests/unit/libutil}/logging.cc | 0 .../tests => tests/unit/libutil}/lru-cache.cc | 0 .../tests => tests/unit/libutil}/pool.cc | 0 .../unit/libutil}/references.cc | 0 .../unit/libutil}/suggestions.cc | 0 .../tests => tests/unit/libutil}/tests.cc | 0 .../tests => tests/unit/libutil}/url.cc | 0 .../unit/libutil}/xml-writer.cc | 0 134 files changed, 464 insertions(+), 352 deletions(-) delete mode 100644 src/libexpr/tests/local.mk delete mode 100644 src/libstore/tests/local.mk delete mode 100644 src/libutil/tests/local.mk create mode 100644 tests/unit/libexpr-support/local.mk rename {src/libexpr => tests/unit/libexpr-support}/tests/libexpr.hh (100%) create mode 100644 tests/unit/libexpr-support/tests/value/context.cc rename {src/libexpr => tests/unit/libexpr-support}/tests/value/context.hh (95%) rename {src/libexpr/tests => tests/unit/libexpr}/derived-path.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/error_traces.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/flakeref.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/json.cc (100%) create mode 100644 tests/unit/libexpr/local.mk rename {src/libexpr/tests => tests/unit/libexpr}/primops.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/search-path.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/trivial.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/value/context.cc (83%) rename {src/libexpr/tests => tests/unit/libexpr}/value/print.cc (100%) create mode 100644 tests/unit/libstore-support/local.mk create mode 100644 tests/unit/libstore-support/tests/derived-path.cc rename {src/libstore => tests/unit/libstore-support}/tests/derived-path.hh (100%) rename {src/libstore => tests/unit/libstore-support}/tests/libstore.hh (100%) create mode 100644 tests/unit/libstore-support/tests/outputs-spec.cc rename {src/libstore => tests/unit/libstore-support}/tests/outputs-spec.hh (89%) create mode 100644 tests/unit/libstore-support/tests/path.cc rename {src/libstore => tests/unit/libstore-support}/tests/path.hh (82%) rename {src/libstore => tests/unit/libstore-support}/tests/protocol.hh (96%) rename {src/libstore/tests => tests/unit/libstore}/common-protocol.cc (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/drv-output.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/optional-content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/optional-store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/realisation.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/set.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/string.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/vector.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/bad-old-version-dyn-deps.drv (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/bad-version.drv (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/dynDerivationDeps.drv (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/dynDerivationDeps.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFixedFlat.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFixedNAR.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFixedText.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFloating.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-deferred.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-impure.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-inputAddressed.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/simple.drv (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/simple.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/nar-info/impure.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/nar-info/pure.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/path-info/impure.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/path-info/pure.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/build-result-2.2.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/build-result-2.3.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/build-result-2.6.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/drv-output.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/optional-content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/optional-store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/realisation.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/set.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/string.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/vector.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/build-result-1.27.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/build-result-1.28.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/build-result-1.29.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/derived-path-1.29.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/derived-path-1.30.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/drv-output.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/keyed-build-result-1.29.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/optional-content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/optional-store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/optional-trusted-flag.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/realisation.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/set.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/string.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/unkeyed-valid-path-info-1.15.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/valid-path-info-1.15.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/valid-path-info-1.16.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/vector.bin (100%) rename {src/libstore/tests => tests/unit/libstore}/derivation.cc (99%) rename {src/libstore/tests => tests/unit/libstore}/derived-path.cc (66%) rename {src/libstore/tests => tests/unit/libstore}/downstream-placeholder.cc (100%) create mode 100644 tests/unit/libstore/local.mk rename {src/libstore/tests => tests/unit/libstore}/machines.cc (97%) rename {src/libstore/tests => tests/unit/libstore}/nar-info-disk-cache.cc (100%) rename {src/libstore/tests => tests/unit/libstore}/nar-info.cc (98%) rename {src/libstore/tests => tests/unit/libstore}/outputs-spec.cc (92%) rename {src/libstore/tests => tests/unit/libstore}/path-info.cc (97%) rename {src/libstore/tests => tests/unit/libstore}/path.cc (59%) rename {src/libstore/tests => tests/unit/libstore}/references.cc (100%) rename {src/libstore/tests => tests/unit/libstore}/serve-protocol.cc (100%) rename {src/libstore/tests => tests/unit/libstore}/test-data/machines.bad_format (100%) rename {src/libstore/tests => tests/unit/libstore}/test-data/machines.valid (100%) rename {src/libstore/tests => tests/unit/libstore}/worker-protocol.cc (100%) create mode 100644 tests/unit/libutil-support/local.mk rename {src/libutil => tests/unit/libutil-support}/tests/characterization.hh (95%) create mode 100644 tests/unit/libutil-support/tests/hash.cc rename {src/libutil => tests/unit/libutil-support}/tests/hash.hh (100%) rename {src/libutil/tests => tests/unit/libutil}/args.cc (98%) rename {src/libutil/tests => tests/unit/libutil}/canon-path.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/chunked-vector.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/closure.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/compression.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/config.cc (100%) rename {unit-test-data/libutil => tests/unit/libutil/data}/git/check-data.sh (98%) rename {unit-test-data/libutil => tests/unit/libutil/data}/git/hello-world-blob.bin (100%) rename {unit-test-data/libutil => tests/unit/libutil/data}/git/hello-world.bin (100%) rename {unit-test-data/libutil => tests/unit/libutil/data}/git/tree.bin (100%) rename {unit-test-data/libutil => tests/unit/libutil/data}/git/tree.txt (100%) rename {src/libutil/tests => tests/unit/libutil}/git.cc (97%) rename {src/libutil/tests => tests/unit/libutil}/hash.cc (92%) rename {src/libutil/tests => tests/unit/libutil}/hilite.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/json-utils.cc (100%) create mode 100644 tests/unit/libutil/local.mk rename {src/libutil/tests => tests/unit/libutil}/logging.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/lru-cache.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/pool.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/references.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/suggestions.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/tests.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/url.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/xml-writer.cc (100%) diff --git a/.gitignore b/.gitignore index 767a5d6ed..d9f9d949b 100644 --- a/.gitignore +++ b/.gitignore @@ -45,14 +45,14 @@ perl/Makefile.config /src/libexpr/parser-tab.hh /src/libexpr/parser-tab.output /src/libexpr/nix.tbl -/src/libexpr/tests/libnixexpr-tests +/tests/unit/libexpr/libnixexpr-tests # /src/libstore/ *.gen.* -/src/libstore/tests/libnixstore-tests +/tests/unit/libstore/libnixstore-tests # /src/libutil/ -/src/libutil/tests/libnixutil-tests +/tests/unit/libutil/libnixutil-tests /src/nix/nix diff --git a/Makefile b/Makefile index 92727bea5..eea297c89 100644 --- a/Makefile +++ b/Makefile @@ -25,11 +25,13 @@ makefiles = \ endif ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes) -UNIT_TEST_ENV = _NIX_TEST_UNIT_DATA=unit-test-data makefiles += \ - src/libutil/tests/local.mk \ - src/libstore/tests/local.mk \ - src/libexpr/tests/local.mk + tests/unit/libutil/local.mk \ + tests/unit/libutil-support/local.mk \ + tests/unit/libstore/local.mk \ + tests/unit/libstore-support/local.mk \ + tests/unit/libexpr/local.mk \ + tests/unit/libexpr-support/local.mk endif ifeq ($(ENABLE_TESTS), yes) diff --git a/doc/internal-api/doxygen.cfg.in b/doc/internal-api/doxygen.cfg.in index 599be2470..ad5af97e6 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/doc/internal-api/doxygen.cfg.in @@ -39,17 +39,21 @@ INPUT = \ src/libcmd \ src/libexpr \ src/libexpr/flake \ - src/libexpr/tests \ - src/libexpr/tests/value \ + tests/unit/libexpr \ + tests/unit/libexpr/value \ + tests/unit/libexpr/test \ + tests/unit/libexpr/test/value \ src/libexpr/value \ src/libfetchers \ src/libmain \ src/libstore \ src/libstore/build \ src/libstore/builtins \ - src/libstore/tests \ + tests/unit/libstore \ + tests/unit/libstore/test \ src/libutil \ - src/libutil/tests \ + tests/unit/libutil \ + tests/unit/libutil/test \ src/nix \ src/nix-env \ src/nix-store diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index 0b45b88a3..d8d162379 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -20,6 +20,7 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. [googletest]: https://google.github.io/googletest/ [rapidcheck]: https://github.com/emil-e/rapidcheck +[property testing]: https://en.wikipedia.org/wiki/Property_testing ### Source and header layout @@ -28,34 +29,50 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. > ``` > src > ├── libexpr +> │ ├── local.mk > │ ├── value/context.hh > │ ├── value/context.cc +> │ … +> │ +> ├── tests > │ │ > │ … -> └── tests -> │ ├── value/context.hh -> │ ├── value/context.cc +> │ └── unit +> │ ├── libutil +> │ │ ├── local.mk +> │ │ … +> │ │ └── data +> │ │ ├── git/tree.txt +> │ │ … > │ │ -> │ … -> │ -> ├── unit-test-data -> │ ├── libstore -> │ │ ├── worker-protocol/content-address.bin -> │ │ … -> │ … +> │ ├── libexpr-support +> │ │ ├── local.mk +> │ │ └── tests +> │ │ ├── value/context.hh +> │ │ ├── value/context.cc +> │ │ … +> │ │ +> │ ├── libexpr +> │ … ├── local.mk +> │ ├── value/context.cc +> │ … > … > ``` -The unit tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_shortname}/tests` within the directory for the library (`src/${library_shortname}`). +The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`. +Given a interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`. -The data is in `unit-test-data`, with one subdir per library, with the same name as where the code goes. -For example, `libnixstore` code is in `src/libstore`, and its test data is in `unit-test-data/libstore`. -The path to the `unit-test-data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +Data for unit tests is stored in a `data` subdir of the directory for each unit test executable. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`. +The path to the `tests/unit/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +Note that each executable only gets the data for its tests. -> **Note** -> Due to the way googletest works, downstream unit test executables will actually include and re-run upstream library tests. -> Therefore it is important that the same value for `_NIX_TEST_UNIT_DATA` be used with the tests for each library. -> That is why we have the test data nested within a single `unit-test-data` directory. +The unit test libraries are in `tests/unit/${library_name_without-nix}-lib`. +All headers are in a `tests` subdirectory so they are included with `#include "tests/"`. + +The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing. +But organizing the tests this way has one big benefit: +there is no risk of any build-system wildcards for the library accidentally picking up test code that should not built and installed as part of the library. ### Running tests @@ -69,7 +86,7 @@ See [functional characterisation testing](#characterisation-testing-functional) Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used. For example: ```shell-session -$ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN +$ _NIX_TEST_ACCEPT=1 make libstore-tests_RUN ... [ SKIPPED ] WorkerProtoTest.string_read [ SKIPPED ] WorkerProtoTest.string_write @@ -80,6 +97,18 @@ $ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN will regenerate the "golden master" expected result for the `libnixstore` characterisation tests. The characterisation tests will mark themselves "skipped" since they regenerated the expected result instead of actually testing anything. +### Unit test support libraries + +There are headers and code which are not just used to test the library in question, but also downstream libraries. +For example, we do [property testing] with the [rapidcheck] library. +This requires writing `Arbitrary` "instances", which are used to describe how to generate values of a given type for the sake of running property tests. +Because types contain other types, `Arbitrary` "instances" for some type are not just useful for testing that type, but also any other type that contains it. +Downstream types frequently contain upstream types, so it is very important that we share arbitrary instances so that downstream libraries' property tests can also use them. + +It is important that these testing libraries don't contain any actual tests themselves. +On some platforms they would be run as part of every test executable that uses them, which is redundant. +On other platforms they wouldn't be run at all. + ## Functional tests The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. diff --git a/flake.nix b/flake.nix index 822b3d31e..e2e510cbc 100644 --- a/flake.nix +++ b/flake.nix @@ -93,7 +93,7 @@ ./misc ./precompiled-headers.h ./src - ./unit-test-data + ./tests/unit ./COPYING ./scripts/local.mk functionalTestFiles diff --git a/mk/common-test.sh b/mk/common-test.sh index 00ccd1584..2783d293b 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -1,7 +1,7 @@ # Remove overall test dir (at most one of the two should match) and # remove file extension. test_name=$(echo -n "$test" | sed \ - -e "s|^unit-test-data/||" \ + -e "s|^tests/unit/[^/]*/data/||" \ -e "s|^tests/functional/||" \ -e "s|\.sh$||" \ ) diff --git a/mk/programs.mk b/mk/programs.mk index a88d9d949..6235311e9 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -87,6 +87,6 @@ define build-program # Phony target to run this program (typically as a dependency of 'check'). .PHONY: $(1)_RUN $(1)_RUN: $$($(1)_PATH) - $(trace-test) $$(UNIT_TEST_ENV) $$($(1)_PATH) + $(trace-test) $$($(1)_ENV) $$($(1)_PATH) endef diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk deleted file mode 100644 index 7689a03e0..000000000 --- a/src/libexpr/tests/local.mk +++ /dev/null @@ -1,23 +0,0 @@ -check: libexpr-tests_RUN - -programs += libexpr-tests - -libexpr-tests_NAME := libnixexpr-tests - -libexpr-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libexpr-tests_INSTALL_DIR := $(checkbindir) -else - libexpr-tests_INSTALL_DIR := -endif - -libexpr-tests_SOURCES := \ - $(wildcard $(d)/*.cc) \ - $(wildcard $(d)/value/*.cc) - -libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -I src/libfetchers - -libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers - -libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk deleted file mode 100644 index e9b8b4f99..000000000 --- a/src/libstore/tests/local.mk +++ /dev/null @@ -1,37 +0,0 @@ -check: libstore-tests-exe_RUN - -programs += libstore-tests-exe - -libstore-tests-exe_NAME = libnixstore-tests - -libstore-tests-exe_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libstore-tests-exe_INSTALL_DIR := $(checkbindir) -else - libstore-tests-exe_INSTALL_DIR := -endif - -libstore-tests-exe_LIBS = libstore-tests - -libstore-tests-exe_LDFLAGS := $(GTEST_LIBS) - -libraries += libstore-tests - -libstore-tests_NAME = libnixstore-tests - -libstore-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libstore-tests_INSTALL_DIR := $(checklibdir) -else - libstore-tests_INSTALL_DIR := -endif - -libstore-tests_SOURCES := $(wildcard $(d)/*.cc) - -libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil - -libstore-tests_LIBS = libutil-tests libstore libutil - -libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk deleted file mode 100644 index 66886c45f..000000000 --- a/src/libutil/tests/local.mk +++ /dev/null @@ -1,41 +0,0 @@ -check: libutil-tests-exe_RUN - -programs += libutil-tests-exe - -libutil-tests-exe_NAME = libnixutil-tests - -libutil-tests-exe_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libutil-tests-exe_INSTALL_DIR := $(checkbindir) -else - libutil-tests-exe_INSTALL_DIR := -endif - -libutil-tests-exe_LIBS = libutil-tests - -libutil-tests-exe_LDFLAGS := $(GTEST_LIBS) - -libraries += libutil-tests - -libutil-tests_NAME = libnixutil-tests - -libutil-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libutil-tests_INSTALL_DIR := $(checklibdir) -else - libutil-tests_INSTALL_DIR := -endif - -libutil-tests_SOURCES := $(wildcard $(d)/*.cc) - -libutil-tests_CXXFLAGS += -I src/libutil - -libutil-tests_LIBS = libutil - -libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) - -check: unit-test-data/libutil/git/check-data.sh.test - -$(eval $(call run-test,unit-test-data/libutil/git/check-data.sh)) diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk new file mode 100644 index 000000000..28e87b8f2 --- /dev/null +++ b/tests/unit/libexpr-support/local.mk @@ -0,0 +1,23 @@ +libraries += libexpr-test-support + +libexpr-test-support_NAME = libnixexpr-test-support + +libexpr-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libexpr-test-support_INSTALL_DIR := $(checklibdir) +else + libexpr-test-support_INSTALL_DIR := +endif + +libexpr-test-support_SOURCES := \ + $(wildcard $(d)/tests/*.cc) \ + $(wildcard $(d)/tests/value/*.cc) + +libexpr-test-support_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) + +libexpr-test-support_LIBS = \ + libstore-test-support libutil-test-support \ + libexpr libstore libutil + +libexpr-test-support_LDFLAGS := -lrapidcheck diff --git a/src/libexpr/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh similarity index 100% rename from src/libexpr/tests/libexpr.hh rename to tests/unit/libexpr-support/tests/libexpr.hh diff --git a/tests/unit/libexpr-support/tests/value/context.cc b/tests/unit/libexpr-support/tests/value/context.cc new file mode 100644 index 000000000..8658bdaef --- /dev/null +++ b/tests/unit/libexpr-support/tests/value/context.cc @@ -0,0 +1,30 @@ +#include + +#include "tests/path.hh" +#include "tests/value/context.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + return gen::just(NixStringContextElem::DrvDeep { + .drvPath = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + case 2: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +} diff --git a/src/libexpr/tests/value/context.hh b/tests/unit/libexpr-support/tests/value/context.hh similarity index 95% rename from src/libexpr/tests/value/context.hh rename to tests/unit/libexpr-support/tests/value/context.hh index c0bc97ba3..8c68c78bb 100644 --- a/src/libexpr/tests/value/context.hh +++ b/tests/unit/libexpr-support/tests/value/context.hh @@ -3,7 +3,7 @@ #include -#include +#include "value/context.hh" namespace rc { using namespace nix; diff --git a/src/libexpr/tests/derived-path.cc b/tests/unit/libexpr/derived-path.cc similarity index 100% rename from src/libexpr/tests/derived-path.cc rename to tests/unit/libexpr/derived-path.cc diff --git a/src/libexpr/tests/error_traces.cc b/tests/unit/libexpr/error_traces.cc similarity index 100% rename from src/libexpr/tests/error_traces.cc rename to tests/unit/libexpr/error_traces.cc diff --git a/src/libexpr/tests/flakeref.cc b/tests/unit/libexpr/flakeref.cc similarity index 100% rename from src/libexpr/tests/flakeref.cc rename to tests/unit/libexpr/flakeref.cc diff --git a/src/libexpr/tests/json.cc b/tests/unit/libexpr/json.cc similarity index 100% rename from src/libexpr/tests/json.cc rename to tests/unit/libexpr/json.cc diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk new file mode 100644 index 000000000..5743880d7 --- /dev/null +++ b/tests/unit/libexpr/local.mk @@ -0,0 +1,36 @@ +check: libexpr-tests_RUN + +programs += libexpr-tests + +libexpr-tests_NAME := libnixexpr-tests + +libexpr-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data + +libexpr-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libexpr-tests_INSTALL_DIR := $(checkbindir) +else + libexpr-tests_INSTALL_DIR := +endif + +libexpr-tests_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) + +libexpr-tests_EXTRA_INCLUDES = \ + -I tests/unit/libexpr-support \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + -I src/libexpr \ + -I src/libfetchers \ + -I src/libstore \ + -I src/libutil + +libexpr-tests_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) + +libexpr-tests_LIBS = \ + libexpr-test-support libstore-test-support libutils-test-support \ + libexpr libfetchers libstore libutil + +libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock diff --git a/src/libexpr/tests/primops.cc b/tests/unit/libexpr/primops.cc similarity index 100% rename from src/libexpr/tests/primops.cc rename to tests/unit/libexpr/primops.cc diff --git a/src/libexpr/tests/search-path.cc b/tests/unit/libexpr/search-path.cc similarity index 100% rename from src/libexpr/tests/search-path.cc rename to tests/unit/libexpr/search-path.cc diff --git a/src/libexpr/tests/trivial.cc b/tests/unit/libexpr/trivial.cc similarity index 100% rename from src/libexpr/tests/trivial.cc rename to tests/unit/libexpr/trivial.cc diff --git a/src/libexpr/tests/value/context.cc b/tests/unit/libexpr/value/context.cc similarity index 83% rename from src/libexpr/tests/value/context.cc rename to tests/unit/libexpr/value/context.cc index 92d4889ab..761286dbd 100644 --- a/src/libexpr/tests/value/context.cc +++ b/tests/unit/libexpr/value/context.cc @@ -117,36 +117,6 @@ TEST(NixStringContextElemTest, built_built_xp) { NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"), MissingExperimentalFeature); } -} - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - return gen::just(NixStringContextElem::DrvDeep { - .drvPath = *gen::arbitrary(), - }); -} - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just(*gen::arbitrary()); - case 1: - return gen::just(*gen::arbitrary()); - case 2: - return gen::just(*gen::arbitrary()); - default: - assert(false); - } -} - -} - -namespace nix { - #ifndef COVERAGE RC_GTEST_PROP( diff --git a/src/libexpr/tests/value/print.cc b/tests/unit/libexpr/value/print.cc similarity index 100% rename from src/libexpr/tests/value/print.cc rename to tests/unit/libexpr/value/print.cc diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk new file mode 100644 index 000000000..d5d657c91 --- /dev/null +++ b/tests/unit/libstore-support/local.mk @@ -0,0 +1,21 @@ +libraries += libstore-test-support + +libstore-test-support_NAME = libnixstore-test-support + +libstore-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-test-support_INSTALL_DIR := $(checklibdir) +else + libstore-test-support_INSTALL_DIR := +endif + +libstore-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) + +libstore-test-support_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) + +libstore-test-support_LIBS = \ + libutil-test-support \ + libstore libutil + +libstore-test-support_LDFLAGS := -lrapidcheck diff --git a/tests/unit/libstore-support/tests/derived-path.cc b/tests/unit/libstore-support/tests/derived-path.cc new file mode 100644 index 000000000..091706dba --- /dev/null +++ b/tests/unit/libstore-support/tests/derived-path.cc @@ -0,0 +1,57 @@ +#include + +#include + +#include "tests/derived-path.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + return gen::just(DerivedPath::Opaque { + .path = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(SingleDerivedPath::Built { + .drvPath = make_ref(*gen::arbitrary()), + .output = (*gen::arbitrary()).name, + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(DerivedPath::Built { + .drvPath = make_ref(*gen::arbitrary()), + .outputs = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +} diff --git a/src/libstore/tests/derived-path.hh b/tests/unit/libstore-support/tests/derived-path.hh similarity index 100% rename from src/libstore/tests/derived-path.hh rename to tests/unit/libstore-support/tests/derived-path.hh diff --git a/src/libstore/tests/libstore.hh b/tests/unit/libstore-support/tests/libstore.hh similarity index 100% rename from src/libstore/tests/libstore.hh rename to tests/unit/libstore-support/tests/libstore.hh diff --git a/tests/unit/libstore-support/tests/outputs-spec.cc b/tests/unit/libstore-support/tests/outputs-spec.cc new file mode 100644 index 000000000..e9d602203 --- /dev/null +++ b/tests/unit/libstore-support/tests/outputs-spec.cc @@ -0,0 +1,24 @@ +#include "tests/outputs-spec.hh" + +#include + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just((OutputsSpec) OutputsSpec::All { }); + case 1: + return gen::just((OutputsSpec) OutputsSpec::Names { + *gen::nonEmpty(gen::container(gen::map( + gen::arbitrary(), + [](StorePathName n) { return n.name; }))), + }); + default: + assert(false); + } +} + +} diff --git a/src/libstore/tests/outputs-spec.hh b/tests/unit/libstore-support/tests/outputs-spec.hh similarity index 89% rename from src/libstore/tests/outputs-spec.hh rename to tests/unit/libstore-support/tests/outputs-spec.hh index ded331b33..f5bf9042d 100644 --- a/src/libstore/tests/outputs-spec.hh +++ b/tests/unit/libstore-support/tests/outputs-spec.hh @@ -5,7 +5,7 @@ #include -#include +#include "tests/path.hh" namespace rc { using namespace nix; diff --git a/tests/unit/libstore-support/tests/path.cc b/tests/unit/libstore-support/tests/path.cc new file mode 100644 index 000000000..e5f169e94 --- /dev/null +++ b/tests/unit/libstore-support/tests/path.cc @@ -0,0 +1,82 @@ +#include + +#include + +#include "path-regex.hh" +#include "store-api.hh" + +#include "tests/hash.hh" +#include "tests/path.hh" + +namespace nix { + +void showValue(const StorePath & p, std::ostream & os) +{ + os << p.to_string(); +} + +} + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + auto len = *gen::inRange( + 1, + StorePath::MaxPathLen - StorePath::HashLen); + + std::string pre; + pre.reserve(len); + + for (size_t c = 0; c < len; ++c) { + switch (auto i = *gen::inRange(0, 10 + 2 * 26 + 6)) { + case 0 ... 9: + pre += '0' + i; + case 10 ... 35: + pre += 'A' + (i - 10); + break; + case 36 ... 61: + pre += 'a' + (i - 36); + break; + case 62: + pre += '+'; + break; + case 63: + pre += '-'; + break; + case 64: + // names aren't permitted to start with a period, + // so just fall through to the next case here + if (c != 0) { + pre += '.'; + break; + } + case 65: + pre += '_'; + break; + case 66: + pre += '?'; + break; + case 67: + pre += '='; + break; + default: + assert(false); + } + } + + return gen::just(StorePathName { + .name = std::move(pre), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(StorePath { + *gen::arbitrary(), + (*gen::arbitrary()).name, + }); +} + +} // namespace rc diff --git a/src/libstore/tests/path.hh b/tests/unit/libstore-support/tests/path.hh similarity index 82% rename from src/libstore/tests/path.hh rename to tests/unit/libstore-support/tests/path.hh index 21cb62310..4751b3373 100644 --- a/src/libstore/tests/path.hh +++ b/tests/unit/libstore-support/tests/path.hh @@ -11,6 +11,9 @@ struct StorePathName { std::string name; }; +// For rapidcheck +void showValue(const StorePath & p, std::ostream & os); + } namespace rc { diff --git a/src/libstore/tests/protocol.hh b/tests/unit/libstore-support/tests/protocol.hh similarity index 96% rename from src/libstore/tests/protocol.hh rename to tests/unit/libstore-support/tests/protocol.hh index 466032a79..3c9e52c11 100644 --- a/src/libstore/tests/protocol.hh +++ b/tests/unit/libstore-support/tests/protocol.hh @@ -12,7 +12,7 @@ namespace nix { template class ProtoTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/libstore/" + protocolDir; + Path unitTestData = getUnitTestData() + "/" + protocolDir; Path goldenMaster(std::string_view testStem) const override { return unitTestData + "/" + testStem + ".bin"; diff --git a/src/libstore/tests/common-protocol.cc b/tests/unit/libstore/common-protocol.cc similarity index 100% rename from src/libstore/tests/common-protocol.cc rename to tests/unit/libstore/common-protocol.cc diff --git a/unit-test-data/libstore/common-protocol/content-address.bin b/tests/unit/libstore/data/common-protocol/content-address.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/content-address.bin rename to tests/unit/libstore/data/common-protocol/content-address.bin diff --git a/unit-test-data/libstore/common-protocol/drv-output.bin b/tests/unit/libstore/data/common-protocol/drv-output.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/drv-output.bin rename to tests/unit/libstore/data/common-protocol/drv-output.bin diff --git a/unit-test-data/libstore/common-protocol/optional-content-address.bin b/tests/unit/libstore/data/common-protocol/optional-content-address.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/optional-content-address.bin rename to tests/unit/libstore/data/common-protocol/optional-content-address.bin diff --git a/unit-test-data/libstore/common-protocol/optional-store-path.bin b/tests/unit/libstore/data/common-protocol/optional-store-path.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/optional-store-path.bin rename to tests/unit/libstore/data/common-protocol/optional-store-path.bin diff --git a/unit-test-data/libstore/common-protocol/realisation.bin b/tests/unit/libstore/data/common-protocol/realisation.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/realisation.bin rename to tests/unit/libstore/data/common-protocol/realisation.bin diff --git a/unit-test-data/libstore/common-protocol/set.bin b/tests/unit/libstore/data/common-protocol/set.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/set.bin rename to tests/unit/libstore/data/common-protocol/set.bin diff --git a/unit-test-data/libstore/common-protocol/store-path.bin b/tests/unit/libstore/data/common-protocol/store-path.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/store-path.bin rename to tests/unit/libstore/data/common-protocol/store-path.bin diff --git a/unit-test-data/libstore/common-protocol/string.bin b/tests/unit/libstore/data/common-protocol/string.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/string.bin rename to tests/unit/libstore/data/common-protocol/string.bin diff --git a/unit-test-data/libstore/common-protocol/vector.bin b/tests/unit/libstore/data/common-protocol/vector.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/vector.bin rename to tests/unit/libstore/data/common-protocol/vector.bin diff --git a/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv b/tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv similarity index 100% rename from unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv rename to tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv diff --git a/unit-test-data/libstore/derivation/bad-version.drv b/tests/unit/libstore/data/derivation/bad-version.drv similarity index 100% rename from unit-test-data/libstore/derivation/bad-version.drv rename to tests/unit/libstore/data/derivation/bad-version.drv diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.drv b/tests/unit/libstore/data/derivation/dynDerivationDeps.drv similarity index 100% rename from unit-test-data/libstore/derivation/dynDerivationDeps.drv rename to tests/unit/libstore/data/derivation/dynDerivationDeps.drv diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.json b/tests/unit/libstore/data/derivation/dynDerivationDeps.json similarity index 100% rename from unit-test-data/libstore/derivation/dynDerivationDeps.json rename to tests/unit/libstore/data/derivation/dynDerivationDeps.json diff --git a/unit-test-data/libstore/derivation/output-caFixedFlat.json b/tests/unit/libstore/data/derivation/output-caFixedFlat.json similarity index 100% rename from unit-test-data/libstore/derivation/output-caFixedFlat.json rename to tests/unit/libstore/data/derivation/output-caFixedFlat.json diff --git a/unit-test-data/libstore/derivation/output-caFixedNAR.json b/tests/unit/libstore/data/derivation/output-caFixedNAR.json similarity index 100% rename from unit-test-data/libstore/derivation/output-caFixedNAR.json rename to tests/unit/libstore/data/derivation/output-caFixedNAR.json diff --git a/unit-test-data/libstore/derivation/output-caFixedText.json b/tests/unit/libstore/data/derivation/output-caFixedText.json similarity index 100% rename from unit-test-data/libstore/derivation/output-caFixedText.json rename to tests/unit/libstore/data/derivation/output-caFixedText.json diff --git a/unit-test-data/libstore/derivation/output-caFloating.json b/tests/unit/libstore/data/derivation/output-caFloating.json similarity index 100% rename from unit-test-data/libstore/derivation/output-caFloating.json rename to tests/unit/libstore/data/derivation/output-caFloating.json diff --git a/unit-test-data/libstore/derivation/output-deferred.json b/tests/unit/libstore/data/derivation/output-deferred.json similarity index 100% rename from unit-test-data/libstore/derivation/output-deferred.json rename to tests/unit/libstore/data/derivation/output-deferred.json diff --git a/unit-test-data/libstore/derivation/output-impure.json b/tests/unit/libstore/data/derivation/output-impure.json similarity index 100% rename from unit-test-data/libstore/derivation/output-impure.json rename to tests/unit/libstore/data/derivation/output-impure.json diff --git a/unit-test-data/libstore/derivation/output-inputAddressed.json b/tests/unit/libstore/data/derivation/output-inputAddressed.json similarity index 100% rename from unit-test-data/libstore/derivation/output-inputAddressed.json rename to tests/unit/libstore/data/derivation/output-inputAddressed.json diff --git a/unit-test-data/libstore/derivation/simple.drv b/tests/unit/libstore/data/derivation/simple.drv similarity index 100% rename from unit-test-data/libstore/derivation/simple.drv rename to tests/unit/libstore/data/derivation/simple.drv diff --git a/unit-test-data/libstore/derivation/simple.json b/tests/unit/libstore/data/derivation/simple.json similarity index 100% rename from unit-test-data/libstore/derivation/simple.json rename to tests/unit/libstore/data/derivation/simple.json diff --git a/unit-test-data/libstore/nar-info/impure.json b/tests/unit/libstore/data/nar-info/impure.json similarity index 100% rename from unit-test-data/libstore/nar-info/impure.json rename to tests/unit/libstore/data/nar-info/impure.json diff --git a/unit-test-data/libstore/nar-info/pure.json b/tests/unit/libstore/data/nar-info/pure.json similarity index 100% rename from unit-test-data/libstore/nar-info/pure.json rename to tests/unit/libstore/data/nar-info/pure.json diff --git a/unit-test-data/libstore/path-info/impure.json b/tests/unit/libstore/data/path-info/impure.json similarity index 100% rename from unit-test-data/libstore/path-info/impure.json rename to tests/unit/libstore/data/path-info/impure.json diff --git a/unit-test-data/libstore/path-info/pure.json b/tests/unit/libstore/data/path-info/pure.json similarity index 100% rename from unit-test-data/libstore/path-info/pure.json rename to tests/unit/libstore/data/path-info/pure.json diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.2.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.2.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/build-result-2.2.bin rename to tests/unit/libstore/data/serve-protocol/build-result-2.2.bin diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.3.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.3.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/build-result-2.3.bin rename to tests/unit/libstore/data/serve-protocol/build-result-2.3.bin diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.6.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.6.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/build-result-2.6.bin rename to tests/unit/libstore/data/serve-protocol/build-result-2.6.bin diff --git a/unit-test-data/libstore/serve-protocol/content-address.bin b/tests/unit/libstore/data/serve-protocol/content-address.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/content-address.bin rename to tests/unit/libstore/data/serve-protocol/content-address.bin diff --git a/unit-test-data/libstore/serve-protocol/drv-output.bin b/tests/unit/libstore/data/serve-protocol/drv-output.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/drv-output.bin rename to tests/unit/libstore/data/serve-protocol/drv-output.bin diff --git a/unit-test-data/libstore/serve-protocol/optional-content-address.bin b/tests/unit/libstore/data/serve-protocol/optional-content-address.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/optional-content-address.bin rename to tests/unit/libstore/data/serve-protocol/optional-content-address.bin diff --git a/unit-test-data/libstore/serve-protocol/optional-store-path.bin b/tests/unit/libstore/data/serve-protocol/optional-store-path.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/optional-store-path.bin rename to tests/unit/libstore/data/serve-protocol/optional-store-path.bin diff --git a/unit-test-data/libstore/serve-protocol/realisation.bin b/tests/unit/libstore/data/serve-protocol/realisation.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/realisation.bin rename to tests/unit/libstore/data/serve-protocol/realisation.bin diff --git a/unit-test-data/libstore/serve-protocol/set.bin b/tests/unit/libstore/data/serve-protocol/set.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/set.bin rename to tests/unit/libstore/data/serve-protocol/set.bin diff --git a/unit-test-data/libstore/serve-protocol/store-path.bin b/tests/unit/libstore/data/serve-protocol/store-path.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/store-path.bin rename to tests/unit/libstore/data/serve-protocol/store-path.bin diff --git a/unit-test-data/libstore/serve-protocol/string.bin b/tests/unit/libstore/data/serve-protocol/string.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/string.bin rename to tests/unit/libstore/data/serve-protocol/string.bin diff --git a/unit-test-data/libstore/serve-protocol/vector.bin b/tests/unit/libstore/data/serve-protocol/vector.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/vector.bin rename to tests/unit/libstore/data/serve-protocol/vector.bin diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.27.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.27.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/build-result-1.27.bin rename to tests/unit/libstore/data/worker-protocol/build-result-1.27.bin diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.28.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.28.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/build-result-1.28.bin rename to tests/unit/libstore/data/worker-protocol/build-result-1.28.bin diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.29.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.29.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/build-result-1.29.bin rename to tests/unit/libstore/data/worker-protocol/build-result-1.29.bin diff --git a/unit-test-data/libstore/worker-protocol/content-address.bin b/tests/unit/libstore/data/worker-protocol/content-address.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/content-address.bin rename to tests/unit/libstore/data/worker-protocol/content-address.bin diff --git a/unit-test-data/libstore/worker-protocol/derived-path-1.29.bin b/tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/derived-path-1.29.bin rename to tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin diff --git a/unit-test-data/libstore/worker-protocol/derived-path-1.30.bin b/tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/derived-path-1.30.bin rename to tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin diff --git a/unit-test-data/libstore/worker-protocol/drv-output.bin b/tests/unit/libstore/data/worker-protocol/drv-output.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/drv-output.bin rename to tests/unit/libstore/data/worker-protocol/drv-output.bin diff --git a/unit-test-data/libstore/worker-protocol/keyed-build-result-1.29.bin b/tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/keyed-build-result-1.29.bin rename to tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin diff --git a/unit-test-data/libstore/worker-protocol/optional-content-address.bin b/tests/unit/libstore/data/worker-protocol/optional-content-address.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/optional-content-address.bin rename to tests/unit/libstore/data/worker-protocol/optional-content-address.bin diff --git a/unit-test-data/libstore/worker-protocol/optional-store-path.bin b/tests/unit/libstore/data/worker-protocol/optional-store-path.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/optional-store-path.bin rename to tests/unit/libstore/data/worker-protocol/optional-store-path.bin diff --git a/unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin b/tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin rename to tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin diff --git a/unit-test-data/libstore/worker-protocol/realisation.bin b/tests/unit/libstore/data/worker-protocol/realisation.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/realisation.bin rename to tests/unit/libstore/data/worker-protocol/realisation.bin diff --git a/unit-test-data/libstore/worker-protocol/set.bin b/tests/unit/libstore/data/worker-protocol/set.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/set.bin rename to tests/unit/libstore/data/worker-protocol/set.bin diff --git a/unit-test-data/libstore/worker-protocol/store-path.bin b/tests/unit/libstore/data/worker-protocol/store-path.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/store-path.bin rename to tests/unit/libstore/data/worker-protocol/store-path.bin diff --git a/unit-test-data/libstore/worker-protocol/string.bin b/tests/unit/libstore/data/worker-protocol/string.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/string.bin rename to tests/unit/libstore/data/worker-protocol/string.bin diff --git a/unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin b/tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin rename to tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin diff --git a/unit-test-data/libstore/worker-protocol/valid-path-info-1.15.bin b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/valid-path-info-1.15.bin rename to tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin diff --git a/unit-test-data/libstore/worker-protocol/valid-path-info-1.16.bin b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/valid-path-info-1.16.bin rename to tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin diff --git a/unit-test-data/libstore/worker-protocol/vector.bin b/tests/unit/libstore/data/worker-protocol/vector.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/vector.bin rename to tests/unit/libstore/data/worker-protocol/vector.bin diff --git a/src/libstore/tests/derivation.cc b/tests/unit/libstore/derivation.cc similarity index 99% rename from src/libstore/tests/derivation.cc rename to tests/unit/libstore/derivation.cc index 7becfa5ab..a7f4488fa 100644 --- a/src/libstore/tests/derivation.cc +++ b/tests/unit/libstore/derivation.cc @@ -13,7 +13,7 @@ using nlohmann::json; class DerivationTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/libstore/derivation"; + Path unitTestData = getUnitTestData() + "/derivation"; public: Path goldenMaster(std::string_view testStem) const override { diff --git a/src/libstore/tests/derived-path.cc b/tests/unit/libstore/derived-path.cc similarity index 66% rename from src/libstore/tests/derived-path.cc rename to tests/unit/libstore/derived-path.cc index 3fa3c0801..c62d79a78 100644 --- a/src/libstore/tests/derived-path.cc +++ b/tests/unit/libstore/derived-path.cc @@ -1,64 +1,11 @@ #include -#include #include #include #include "tests/derived-path.hh" #include "tests/libstore.hh" -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - return gen::just(DerivedPath::Opaque { - .path = *gen::arbitrary(), - }); -} - -Gen Arbitrary::arbitrary() -{ - return gen::just(SingleDerivedPath::Built { - .drvPath = make_ref(*gen::arbitrary()), - .output = (*gen::arbitrary()).name, - }); -} - -Gen Arbitrary::arbitrary() -{ - return gen::just(DerivedPath::Built { - .drvPath = make_ref(*gen::arbitrary()), - .outputs = *gen::arbitrary(), - }); -} - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just(*gen::arbitrary()); - case 1: - return gen::just(*gen::arbitrary()); - default: - assert(false); - } -} - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just(*gen::arbitrary()); - case 1: - return gen::just(*gen::arbitrary()); - default: - assert(false); - } -} - -} - namespace nix { class DerivedPathTest : public LibStoreTest diff --git a/src/libstore/tests/downstream-placeholder.cc b/tests/unit/libstore/downstream-placeholder.cc similarity index 100% rename from src/libstore/tests/downstream-placeholder.cc rename to tests/unit/libstore/downstream-placeholder.cc diff --git a/tests/unit/libstore/local.mk b/tests/unit/libstore/local.mk new file mode 100644 index 000000000..63f6d011f --- /dev/null +++ b/tests/unit/libstore/local.mk @@ -0,0 +1,31 @@ +check: libstore-tests_RUN + +programs += libstore-tests + +libstore-tests_NAME = libnixstore-tests + +libstore-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data + +libstore-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-tests_INSTALL_DIR := $(checkbindir) +else + libstore-tests_INSTALL_DIR := +endif + +libstore-tests_SOURCES := $(wildcard $(d)/*.cc) + +libstore-tests_EXTRA_INCLUDES = \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + -I src/libstore \ + -I src/libutil + +libstore-tests_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) + +libstore-tests_LIBS = \ + libstore-test-support libutil-test-support \ + libstore libutil + +libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libstore/tests/machines.cc b/tests/unit/libstore/machines.cc similarity index 97% rename from src/libstore/tests/machines.cc rename to tests/unit/libstore/machines.cc index fede328ea..5b66e5a5b 100644 --- a/src/libstore/tests/machines.cc +++ b/tests/unit/libstore/machines.cc @@ -139,7 +139,7 @@ TEST(machines, getMachinesWithIncorrectFormat) { } TEST(machines, getMachinesWithCorrectFileReference) { - auto path = absPath("src/libstore/tests/test-data/machines.valid"); + auto path = absPath("tests/unit/libstore/test-data/machines.valid"); ASSERT_TRUE(pathExists(path)); settings.builders = std::string("@") + path; @@ -166,6 +166,6 @@ TEST(machines, getMachinesWithIncorrectFileReference) { } TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) { - settings.builders = std::string("@") + absPath("src/libstore/tests/test-data/machines.bad_format"); + settings.builders = std::string("@") + absPath("tests/unit/libstore/test-data/machines.bad_format"); EXPECT_THROW(getMachines(), FormatError); } diff --git a/src/libstore/tests/nar-info-disk-cache.cc b/tests/unit/libstore/nar-info-disk-cache.cc similarity index 100% rename from src/libstore/tests/nar-info-disk-cache.cc rename to tests/unit/libstore/nar-info-disk-cache.cc diff --git a/src/libstore/tests/nar-info.cc b/tests/unit/libstore/nar-info.cc similarity index 98% rename from src/libstore/tests/nar-info.cc rename to tests/unit/libstore/nar-info.cc index 88e6e1add..4f124e89e 100644 --- a/src/libstore/tests/nar-info.cc +++ b/tests/unit/libstore/nar-info.cc @@ -13,7 +13,7 @@ using nlohmann::json; class NarInfoTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/libstore/nar-info"; + Path unitTestData = getUnitTestData() + "/nar-info"; Path goldenMaster(PathView testStem) const override { return unitTestData + "/" + testStem + ".json"; diff --git a/src/libstore/tests/outputs-spec.cc b/tests/unit/libstore/outputs-spec.cc similarity index 92% rename from src/libstore/tests/outputs-spec.cc rename to tests/unit/libstore/outputs-spec.cc index 952945185..456196be1 100644 --- a/src/libstore/tests/outputs-spec.cc +++ b/tests/unit/libstore/outputs-spec.cc @@ -1,4 +1,4 @@ -#include "outputs-spec.hh" +#include "tests/outputs-spec.hh" #include #include @@ -199,31 +199,6 @@ TEST_JSON(ExtendedOutputsSpec, names, R"(["a","b"])", (ExtendedOutputsSpec::Expl #undef TEST_JSON -} - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just((OutputsSpec) OutputsSpec::All { }); - case 1: - return gen::just((OutputsSpec) OutputsSpec::Names { - *gen::nonEmpty(gen::container(gen::map( - gen::arbitrary(), - [](StorePathName n) { return n.name; }))), - }); - default: - assert(false); - } -} - -} - -namespace nix { - #ifndef COVERAGE RC_GTEST_PROP( diff --git a/src/libstore/tests/path-info.cc b/tests/unit/libstore/path-info.cc similarity index 97% rename from src/libstore/tests/path-info.cc rename to tests/unit/libstore/path-info.cc index 49bf623bd..18f00ca19 100644 --- a/src/libstore/tests/path-info.cc +++ b/tests/unit/libstore/path-info.cc @@ -12,7 +12,7 @@ using nlohmann::json; class PathInfoTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/libstore/path-info"; + Path unitTestData = getUnitTestData() + "/path-info"; Path goldenMaster(PathView testStem) const override { return unitTestData + "/" + testStem + ".json"; diff --git a/src/libstore/tests/path.cc b/tests/unit/libstore/path.cc similarity index 59% rename from src/libstore/tests/path.cc rename to tests/unit/libstore/path.cc index 5a84d646c..30631b5fd 100644 --- a/src/libstore/tests/path.cc +++ b/tests/unit/libstore/path.cc @@ -66,79 +66,6 @@ TEST_DO_PARSE(equals_sign, "foo=foo") #undef TEST_DO_PARSE -// For rapidcheck -void showValue(const StorePath & p, std::ostream & os) { - os << p.to_string(); -} - -} - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - auto len = *gen::inRange( - 1, - StorePath::MaxPathLen - std::string_view { HASH_PART }.size()); - - std::string pre; - pre.reserve(len); - - for (size_t c = 0; c < len; ++c) { - switch (auto i = *gen::inRange(0, 10 + 2 * 26 + 6)) { - case 0 ... 9: - pre += '0' + i; - case 10 ... 35: - pre += 'A' + (i - 10); - break; - case 36 ... 61: - pre += 'a' + (i - 36); - break; - case 62: - pre += '+'; - break; - case 63: - pre += '-'; - break; - case 64: - // names aren't permitted to start with a period, - // so just fall through to the next case here - if (c != 0) { - pre += '.'; - break; - } - case 65: - pre += '_'; - break; - case 66: - pre += '?'; - break; - case 67: - pre += '='; - break; - default: - assert(false); - } - } - - return gen::just(StorePathName { - .name = std::move(pre), - }); -} - -Gen Arbitrary::arbitrary() -{ - return gen::just(StorePath { - *gen::arbitrary(), - (*gen::arbitrary()).name, - }); -} - -} // namespace rc - -namespace nix { - #ifndef COVERAGE RC_GTEST_FIXTURE_PROP( diff --git a/src/libstore/tests/references.cc b/tests/unit/libstore/references.cc similarity index 100% rename from src/libstore/tests/references.cc rename to tests/unit/libstore/references.cc diff --git a/src/libstore/tests/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc similarity index 100% rename from src/libstore/tests/serve-protocol.cc rename to tests/unit/libstore/serve-protocol.cc diff --git a/src/libstore/tests/test-data/machines.bad_format b/tests/unit/libstore/test-data/machines.bad_format similarity index 100% rename from src/libstore/tests/test-data/machines.bad_format rename to tests/unit/libstore/test-data/machines.bad_format diff --git a/src/libstore/tests/test-data/machines.valid b/tests/unit/libstore/test-data/machines.valid similarity index 100% rename from src/libstore/tests/test-data/machines.valid rename to tests/unit/libstore/test-data/machines.valid diff --git a/src/libstore/tests/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc similarity index 100% rename from src/libstore/tests/worker-protocol.cc rename to tests/unit/libstore/worker-protocol.cc diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk new file mode 100644 index 000000000..43a1551e5 --- /dev/null +++ b/tests/unit/libutil-support/local.mk @@ -0,0 +1,19 @@ +libraries += libutil-test-support + +libutil-test-support_NAME = libnixutil-test-support + +libutil-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-test-support_INSTALL_DIR := $(checklibdir) +else + libutil-test-support_INSTALL_DIR := +endif + +libutil-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) + +libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) + +libutil-test-support_LIBS = libutil + +libutil-test-support_LDFLAGS := -lrapidcheck diff --git a/src/libutil/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh similarity index 95% rename from src/libutil/tests/characterization.hh rename to tests/unit/libutil-support/tests/characterization.hh index 6eb513d68..9d6c850f0 100644 --- a/src/libutil/tests/characterization.hh +++ b/tests/unit/libutil-support/tests/characterization.hh @@ -9,8 +9,8 @@ namespace nix { /** - * The path to the `unit-test-data` directory. See the contributing - * guide in the manual for further details. + * The path to the unit test data directory. See the contributing guide + * in the manual for further details. */ static Path getUnitTestData() { return getEnv("_NIX_TEST_UNIT_DATA").value(); diff --git a/tests/unit/libutil-support/tests/hash.cc b/tests/unit/libutil-support/tests/hash.cc new file mode 100644 index 000000000..577e9890e --- /dev/null +++ b/tests/unit/libutil-support/tests/hash.cc @@ -0,0 +1,20 @@ +#include + +#include + +#include "hash.hh" + +#include "tests/hash.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + Hash hash(htSHA1); + for (size_t i = 0; i < hash.hashSize; ++i) + hash.hash[i] = *gen::arbitrary(); + return gen::just(hash); +} + +} diff --git a/src/libutil/tests/hash.hh b/tests/unit/libutil-support/tests/hash.hh similarity index 100% rename from src/libutil/tests/hash.hh rename to tests/unit/libutil-support/tests/hash.hh diff --git a/src/libutil/tests/args.cc b/tests/unit/libutil/args.cc similarity index 98% rename from src/libutil/tests/args.cc rename to tests/unit/libutil/args.cc index bea74a8c8..950224430 100644 --- a/src/libutil/tests/args.cc +++ b/tests/unit/libutil/args.cc @@ -1,5 +1,5 @@ -#include "../args.hh" -#include "libutil/fs-sink.hh" +#include "args.hh" +#include "fs-sink.hh" #include #include @@ -165,4 +165,4 @@ RC_GTEST_PROP( #endif -} \ No newline at end of file +} diff --git a/src/libutil/tests/canon-path.cc b/tests/unit/libutil/canon-path.cc similarity index 100% rename from src/libutil/tests/canon-path.cc rename to tests/unit/libutil/canon-path.cc diff --git a/src/libutil/tests/chunked-vector.cc b/tests/unit/libutil/chunked-vector.cc similarity index 100% rename from src/libutil/tests/chunked-vector.cc rename to tests/unit/libutil/chunked-vector.cc diff --git a/src/libutil/tests/closure.cc b/tests/unit/libutil/closure.cc similarity index 100% rename from src/libutil/tests/closure.cc rename to tests/unit/libutil/closure.cc diff --git a/src/libutil/tests/compression.cc b/tests/unit/libutil/compression.cc similarity index 100% rename from src/libutil/tests/compression.cc rename to tests/unit/libutil/compression.cc diff --git a/src/libutil/tests/config.cc b/tests/unit/libutil/config.cc similarity index 100% rename from src/libutil/tests/config.cc rename to tests/unit/libutil/config.cc diff --git a/unit-test-data/libutil/git/check-data.sh b/tests/unit/libutil/data/git/check-data.sh similarity index 98% rename from unit-test-data/libutil/git/check-data.sh rename to tests/unit/libutil/data/git/check-data.sh index 68b705c95..b3f59c4f1 100644 --- a/unit-test-data/libutil/git/check-data.sh +++ b/tests/unit/libutil/data/git/check-data.sh @@ -2,7 +2,7 @@ set -eu -o pipefail -export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/git-hashing/unit-test-data +export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/git-hashing/check-data mkdir -p $TEST_ROOT repo="$TEST_ROOT/scratch" diff --git a/unit-test-data/libutil/git/hello-world-blob.bin b/tests/unit/libutil/data/git/hello-world-blob.bin similarity index 100% rename from unit-test-data/libutil/git/hello-world-blob.bin rename to tests/unit/libutil/data/git/hello-world-blob.bin diff --git a/unit-test-data/libutil/git/hello-world.bin b/tests/unit/libutil/data/git/hello-world.bin similarity index 100% rename from unit-test-data/libutil/git/hello-world.bin rename to tests/unit/libutil/data/git/hello-world.bin diff --git a/unit-test-data/libutil/git/tree.bin b/tests/unit/libutil/data/git/tree.bin similarity index 100% rename from unit-test-data/libutil/git/tree.bin rename to tests/unit/libutil/data/git/tree.bin diff --git a/unit-test-data/libutil/git/tree.txt b/tests/unit/libutil/data/git/tree.txt similarity index 100% rename from unit-test-data/libutil/git/tree.txt rename to tests/unit/libutil/data/git/tree.txt diff --git a/src/libutil/tests/git.cc b/tests/unit/libutil/git.cc similarity index 97% rename from src/libutil/tests/git.cc rename to tests/unit/libutil/git.cc index 2842ea4d0..551a2d105 100644 --- a/src/libutil/tests/git.cc +++ b/tests/unit/libutil/git.cc @@ -11,7 +11,7 @@ using namespace git; class GitTest : public CharacterizationTest { - Path unitTestData = getUnitTestData() + "/libutil/git"; + Path unitTestData = getUnitTestData() + "/git"; public: @@ -86,8 +86,8 @@ TEST_F(GitTest, blob_write) { /** * This data is for "shallow" tree tests. However, we use "real" hashes - * so that we can check our test data in the corresponding functional - * test (`git-hashing/unit-test-data`). + * so that we can check our test data in a small shell script test test + * (`tests/unit/libutil/data/git/check-data.sh`). */ const static Tree tree = { { diff --git a/src/libutil/tests/hash.cc b/tests/unit/libutil/hash.cc similarity index 92% rename from src/libutil/tests/hash.cc rename to tests/unit/libutil/hash.cc index 9a5ebbb30..92291afce 100644 --- a/src/libutil/tests/hash.cc +++ b/tests/unit/libutil/hash.cc @@ -1,12 +1,8 @@ #include -#include #include -#include -#include - -#include "tests/hash.hh" +#include "hash.hh" namespace nix { @@ -68,7 +64,6 @@ namespace nix { "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" "454d4423643ce80e2a9ac94fa54ca49f"); } - TEST(hashString, testKnownSHA512Hashes2) { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; @@ -95,16 +90,3 @@ namespace nix { ASSERT_EQ(parseHashFormatOpt("sha0042"), std::nullopt); } } - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - Hash hash(htSHA1); - for (size_t i = 0; i < hash.hashSize; ++i) - hash.hash[i] = *gen::arbitrary(); - return gen::just(hash); -} - -} diff --git a/src/libutil/tests/hilite.cc b/tests/unit/libutil/hilite.cc similarity index 100% rename from src/libutil/tests/hilite.cc rename to tests/unit/libutil/hilite.cc diff --git a/src/libutil/tests/json-utils.cc b/tests/unit/libutil/json-utils.cc similarity index 100% rename from src/libutil/tests/json-utils.cc rename to tests/unit/libutil/json-utils.cc diff --git a/tests/unit/libutil/local.mk b/tests/unit/libutil/local.mk new file mode 100644 index 000000000..930efb90b --- /dev/null +++ b/tests/unit/libutil/local.mk @@ -0,0 +1,31 @@ +check: libutil-tests_RUN + +programs += libutil-tests + +libutil-tests_NAME = libnixutil-tests + +libutil-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data + +libutil-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-tests_INSTALL_DIR := $(checkbindir) +else + libutil-tests_INSTALL_DIR := +endif + +libutil-tests_SOURCES := $(wildcard $(d)/*.cc) + +libutil-tests_EXTRA_INCLUDES = \ + -I tests/unit/libutil-support \ + -I src/libutil + +libutil-tests_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) + +libutil-tests_LIBS = libutil-test-support libutil + +libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) + +check: $(d)/data/git/check-data.sh.test + +$(eval $(call run-test,$(d)/data/git/check-data.sh)) diff --git a/src/libutil/tests/logging.cc b/tests/unit/libutil/logging.cc similarity index 100% rename from src/libutil/tests/logging.cc rename to tests/unit/libutil/logging.cc diff --git a/src/libutil/tests/lru-cache.cc b/tests/unit/libutil/lru-cache.cc similarity index 100% rename from src/libutil/tests/lru-cache.cc rename to tests/unit/libutil/lru-cache.cc diff --git a/src/libutil/tests/pool.cc b/tests/unit/libutil/pool.cc similarity index 100% rename from src/libutil/tests/pool.cc rename to tests/unit/libutil/pool.cc diff --git a/src/libutil/tests/references.cc b/tests/unit/libutil/references.cc similarity index 100% rename from src/libutil/tests/references.cc rename to tests/unit/libutil/references.cc diff --git a/src/libutil/tests/suggestions.cc b/tests/unit/libutil/suggestions.cc similarity index 100% rename from src/libutil/tests/suggestions.cc rename to tests/unit/libutil/suggestions.cc diff --git a/src/libutil/tests/tests.cc b/tests/unit/libutil/tests.cc similarity index 100% rename from src/libutil/tests/tests.cc rename to tests/unit/libutil/tests.cc diff --git a/src/libutil/tests/url.cc b/tests/unit/libutil/url.cc similarity index 100% rename from src/libutil/tests/url.cc rename to tests/unit/libutil/url.cc diff --git a/src/libutil/tests/xml-writer.cc b/tests/unit/libutil/xml-writer.cc similarity index 100% rename from src/libutil/tests/xml-writer.cc rename to tests/unit/libutil/xml-writer.cc From 7355a48b1a4ce2e393598c2a72ef520cba9d172d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 1 Dec 2023 15:55:18 -0500 Subject: [PATCH 717/909] flake.lock: Update Nixpkgs to fix static build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The problem was since switching to use libgit2, we had a package in our closure (`http-parser`) that was always trying to build as a shared object. Underlying Nixpkgs PR (a 23.05 backport) https://github.com/NixOS/nixpkgs/pull/271202 Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/9ba29e2346bc542e9909d1021e8fd7d4b3f64db0' (2023-11-13) → 'github:NixOS/nixpkgs/36c4ac09e9bebcec1fa7b7539cddb0c9e837409c' (2023-11-30) --- flake.lock | 8 ++++---- flake.nix | 8 +++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index f120d3b5f..3cb9e72c9 100644 --- a/flake.lock +++ b/flake.lock @@ -50,16 +50,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1700748986, - "narHash": "sha256-/nqLrNU297h3PCw4QyDpZKZEUHmialJdZW2ceYFobds=", + "lastModified": 1701355166, + "narHash": "sha256-4V7XMI0Gd+y0zsi++cEHd99u3GNL0xSTGRmiWKzGnUQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "9ba29e2346bc542e9909d1021e8fd7d4b3f64db0", + "rev": "36c4ac09e9bebcec1fa7b7539cddb0c9e837409c", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.05-small", + "ref": "staging-23.05", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index e2e510cbc..dbd45f053 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,13 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; + # TODO Go back to nixos-23.05-small once + # https://github.com/NixOS/nixpkgs/pull/271202 is merged. + # + # Also, do not grab arbitrary further staging commits. This PR was + # carefully made to be based on release-23.05 and just contain + # rebuild-causing changes to packages that Nix actually uses. + inputs.nixpkgs.url = "github:NixOS/nixpkgs/staging-23.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; From 59c4c82aebb814d864548c3ad2e9128ab6e902bf Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 2 Dec 2023 00:56:23 +0100 Subject: [PATCH 718/909] fix links in stores overview --- doc/manual/generate-manpage.nix | 3 ++- doc/manual/src/store/types/index.md.in | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 365422af7..ae31b2a1f 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -103,7 +103,8 @@ let ${allStores} ''; index = replaceStrings - [ "@store-types@" ] [ storesOverview ] + [ "@store-types@" "./local-store.md" "./local-daemon-store.md" ] + [ storesOverview "#local-store" "#local-daemon-store" ] details.doc; storesOverview = let diff --git a/doc/manual/src/store/types/index.md.in b/doc/manual/src/store/types/index.md.in index b4db553a2..a35161ce8 100644 --- a/doc/manual/src/store/types/index.md.in +++ b/doc/manual/src/store/types/index.md.in @@ -29,15 +29,15 @@ supported settings for each store type are documented below. The special store URL `auto` causes Nix to automatically select a store as follows: -* Use the [local store](#local-store) `/nix/store` if `/nix/var/nix` +* Use the [local store](./local-store.md) `/nix/store` if `/nix/var/nix` is writable by the current user. * Otherwise, if `/nix/var/nix/daemon-socket/socket` exists, [connect - to the Nix daemon listening on that socket](#local-daemon-store). + to the Nix daemon listening on that socket](./local-daemon-store.md). -* Otherwise, on Linux only, use the [local chroot store](#local-store) +* Otherwise, on Linux only, use the [local chroot store](./local-store.md) `~/.local/share/nix/root`, which will be created automatically if it does not exist. -* Otherwise, use the [local store](#local-store) `/nix/store`. +* Otherwise, use the [local store](./local-store.md) `/nix/store`. From 51adfb9b277fe54ad03fa2c9981585f123fcc200 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 2 Dec 2023 02:21:17 +0100 Subject: [PATCH 719/909] reword documentation on settings and attributes related to substitution - add links - be more concise - clarify the distinction between `preferLocalBuild` and `allowSubstitutes` --- .../src/language/advanced-attributes.md | 23 +++++-------------- src/libstore/globals.hh | 4 +--- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 282b75af2..5a6c00cd4 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -257,29 +257,18 @@ Derivations can declare some infrequently used optional attributes. of the environment (typically, a few hundred kilobyte). - [`preferLocalBuild`]{#adv-attr-preferLocalBuild}\ - If this attribute is set to `true` and [distributed building is - enabled](../advanced-topics/distributed-builds.md), then, if - possible, the derivation will be built locally instead of forwarded - to a remote machine. This is appropriate for trivial builders - where the cost of doing a download or remote build would exceed - the cost of building locally. + If this attribute is set to `true` and [distributed building is enabled](../advanced-topics/distributed-builds.md), then, if possible, the derivation will be built locally instead of being forwarded to a remote machine. + This is useful for derivations that are cheapest to build locally. - [`allowSubstitutes`]{#adv-attr-allowSubstitutes}\ - If this attribute is set to `false`, then Nix will always build this - derivation; it will not try to substitute its outputs. This is - useful for very trivial derivations (such as `writeText` in Nixpkgs) - that are cheaper to build than to substitute from a binary cache. + If this attribute is set to `false`, then Nix will always build this derivation (locally or remotely); it will not try to substitute its outputs. + This is useful for derivations that are cheaper to build than to substitute. - You may disable the effects of this attibute by enabling the - `always-allow-substitutes` configuration option in Nix. + This attribute can be ignored by setting [`always-allow-substitutes`](@docroot@/command-ref/conf-file.md#conf-always-allow-substitutes) to `true`. > **Note** > - > You need to have a builder configured which satisfies the - > derivation’s `system` attribute, since the derivation cannot be - > substituted. Thus it is usually a good idea to align `system` with - > `builtins.currentSystem` when setting `allowSubstitutes` to - > `false`. For most trivial derivations this should be the case. + > If set to `false`, the [`builder`](./derivations.md#attr-builder) should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ If the special attribute `__structuredAttrs` is set to `true`, the other derivation diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 38b0d516c..36ba51e23 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -268,9 +268,7 @@ public: Setting alwaysAllowSubstitutes{ this, false, "always-allow-substitutes", R"( - If set to `true`, Nix will ignore the `allowSubstitutes` attribute in - derivations and always attempt to use available substituters. - For more information on `allowSubstitutes`, see [the manual chapter on advanced attributes](../language/advanced-attributes.md). + If set to `true`, Nix will ignore the [`allowSubstitutes`](@docroot@/language/advanced-attributes.md) attribute in derivations and always attempt to use [available substituters](#conf-substituters). )"}; Setting buildersUseSubstitutes{ From 24b781773f3d24b62b1c36154c36fc98417cbcdb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 2 Dec 2023 03:02:58 +0100 Subject: [PATCH 720/909] fix random docs errors remove link to the contributing guide from user documentation. it doesn't help here, and the target at first glance shows redundant information. --- src/libexpr/primops.cc | 2 +- src/libstore/globals.hh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 63c90795a..c2499bdae 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4400,7 +4400,7 @@ void EvalState::createBaseEnv() addConstant("__currentSystem", v, { .type = nString, .doc = R"( - The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-pure-eval). + The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-system). It can be used to set the `system` attribute for [`builtins.derivation`](@docroot@/language/derivations.md) such that the resulting derivation can be built on the same system that evaluates the Nix expression: diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 38b0d516c..8da9e371f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -201,7 +201,7 @@ public: Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in [`extra-platforms`](#conf-extra-platforms). The default value is set when Nix itself is compiled for the system it will run on. - The following system types are widely used, as [Nix is actively supported on these platforms](@docroot@/contributing/hacking.md#platforms): + The following system types are widely used, as Nix is actively supported on these platforms: - `x86_64-linux` - `x86_64-darwin` @@ -761,7 +761,7 @@ public: "substituters", R"( A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace. - A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them. + A substituter is an additional [store](@docroot@/glossary.md#gloss-store) from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them. Substituters are tried based on their priority value, which each substituter can set independently. Lower value means higher priority. From 368fdb482da039fd40a1a51bbf851c54f65eb4c5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 2 Dec 2023 03:06:47 +0100 Subject: [PATCH 721/909] reword description of the `builders-use-substitutes` setting --- src/libstore/globals.hh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 38b0d516c..dcdcd31d5 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -276,13 +276,10 @@ public: Setting buildersUseSubstitutes{ this, false, "builders-use-substitutes", R"( - If set to `true`, Nix will instruct remote build machines to use - their own binary substitutes if available. In practical terms, this - means that remote hosts will fetch as many build dependencies as - possible from their own substitutes (e.g, from `cache.nixos.org`), - instead of waiting for this host to upload them all. This can - drastically reduce build times if the network connection between - this computer and the remote build host is slow. + If set to `true`, Nix will instruct [remote build machines](#conf-builders) to use their own [`substituters`](#conf-substituters) if available. + + It means that remote build hosts will fetch as many dependencies as possible from their own substituters (e.g, from `cache.nixos.org`) instead of waiting for the local machine to upload them all. + This can drastically reduce build times if the network connection between the local machine and the remote build host is slow. )"}; Setting reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space", From 2c3749a335d4462412ac73eb77a81d949e1e8ba6 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Sat, 2 Dec 2023 16:08:06 +0000 Subject: [PATCH 722/909] Fix cross builds --- package.nix | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/package.nix b/package.nix index bed77ba3b..9f30eef2f 100644 --- a/package.nix +++ b/package.nix @@ -123,6 +123,10 @@ stdenv.mkDerivation (finalAttrs: { openssl sqlite xz + + # These could be checkInputs but the configure phase fails w/o them + gtest + rapidcheck ] ++ lib.optional stdenv.isLinux libseccomp ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid @@ -137,14 +141,13 @@ stdenv.mkDerivation (finalAttrs: { doCheck = true; checkInputs = [ - gtest - rapidcheck + # see buildInputs. The configure script always wants its test libs ]; nativeCheckInputs = [ git - mercurial # FIXME: remove? only needed for tests - openssh # only needed for tests (ssh-keygen) + mercurial + openssh ]; propagatedBuildInputs = [ From ca598328085fe7a379bff8777031101fba80921b Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Sat, 2 Dec 2023 16:36:59 +0000 Subject: [PATCH 723/909] Fix coverage.nix --- coverage.nix | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/coverage.nix b/coverage.nix index 2390ef52d..f952d8b09 100644 --- a/coverage.nix +++ b/coverage.nix @@ -14,19 +14,21 @@ releaseTools.coverageAnalysis { inherit (nix) src - configureFlags - nativeBuildInputs buildInputs - #checkInputs + nativeBuildInputs + propagatedBuildInputs + configureFlags + makeFlags + installFlags + doInstallCheck + installCheckFlags + installCheckTarget ; enableParallelBuilding = true; dontInstall = false; - doInstallCheck = true; - installCheckTarget = "installcheck"; # work around buggy detection in stdenv - lcovFilter = [ "*/boost/*" "*-tab.*" ]; hardeningDisable = ["fortify"]; From 118fa9689ab0e6d12b360708177f9a1b56f3d466 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Sat, 2 Dec 2023 16:42:01 +0000 Subject: [PATCH 724/909] Create internal-api-docs.nix --- flake.nix | 17 +---------------- internal-api-docs.nix | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 16 deletions(-) create mode 100644 internal-api-docs.nix diff --git a/flake.nix b/flake.nix index c0841a76d..b1c3a777e 100644 --- a/flake.nix +++ b/flake.nix @@ -482,22 +482,7 @@ coverage = nixpkgsFor.x86_64-linux.native.callPackage ./coverage.nix {}; # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = nixpkgsFor.x86_64-linux.native.nix.overrideAttrs (old: { - pname = "nix-internal-api-docs"; - - configureFlags = old.configureFlags ++ [ "--enable-internal-api-docs" ]; - nativeBuildInputs = old.nativeBuildInputs ++ [ nixpkgsFor.x86_64-linux.native.doxygen ]; - - dontBuild = true; - doCheck = false; - - installTargets = [ "internal-api-html" ]; - - postInstall = '' - mkdir -p $out/nix-support - echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products - ''; - }); + internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./internal-api-docs.nix {}; # System tests. tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { diff --git a/internal-api-docs.nix b/internal-api-docs.nix new file mode 100644 index 000000000..ddd3fa891 --- /dev/null +++ b/internal-api-docs.nix @@ -0,0 +1,24 @@ +{ nix +, doxygen +}: + +nix.overrideAttrs (old: { + pname = "nix-internal-api-docs"; + + configureFlags = old.configureFlags ++ [ + "--enable-internal-api-docs" + ]; + nativeBuildInputs = old.nativeBuildInputs ++ [ + doxygen + ]; + + dontBuild = true; + doCheck = false; + + installTargets = [ "internal-api-html" ]; + + postInstall = '' + mkdir -p $out/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products + ''; +}) From 19d41fb20a45d2bf66f78813514bf5c5fd420a8b Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Sat, 2 Dec 2023 17:25:47 +0000 Subject: [PATCH 725/909] Fix stuff --- flake.nix | 230 +++--------------------------------------- package.nix | 60 ++++++----- test-nix-versions.nix | 50 +++++++++ 3 files changed, 96 insertions(+), 244 deletions(-) create mode 100644 test-nix-versions.nix diff --git a/flake.nix b/flake.nix index b1c3a777e..fbce13604 100644 --- a/flake.nix +++ b/flake.nix @@ -12,6 +12,14 @@ let inherit (nixpkgs) lib; + # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981 + # Not an "idiomatic" flake input because: + # - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730 + # - Subflake would download redundant and huge parent flake + # - No git tree hash support: https://github.com/NixOS/nix/issues/6044 + inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; })) + fileset; + officialRelease = false; # Set to true to build the release notes for the next release. @@ -56,57 +64,6 @@ }) stdenvs); - # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981 - # Not an "idiomatic" flake input because: - # - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730 - # - Subflake would download redundant and huge parent flake - # - No git tree hash support: https://github.com/NixOS/nix/issues/6044 - inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; })) - fileset; - - baseFiles = - # .gitignore has already been processed, so any changes in it are irrelevant - # at this point. It is not represented verbatim for test purposes because - # that would interfere with repo semantics. - fileset.fileFilter (f: f.name != ".gitignore") ./.; - - configureFiles = fileset.unions [ - ./.version - ./configure.ac - ./m4 - # TODO: do we really need README.md? It doesn't seem used in the build. - ./README.md - ]; - - topLevelBuildFiles = fileset.unions [ - ./local.mk - ./Makefile - ./Makefile.config.in - ./mk - ]; - - functionalTestFiles = fileset.unions [ - ./tests/functional - (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) - ]; - - nixSrc = fileset.toSource { - root = ./.; - fileset = fileset.intersect baseFiles (fileset.unions [ - configureFiles - topLevelBuildFiles - ./boehmgc-coroutine-sp-fallback.diff - ./doc - ./misc - ./precompiled-headers.h - ./src - ./unit-test-data - ./COPYING - ./scripts/local.mk - functionalTestFiles - ]); - }; - # Memoize nixpkgs for different platforms for efficiency. nixpkgsFor = forAllSystems (system: let @@ -131,130 +88,6 @@ cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); - commonDeps = - { pkgs - , isStatic ? pkgs.stdenv.hostPlatform.isStatic - }: - with pkgs; rec { - # Use "busybox-sandbox-shell" if present, - # if not (legacy) fallback and hope it's sufficient. - sh = pkgs.busybox-sandbox-shell or (busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y - - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }); - - configureFlags = - lib.optionals stdenv.isLinux [ - "--with-boost=${boost-nix}/lib" - "--with-sandbox-shell=${sh}/bin/busybox" - ] - ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [ - "LDFLAGS=-fuse-ld=gold" - ]; - - testConfigureFlags = [ - "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" - ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ - "--enable-install-unit-tests" - "--with-check-bin-dir=${builtins.placeholder "check"}/bin" - "--with-check-lib-dir=${builtins.placeholder "check"}/lib" - ]; - - internalApiDocsConfigureFlags = [ - "--enable-internal-api-docs" - ]; - - inherit (pkgs.buildPackages) changelog-d; - - nativeBuildDeps = - [ - buildPackages.bison - buildPackages.flex - (lib.getBin buildPackages.lowdown-nix) - buildPackages.mdbook - buildPackages.mdbook-linkcheck - buildPackages.autoconf-archive - buildPackages.autoreconfHook - buildPackages.pkg-config - - # Tests - buildPackages.git - buildPackages.mercurial # FIXME: remove? only needed for tests - buildPackages.jq # Also for custom mdBook preprocessor. - buildPackages.openssh # only needed for tests (ssh-keygen) - ] - ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)] - # Official releases don't have rl-next, so we don't need to compile a changelog - ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d - ; - - buildDeps = - [ curl - bzip2 xz brotli editline - openssl sqlite - libarchive - (pkgs.libgit2.overrideAttrs (attrs: { - src = libgit2; - version = libgit2.lastModifiedDate; - cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"]; - })) - boost-nix - lowdown-nix - libsodium - ] - ++ lib.optionals stdenv.isLinux [libseccomp] - ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; - - checkDeps = [ - gtest - rapidcheck - ]; - - internalApiDocsDeps = [ - buildPackages.doxygen - ]; - - awsDeps = lib.optional (stdenv.isLinux || stdenv.isDarwin) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }); - - propagatedDeps = - [ ((boehmgc.override { - enableLargeConfig = true; - }).overrideAttrs(o: { - patches = (o.patches or []) ++ [ - ./boehmgc-coroutine-sp-fallback.diff - - # https://github.com/ivmai/bdwgc/pull/586 - ./boehmgc-traceable_allocator-public.diff - ]; - }) - ) - nlohmann_json - ]; - }; - installScriptFor = systems: with nixpkgsFor.x86_64-linux.native; runCommand "installer-script" @@ -289,50 +122,11 @@ echo "file installer $out/install" >> $out/nix-support/hydra-build-products ''; - testNixVersions = pkgs: client: daemon: with commonDeps { inherit pkgs; }; with pkgs.lib; pkgs.stdenv.mkDerivation { - NIX_DAEMON_PACKAGE = daemon; - NIX_CLIENT_PACKAGE = client; - name = - "nix-tests" - + optionalString - (versionAtLeast daemon.version "2.4pre20211005" && - versionAtLeast client.version "2.4pre20211005") - "-${client.version}-against-${daemon.version}"; - inherit version; - - src = fileset.toSource { - root = ./.; - fileset = fileset.intersect baseFiles (fileset.unions [ - configureFiles - topLevelBuildFiles - functionalTestFiles - ]); + testNixVersions = pkgs: client: daemon: + pkgs.callPackage ./test-nix-versions.nix { + inherit client daemon fileset; }; - VERSION_SUFFIX = versionSuffix; - - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ awsDeps ++ checkDeps; - propagatedBuildInputs = propagatedDeps; - - enableParallelBuilding = true; - - configureFlags = - testConfigureFlags # otherwise configure fails - ++ [ "--disable-build" ]; - dontBuild = true; - doInstallCheck = true; - - installPhase = '' - mkdir -p $out - ''; - - installCheckPhase = '' - mkdir -p src/nix-channel - make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES - ''; - }; - binaryTarball = nix: pkgs: pkgs.callPackage ./binary-tarball.nix { inherit nix; }; @@ -491,7 +285,7 @@ # on a particular version of Nixpkgs. evalNixpkgs = let - inherit (nixpkgsFor.x86_64-linux.native) runCommand nix nixpkgs-regression; + inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; in runCommand "eval-nixos" { buildInputs = [ nix ]; } '' diff --git a/package.nix b/package.nix index 9f30eef2f..e4c66958b 100644 --- a/package.nix +++ b/package.nix @@ -43,6 +43,30 @@ let version = lib.fileContents ./.version + versionSuffix; canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + + filesets = { + baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.; + + configureFiles = fileset.unions [ + ./.version + ./configure.ac + ./m4 + # TODO: do we really need README.md? It doesn't seem used in the build. + ./README.md + ]; + + topLevelBuildFiles = fileset.unions [ + ./local.mk + ./Makefile + ./Makefile.config.in + ./mk + ]; + + functionalTestFiles = fileset.unions [ + ./tests/functional + (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) + ]; + }; in stdenv.mkDerivation (finalAttrs: { @@ -51,33 +75,13 @@ stdenv.mkDerivation (finalAttrs: { src = let - baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.; - configureFiles = fileset.unions [ - ./.version - ./configure.ac - ./m4 - # TODO: do we really need README.md? It doesn't seem used in the build. - ./README.md - ]; - - topLevelBuildFiles = fileset.unions [ - ./local.mk - ./Makefile - ./Makefile.config.in - ./mk - ]; - - functionalTestFiles = fileset.unions [ - ./tests/functional - (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) - ]; in fileset.toSource { root = ./.; - fileset = fileset.intersect baseFiles (fileset.unions [ - configureFiles - topLevelBuildFiles + fileset = fileset.intersect filesets.baseFiles (fileset.unions [ + filesets.configureFiles + filesets.topLevelBuildFiles ./boehmgc-coroutine-sp-fallback.diff ./doc ./misc @@ -86,7 +90,7 @@ stdenv.mkDerivation (finalAttrs: { ./unit-test-data ./COPYING ./scripts/local.mk - functionalTestFiles + filesets.functionalTestFiles ]); }; @@ -231,8 +235,12 @@ stdenv.mkDerivation (finalAttrs: { hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - passthru.perl-bindings = callPackage ./perl { - inherit fileset stdenv; + passthru ={ + inherit filesets; + + perl-bindings = callPackage ./perl { + inherit fileset stdenv; + }; }; meta.platforms = lib.platforms.unix; diff --git a/test-nix-versions.nix b/test-nix-versions.nix new file mode 100644 index 000000000..15f6cd8d0 --- /dev/null +++ b/test-nix-versions.nix @@ -0,0 +1,50 @@ +{ lib +, fileset +, stdenv +, client +, daemon +}: + +stdenv.mkDerivation { + NIX_DAEMON_PACKAGE = daemon; + NIX_CLIENT_PACKAGE = client; + name = + "nix-tests" + + lib.optionalString + (lib.versionAtLeast daemon.version "2.4pre20211005" && + lib.versionAtLeast client.version "2.4pre20211005") + "-${client.version}-against-${daemon.version}"; + + inherit (client) + version + VERSION_SUFFIX + nativeBuildInputs + buildInputs + propagatedBuildInputs + ; + + src = fileset.toSource { + root = ./.; + fileset = with client.passthru.filesets; + fileset.intersect baseFiles (fileset.unions [ + configureFiles + topLevelBuildFiles + functionalTestFiles + ]); + }; + + configureFlags = client.configureFlags # otherwise configure fails + ++ [ "--disable-build" ]; + + dontBuild = true; + doInstallCheck = true; + + installPhase = '' + mkdir -p $out + ''; + + installCheckPhase = '' + mkdir -p src/nix-channel + make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES + ''; +} From 0ca49b0c8663ae82931780ae3f1f45115b966285 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 12:47:07 -0500 Subject: [PATCH 726/909] Add installing unit test flags --- package.nix | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/package.nix b/package.nix index e4c66958b..c1a3b9455 100644 --- a/package.nix +++ b/package.nix @@ -38,6 +38,17 @@ , sqlite , util-linux , xz + +# Configuration Options +# +# This probably seems like too many degrees of freedom, but it +# faithfully reflects how the underlying configure + make build system +# work. The top-level flake.nix will choose useful combinations. + +# Whether to install unit tests. This is useful when cross compiling +# since we cannot run them natively during the build, but can do so +# later. +, installUnitTests ? stdenv.hostPlatform != stdenv.buildPlatform }: let @@ -69,7 +80,13 @@ let }; in -stdenv.mkDerivation (finalAttrs: { +stdenv.mkDerivation (finalAttrs: let + + # Either running the unit tests during the build, or installing them + # to be run later, requiresthe unit tests to be built. + buildUnitTests = finalAttrs.doCheck || installUnitTests; + +in { pname = "nix"; inherit version; @@ -97,7 +114,7 @@ stdenv.mkDerivation (finalAttrs: { VERSION_SUFFIX = versionSuffix; outputs = [ "out" "dev" "doc" ] - ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "check"; + ++ lib.optional installUnitTests "check"; nativeBuildInputs = [ bison @@ -142,7 +159,7 @@ stdenv.mkDerivation (finalAttrs: { }) ; - doCheck = true; + doCheck = stdenv.hostPlatform != stdenv.buildPlatform; checkInputs = [ # see buildInputs. The configure script always wants its test libs @@ -190,14 +207,12 @@ stdenv.mkDerivation (finalAttrs: { "LDFLAGS=-fuse-ld=gold" ++ [ "--sysconfdir=/etc" ] ++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" - ++ [ (lib.enableFeature finalAttrs.doCheck "tests") ] - ++ lib.optionals finalAttrs.doCheck ( - [ "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" ] - ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ - "--enable-install-unit-tests" - "--with-check-bin-dir=${builtins.placeholder "check"}/bin" - "--with-check-lib-dir=${builtins.placeholder "check"}/lib" - ]) + ++ lib.optional buildUnitTests "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" + ++ lib.optionals installUnitTests [ + "--enable-install-unit-tests" + "--with-check-bin-dir=${builtins.placeholder "check"}/bin" + "--with-check-lib-dir=${builtins.placeholder "check"}/lib" + ] ++ lib.optional (!canRunInstalled) "--disable-doc-gen"; enableParallelBuilding = true; From ce598bae144c49c61b33cdf55679ef597ede9485 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 14:10:09 -0500 Subject: [PATCH 727/909] WIP --- coverage.nix | 8 --- flake.nix | 7 ++- internal-api-docs.nix | 24 ------- package.nix | 143 ++++++++++++++++++++++++++++++++---------- test-nix-versions.nix | 35 ----------- 5 files changed, 117 insertions(+), 100 deletions(-) delete mode 100644 internal-api-docs.nix diff --git a/coverage.nix b/coverage.nix index f952d8b09..2c5e4a06d 100644 --- a/coverage.nix +++ b/coverage.nix @@ -26,12 +26,4 @@ releaseTools.coverageAnalysis { ; enableParallelBuilding = true; - - dontInstall = false; - - lcovFilter = [ "*/boost/*" "*-tab.*" ]; - - hardeningDisable = ["fortify"]; - - NIX_CFLAGS_COMPILE = "-DCOVERAGE=1"; } diff --git a/flake.nix b/flake.nix index fbce13604..44ce2d306 100644 --- a/flake.nix +++ b/flake.nix @@ -276,7 +276,12 @@ coverage = nixpkgsFor.x86_64-linux.native.callPackage ./coverage.nix {}; # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./internal-api-docs.nix {}; + internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix { + doBuild = false; + doCheck = false; + doInstallCheck = false; + enableInternalAPIDocs = true; + }; # System tests. tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { diff --git a/internal-api-docs.nix b/internal-api-docs.nix deleted file mode 100644 index ddd3fa891..000000000 --- a/internal-api-docs.nix +++ /dev/null @@ -1,24 +0,0 @@ -{ nix -, doxygen -}: - -nix.overrideAttrs (old: { - pname = "nix-internal-api-docs"; - - configureFlags = old.configureFlags ++ [ - "--enable-internal-api-docs" - ]; - nativeBuildInputs = old.nativeBuildInputs ++ [ - doxygen - ]; - - dontBuild = true; - doCheck = false; - - installTargets = [ "internal-api-html" ]; - - postInstall = '' - mkdir -p $out/nix-support - echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products - ''; -}) diff --git a/package.nix b/package.nix index c1a3b9455..39fee8472 100644 --- a/package.nix +++ b/package.nix @@ -1,6 +1,7 @@ { lib , callPackage , stdenv +, releaseTools , versionSuffix ? "" , officialRelease ? false , buildUnreleasedNotes ? false @@ -21,6 +22,7 @@ , git , gtest , jq +, doxygen , libarchive , libcpuid , libgit2 @@ -45,16 +47,35 @@ # faithfully reflects how the underlying configure + make build system # work. The top-level flake.nix will choose useful combinations. +, pname ? "nix" + +, doBuild ? true +, doCheck ? stdenv.buildPlatform.canExecute stdenv.hostPlatform +, doInstallCheck ? stdenv.buildPlatform.canExecute stdenv.hostPlatform + +, withCoverageChecks ? false + +# Whether to build the internal API docs, can be done separately from +# everything else. +, enableInternalAPIDocs ? false + # Whether to install unit tests. This is useful when cross compiling # since we cannot run them natively during the build, but can do so # later. , installUnitTests ? stdenv.hostPlatform != stdenv.buildPlatform + +, test-daemon ? null +, test-client ? null }: let version = lib.fileContents ./.version + versionSuffix; canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + attrs = { + inherit doBuild doCheck doInstallCheck; + }; + filesets = { baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.; @@ -78,17 +99,30 @@ let (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) ]; }; + + mkDerivation = + if withCoverageChecks + then releaseTools.coverageAnalysis + else stdenv.mkDerivation; in -stdenv.mkDerivation (finalAttrs: let +mkDerivation (finalAttrs: let + + inherit (finalAttrs) + doCheck + doInstallCheck + ; + + doBuild = !finalAttrs.dontBuild; # Either running the unit tests during the build, or installing them # to be run later, requiresthe unit tests to be built. - buildUnitTests = finalAttrs.doCheck || installUnitTests; + buildUnitTests = doCheck || installUnitTests; + + anySortOfTesting = buildUnitTests || doInstallCheck; in { - pname = "nix"; - inherit version; + inherit pname version; src = let @@ -96,9 +130,10 @@ in { in fileset.toSource { root = ./.; - fileset = fileset.intersect filesets.baseFiles (fileset.unions [ + fileset = fileset.intersect filesets.baseFiles (fileset.unions ([ filesets.configureFiles filesets.topLevelBuildFiles + ] ++ lib.optionals doBuild [ ./boehmgc-coroutine-sp-fallback.diff ./doc ./misc @@ -107,8 +142,9 @@ in { ./unit-test-data ./COPYING ./scripts/local.mk + ] ++ lib.optionals anySortOfTesting [ filesets.functionalTestFiles - ]); + ])); }; VERSION_SUFFIX = versionSuffix; @@ -159,7 +195,13 @@ in { }) ; - doCheck = stdenv.hostPlatform != stdenv.buildPlatform; + propagatedBuildInputs = [ + boehmgc + nlohmann_json + ]; + + dontBuild = !attrs.doBuild; + doCheck = attrs.doCheck; checkInputs = [ # see buildInputs. The configure script always wants its test libs @@ -169,11 +211,8 @@ in { git mercurial openssh - ]; - - propagatedBuildInputs = [ - boehmgc - nlohmann_json + ] ++ lib.optionals enableInternalAPIDocs [ + doxygen ]; disallowedReferences = [ boost ]; @@ -198,30 +237,41 @@ in { ''} ''; - configureFlags = - lib.optionals stdenv.isLinux [ - "--with-boost=${boost}/lib" - "--with-sandbox-shell=${sh}/bin/busybox" - ] - ++ lib.optional (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) - "LDFLAGS=-fuse-ld=gold" - ++ [ "--sysconfdir=/etc" ] + configureFlags = [ + "--sysconfdir=/etc" + (lib.enableFeature doBuild "build") + (lib.enableFeature anySortOfTesting "test") + (lib.enableFeature enableInternalAPIDocs "internal-api-docs") + (lib.enableFeature canRunInstalled "doc-gen") + (lib.enableFeature installUnitTests "install-unit-tests") + ] ++ lib.optionals installUnitTests [ + "--with-check-bin-dir=${builtins.placeholder "check"}/bin" + "--with-check-lib-dir=${builtins.placeholder "check"}/lib" + ] ++ lib.optionals stdenv.isLinux [ + "--with-boost=${boost}/lib" + "--with-sandbox-shell=${sh}/bin/busybox" + ] ++ lib.optional (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) + "LDFLAGS=-fuse-ld=gold" ++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" - ++ lib.optional buildUnitTests "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" - ++ lib.optionals installUnitTests [ - "--enable-install-unit-tests" - "--with-check-bin-dir=${builtins.placeholder "check"}/bin" - "--with-check-lib-dir=${builtins.placeholder "check"}/lib" - ] - ++ lib.optional (!canRunInstalled) "--disable-doc-gen"; + ++ lib.optional buildUnitTests "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include"; enableParallelBuilding = true; makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; + installTargets = lib.optional doBuild "install" + ++ lib.optional enableInternalAPIDocs "internal-api-html"; + installFlags = "sysconfdir=$(out)/etc"; - postInstall = '' + # In this case we are probably just running tests, and so there isn't + # anything to install, we just make an empty directory to signify tests + # succeeded. + installPhase = if finalAttrs.installTargets != [] then null else '' + mkdir -p $out + ''; + + postInstall = lib.optionalString doBuild '' mkdir -p $doc/nix-support echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products ${lib.optionalString stdenv.hostPlatform.isStatic '' @@ -238,19 +288,29 @@ in { $out/lib/libboost_regex.dylib \ $out/lib/libnixexpr.dylib ''} + '' + lib.optionalString enableInternalAPIDocs '' + mkdir -p $out/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products ''; - doInstallCheck = finalAttrs.doCheck; + doInstallCheck = attrs.doInstallCheck; + installCheckFlags = "sysconfdir=$(out)/etc"; installCheckTarget = "installcheck"; # work around buggy detection in stdenv + # Needed for tests if we are not doing a build, but testing existing + # built Nix. + preInstallCheck = lib.optionalString (! doBuild) '' + mkdir -p src/nix-channel + ''; + separateDebugInfo = !stdenv.hostPlatform.isStatic; strictDeps = true; hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - passthru ={ + passthru = { inherit filesets; perl-bindings = callPackage ./perl { @@ -258,6 +318,25 @@ in { }; }; - meta.platforms = lib.platforms.unix; - meta.mainProgram = "nix"; + meta = { + platforms = lib.platforms.unix; + mainProgram = "nix"; + broken = !(lib.all (a: a) [ + (installUnitTests -> doBuild) + (doCheck -> doBuild) + ]); + }; + +} // lib.optionalAttrs withCoverageChecks { + lcovFilter = [ "*/boost/*" "*-tab.*" ]; + + hardeningDisable = ["fortify"]; + + NIX_CFLAGS_COMPILE = "-DCOVERAGE=1"; + + dontInstall = false; +} // lib.optionalAttrs (test-daemon != null) { + NIX_DAEMON_PACKAGE = test-daemon; +} // lib.optionalAttrs (test-client != null) { + NIX_CLIENT_PACKAGE = test-client; }) diff --git a/test-nix-versions.nix b/test-nix-versions.nix index 15f6cd8d0..bda4621a1 100644 --- a/test-nix-versions.nix +++ b/test-nix-versions.nix @@ -6,45 +6,10 @@ }: stdenv.mkDerivation { - NIX_DAEMON_PACKAGE = daemon; - NIX_CLIENT_PACKAGE = client; name = "nix-tests" + lib.optionalString (lib.versionAtLeast daemon.version "2.4pre20211005" && lib.versionAtLeast client.version "2.4pre20211005") "-${client.version}-against-${daemon.version}"; - - inherit (client) - version - VERSION_SUFFIX - nativeBuildInputs - buildInputs - propagatedBuildInputs - ; - - src = fileset.toSource { - root = ./.; - fileset = with client.passthru.filesets; - fileset.intersect baseFiles (fileset.unions [ - configureFiles - topLevelBuildFiles - functionalTestFiles - ]); - }; - - configureFlags = client.configureFlags # otherwise configure fails - ++ [ "--disable-build" ]; - - dontBuild = true; - doInstallCheck = true; - - installPhase = '' - mkdir -p $out - ''; - - installCheckPhase = '' - mkdir -p src/nix-channel - make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES - ''; } From 3d47e024837a4340b1a0b6b6b8114e9e9e0c38a4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 16:48:50 -0500 Subject: [PATCH 728/909] WIP --- flake.nix | 9 +++--- package.nix | 90 +++++++++++++++++++++++++++++++---------------------- 2 files changed, 56 insertions(+), 43 deletions(-) diff --git a/flake.nix b/flake.nix index 44ce2d306..85ea1d052 100644 --- a/flake.nix +++ b/flake.nix @@ -153,7 +153,7 @@ then "" else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - sh = final.busybox-sandbox-shell or (final.busybox.override { + default-busybox-sandbox-shell = final.busybox.override { useMusl = true; enableStatic = true; enableMinimal = true; @@ -175,7 +175,7 @@ CONFIG_ASH_PRINTF y CONFIG_ASH_TEST y ''; - }); + }; boehmgc = (final.boehmgc.override { enableLargeConfig = true; @@ -192,10 +192,10 @@ inherit boehmgc fileset - sh stdenv versionSuffix ; + busybox-sandbox-shell = final.busybox-sandbox-shell or default-busybox-sandbox-shell; boost = final.boost.override { enableIcu = false; }; libgit2 = final.libgit2.overrideAttrs (attrs: { src = libgit2; @@ -277,9 +277,8 @@ # API docs for Nix's unstable internal C++ interfaces. internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix { + inherit fileset; doBuild = false; - doCheck = false; - doInstallCheck = false; enableInternalAPIDocs = true; }; diff --git a/package.nix b/package.nix index 39fee8472..15fe52b07 100644 --- a/package.nix +++ b/package.nix @@ -36,11 +36,12 @@ , openssl , pkg-config , rapidcheck -, sh , sqlite , util-linux , xz +, busybox-sandbox-shell ? null + # Configuration Options # # This probably seems like too many degrees of freedom, but it @@ -50,11 +51,13 @@ , pname ? "nix" , doBuild ? true -, doCheck ? stdenv.buildPlatform.canExecute stdenv.hostPlatform -, doInstallCheck ? stdenv.buildPlatform.canExecute stdenv.hostPlatform +, doCheck ? __forDefaults.canRunInstalled +, doInstallCheck ? __forDefaults.canRunInstalled , withCoverageChecks ? false +# Whether to build the regular manual +, enableManual ? __forDefaults.canRunInstalled # Whether to build the internal API docs, can be done separately from # everything else. , enableInternalAPIDocs ? false @@ -62,16 +65,26 @@ # Whether to install unit tests. This is useful when cross compiling # since we cannot run them natively during the build, but can do so # later. -, installUnitTests ? stdenv.hostPlatform != stdenv.buildPlatform +, installUnitTests ? __forDefaults.canRunInstalled +# For running the functional tests against a pre-built Nix. Probably +# want to use in conjunction with `doBuild = false;`. , test-daemon ? null , test-client ? null -}: + +# Not a real argument, just the only way to approximate let-binding some +# stuff for argument defaults. +, __forDefaults ? { + canRunInstalled = doBuild && stdenv.buildPlatform.canExecute stdenv.hostPlatform; + } +} @ attrs0: let version = lib.fileContents ./.version + versionSuffix; - canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + # selected attributes with defaults, will be used to define some + # things which should instead be gotten via `finalAttrs` in order to + # work with overriding. attrs = { inherit doBuild doCheck doInstallCheck; }; @@ -149,7 +162,11 @@ in { VERSION_SUFFIX = versionSuffix; - outputs = [ "out" "dev" "doc" ] + outputs = [ "out" ] + ++ lib.optional doBuild "dev" + # If we are doing just build or just docs, the one thing will use + # "out". We only need additional outputs if we are doing both. + ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs)) "doc" ++ lib.optional installUnitTests "check"; nativeBuildInputs = [ @@ -164,10 +181,11 @@ in { pkg-config ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux - # Official releases don't have rl-next, so we don't need to compile a changelog + # Official releases don't have rl-next, so we don't need to compile a + # changelog ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d; - buildInputs = [ + buildInputs = lib.optionals doBuild [ boost brotli bzip2 @@ -180,19 +198,14 @@ in { openssl sqlite xz - - # These could be checkInputs but the configure phase fails w/o them - gtest - rapidcheck - ] - ++ lib.optional stdenv.isLinux libseccomp - ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid - # There have been issues building these dependencies - ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }) + ] ++ lib.optional stdenv.isLinux libseccomp + ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid + # There have been issues building these dependencies + ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) + (aws-sdk-cpp.override { + apis = ["s3" "transfer"]; + customMemoryManagement = false; + }) ; propagatedBuildInputs = [ @@ -204,7 +217,8 @@ in { doCheck = attrs.doCheck; checkInputs = [ - # see buildInputs. The configure script always wants its test libs + gtest + rapidcheck ]; nativeCheckInputs = [ @@ -242,17 +256,17 @@ in { (lib.enableFeature doBuild "build") (lib.enableFeature anySortOfTesting "test") (lib.enableFeature enableInternalAPIDocs "internal-api-docs") - (lib.enableFeature canRunInstalled "doc-gen") + (lib.enableFeature enableManual "doc-gen") (lib.enableFeature installUnitTests "install-unit-tests") ] ++ lib.optionals installUnitTests [ "--with-check-bin-dir=${builtins.placeholder "check"}/bin" "--with-check-lib-dir=${builtins.placeholder "check"}/lib" - ] ++ lib.optionals stdenv.isLinux [ + ] ++ lib.optionals (doBuild && stdenv.isLinux) [ "--with-boost=${boost}/lib" - "--with-sandbox-shell=${sh}/bin/busybox" - ] ++ lib.optional (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) + "--with-sandbox-shell=${busybox-sandbox-shell}/bin/busybox" + ] ++ lib.optional (doBuild && stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) "LDFLAGS=-fuse-ld=gold" - ++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" + ++ lib.optional (doBuild && stdenv.hostPlatform.isStatic) "--enable-embedded-sandbox-shell" ++ lib.optional buildUnitTests "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include"; enableParallelBuilding = true; @@ -271,14 +285,14 @@ in { mkdir -p $out ''; - postInstall = lib.optionalString doBuild '' - mkdir -p $doc/nix-support - echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products - ${lib.optionalString stdenv.hostPlatform.isStatic '' + postInstall = lib.optionalString doBuild ( + '' + mkdir -p $doc/nix-support + echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products + '' + lib.optionalString stdenv.hostPlatform.isStatic '' mkdir -p $out/nix-support echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - ''} - ${lib.optionalString stdenv.isDarwin '' + '' + lib.optionalString stdenv.isDarwin '' install_name_tool \ -change ${boost}/lib/libboost_context.dylib \ $out/lib/libboost_context.dylib \ @@ -287,10 +301,10 @@ in { -change ${boost}/lib/libboost_regex.dylib \ $out/lib/libboost_regex.dylib \ $out/lib/libnixexpr.dylib - ''} - '' + lib.optionalString enableInternalAPIDocs '' - mkdir -p $out/nix-support - echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products + '' + ) + lib.optionalString enableInternalAPIDocs '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products ''; doInstallCheck = attrs.doInstallCheck; From c71d987553530dcf02bcd7bf4c682634d7e5b6be Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 17:12:38 -0500 Subject: [PATCH 729/909] Fix incorrect flag name --- package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.nix b/package.nix index d2498ade2..f688cc819 100644 --- a/package.nix +++ b/package.nix @@ -254,7 +254,7 @@ in { configureFlags = [ "--sysconfdir=/etc" (lib.enableFeature doBuild "build") - (lib.enableFeature anySortOfTesting "test") + (lib.enableFeature anySortOfTesting "tests") (lib.enableFeature enableInternalAPIDocs "internal-api-docs") (lib.enableFeature enableManual "doc-gen") (lib.enableFeature installUnitTests "install-unit-tests") From 7b51086d736f8cf983744510ff40e5afbc313079 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:12:05 -0500 Subject: [PATCH 730/909] More fixes --- coverage.nix | 29 ---------------- flake.nix | 97 ++++++++++++++++++++++++++++++---------------------- package.nix | 61 +++++++++++++++++++++------------ 3 files changed, 95 insertions(+), 92 deletions(-) delete mode 100644 coverage.nix diff --git a/coverage.nix b/coverage.nix deleted file mode 100644 index 2c5e4a06d..000000000 --- a/coverage.nix +++ /dev/null @@ -1,29 +0,0 @@ -{ lib -, releaseTools -, nix -, stdenv -}: - -let - inherit (nix) version; - -in - -releaseTools.coverageAnalysis { - name = "nix-coverage-${version}"; - - inherit (nix) - src - buildInputs - nativeBuildInputs - propagatedBuildInputs - configureFlags - makeFlags - installFlags - doInstallCheck - installCheckFlags - installCheckTarget - ; - - enableParallelBuilding = true; -} diff --git a/flake.nix b/flake.nix index aafcfd71b..fab8c45be 100644 --- a/flake.nix +++ b/flake.nix @@ -123,8 +123,20 @@ ''; testNixVersions = pkgs: client: daemon: - pkgs.callPackage ./test-nix-versions.nix { - inherit client daemon fileset; + pkgs.callPackage ./package.nix { + pname = + "nix-tests" + + lib.optionalString + (lib.versionAtLeast daemon.version "2.4pre20211005" && + lib.versionAtLeast client.version "2.4pre20211005") + "-${client.version}-against-${daemon.version}"; + + inherit fileset; + + test-client = client; + test-daemon = daemon; + + doBuild = false; }; binaryTarball = nix: pkgs: pkgs.callPackage ./binary-tarball.nix { @@ -134,10 +146,6 @@ overlayFor = getStdenv: final: prev: let stdenv = getStdenv final; - - lowdown-nix = final.callPackage ./lowdown.nix { - inherit lowdown-src stdenv; - }; in { nixStable = prev.nix; @@ -145,6 +153,41 @@ # Forward from the previous stage as we don’t want it to pick the lowdown override inherit (prev) nixUnstable; + default-busybox-sandbox-shell = final.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y + + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }; + + lowdown-nix = final.callPackage ./lowdown.nix { + inherit lowdown-src stdenv; + }; + + libgit2-nix = final.libgit2.overrideAttrs (attrs: { + src = libgit2; + version = libgit2.lastModifiedDate; + cmakeFlags = attrs.cmakeFlags or [] + ++ [ "-DUSE_SSH=exec" ]; + }); + nix = let officialRelease = false; @@ -153,30 +196,6 @@ then "" else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - default-busybox-sandbox-shell = final.busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y - - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }; - boehmgc = (final.boehmgc.override { enableLargeConfig = true; }).overrideAttrs(o: { @@ -195,18 +214,11 @@ stdenv versionSuffix ; - busybox-sandbox-shell = final.busybox-sandbox-shell or default-busybox-sandbox-shell; - libgit2 = final.libgit2.overrideAttrs (attrs: { - src = libgit2; - version = libgit2.lastModifiedDate; - cmakeFlags = attrs.cmakeFlags or [] - ++ [ "-DUSE_SSH=exec" ]; - }); - lowdown = lowdown-nix; officialRelease = false; + libgit2 = final.libgit2-nix; + lowdown = final.lowdown-nix; + busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; }; - - inherit lowdown-nix; }; in { @@ -272,7 +284,10 @@ dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); # Line coverage analysis. - coverage = nixpkgsFor.x86_64-linux.native.callPackage ./coverage.nix {}; + coverage = nixpkgsFor.x86_64-linux.native.nix.override { + pname = "nix-coverage"; + withCoverageChecks = true; + }; # API docs for Nix's unstable internal C++ interfaces. internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix { diff --git a/package.nix b/package.nix index f688cc819..0758f989e 100644 --- a/package.nix +++ b/package.nix @@ -2,9 +2,6 @@ , callPackage , stdenv , releaseTools -, versionSuffix ? "" -, officialRelease ? false -, buildUnreleasedNotes ? false , autoconf-archive , autoreconfHook , aws-sdk-cpp @@ -43,21 +40,25 @@ , busybox-sandbox-shell ? null # Configuration Options -# +#: # This probably seems like too many degrees of freedom, but it # faithfully reflects how the underlying configure + make build system # work. The top-level flake.nix will choose useful combinations. , pname ? "nix" +, versionSuffix ? "" +, officialRelease ? false + , doBuild ? true , doCheck ? __forDefaults.canRunInstalled -, doInstallCheck ? __forDefaults.canRunInstalled +, doInstallCheck ? test-client != null || __forDefaults.canRunInstalled , withCoverageChecks ? false # Whether to build the regular manual , enableManual ? __forDefaults.canRunInstalled +, buildUnreleasedNotes ? false # Whether to build the internal API docs, can be done separately from # everything else. , enableInternalAPIDocs ? false @@ -115,7 +116,11 @@ let mkDerivation = if withCoverageChecks - then releaseTools.coverageAnalysis + then + # TODO support `finalAttrs` args function in + # `releaseTools.coverageAnalysis`. + argsFun: + releaseTools.coverageAnalysis (let args = argsFun args; in args) else stdenv.mkDerivation; in @@ -146,6 +151,7 @@ in { fileset = fileset.intersect filesets.baseFiles (fileset.unions ([ filesets.configureFiles filesets.topLevelBuildFiles + ./doc/internal-api ] ++ lib.optionals doBuild [ ./boehmgc-coroutine-sp-fallback.diff ./doc @@ -170,20 +176,24 @@ in { ++ lib.optional installUnitTests "check"; nativeBuildInputs = [ - bison - flex - (lib.getBin lowdown) - jq # Also for custom mdBook preprocessor. - mdbook - mdbook-linkcheck autoconf-archive autoreconfHook pkg-config - ] - ++ lib.optional stdenv.hostPlatform.isLinux util-linux - # Official releases don't have rl-next, so we don't need to compile a - # changelog - ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d; + ] ++ lib.optionals doBuild [ + bison + flex + ] ++ lib.optionals enableManual [ + (lib.getBin lowdown) + mdbook + mdbook-linkcheck + ] ++ lib.optionals (doInstallCheck || enableManual) [ + jq # Also for custom mdBook preprocessor. + ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux + # Official releases don't have rl-next, so we don't need to compile a + # changelog + ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d + ++ lib.optional enableInternalAPIDocs doxygen + ; buildInputs = lib.optionals doBuild [ boost @@ -225,13 +235,11 @@ in { git mercurial openssh - ] ++ lib.optionals enableInternalAPIDocs [ - doxygen ]; disallowedReferences = [ boost ]; - preConfigure = lib.optionalString (! stdenv.hostPlatform.isStatic) '' + preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) '' # Copy libboost_context so we don't get all of Boost in our closure. # https://github.com/NixOS/nixpkgs/issues/45462 mkdir -p $out/lib @@ -307,7 +315,14 @@ in { doInstallCheck = attrs.doInstallCheck; installCheckFlags = "sysconfdir=$(out)/etc"; - installCheckTarget = "installcheck"; # work around buggy detection in stdenv + # work around buggy detection in stdenv + installCheckTarget = "installcheck"; + + # work around weird bug where it doesn't want to do anything + installCheckPhase = if (!doBuild && doInstallCheck) then '' + mkdir -p src/nix-channel + make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES + '' else null; # Needed for tests if we are not doing a build, but testing existing # built Nix. @@ -317,7 +332,9 @@ in { separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; + # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated + # to work with `strictDeps`. + strictDeps = !withCoverageChecks; hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; From c160c6251566e758dd4d8fd409df3fa3b2f832b9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:12:22 -0500 Subject: [PATCH 731/909] Fix underlying build system so `--disable-build` works better - Internal API docs once again work - configure skips checks for a bunch of things it doesn't need --- Makefile | 2 +- configure.ac | 50 ++++++++++++++++++++++++++++---------------------- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index eea297c89..0b2b408ca 100644 --- a/Makefile +++ b/Makefile @@ -61,7 +61,7 @@ include mk/lib.mk # by the library. Rules are not "lazy" like variables, unfortunately. ifeq ($(ENABLE_BUILD), yes) $(eval $(call include-sub-makefile, doc/manual/local.mk)) -$(eval $(call include-sub-makefile, doc/internal-api/local.mk)) endif +$(eval $(call include-sub-makefile, doc/internal-api/local.mk)) GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src diff --git a/configure.ac b/configure.ac index f8b937eb5..f9ad3c840 100644 --- a/configure.ac +++ b/configure.ac @@ -122,7 +122,6 @@ AC_PATH_PROG(flex, flex, false) AC_PATH_PROG(bison, bison, false) AC_PATH_PROG(dot, dot) AC_PATH_PROG(lsof, lsof, lsof) -NEED_PROG(jq, jq) AC_SUBST(coreutils, [$(dirname $(type -p cat))]) @@ -133,6 +132,30 @@ AC_ARG_WITH(store-dir, AS_HELP_STRING([--with-store-dir=PATH],[path of the Nix s AC_SUBST(storedir) +# Running the functional tests without building Nix is useful for testing +# different pre-built versions of Nix against each other. +AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]), + ENABLE_BUILD=$enableval, ENABLE_BUILD=yes) +AC_SUBST(ENABLE_BUILD) + +# Building without tests is useful for bootstrapping with a smaller footprint +# or running the tests in a separate derivation. Otherwise, we do compile and +# run them. +AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]), + ENABLE_TESTS=$enableval, ENABLE_TESTS=yes) +AC_SUBST(ENABLE_TESTS) + +# Building without API docs is the default as Nix' C++ interfaces are internal and unstable. +AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]), + internal_api_docs=$enableval, internal_api_docs=no) +AC_SUBST(internal_api_docs) + +AS_IF( + [test "$ENABLE_BUILD" == "yes" || test "$ENABLE_TEST" == "yes"], + [NEED_PROG(jq, jq)]) + +AS_IF([test "$ENABLE_BUILD" == "yes"],[ + # Look for boost, a required dependency. # Note that AX_BOOST_BASE only exports *CPP* BOOST_CPPFLAGS, no CXX flags, # and CPPFLAGS are not passed to the C++ compiler automatically. @@ -155,18 +178,6 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then LDFLAGS="-latomic $LDFLAGS" fi -# Running the functional tests without building Nix is useful for testing -# different pre-built versions of Nix against each other. -AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]), - ENABLE_BUILD=$enableval, ENABLE_BUILD=yes) -AC_SUBST(ENABLE_BUILD) -# Building without tests is useful for bootstrapping with a smaller footprint -# or running the tests in a separate derivation. Otherwise, we do compile and -# run them. -AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]), - ENABLE_TESTS=$enableval, ENABLE_TESTS=yes) -AC_SUBST(ENABLE_TESTS) - AC_ARG_ENABLE(install-unit-tests, AS_HELP_STRING([--enable-install-unit-tests],[Install the unit tests for running later (default no)]), INSTALL_UNIT_TESTS=$enableval, INSTALL_UNIT_TESTS=no) AC_SUBST(INSTALL_UNIT_TESTS) @@ -179,11 +190,6 @@ AC_ARG_WITH(check-lib-dir, AS_HELP_STRING([--with-check-lib-dir=PATH],[path to i checklibdir=$withval, checklibdir=$libdir) AC_SUBST(checklibdir) -# Building without API docs is the default as Nix' C++ interfaces are internal and unstable. -AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]), - internal_api_docs=$enableval, internal_api_docs=no) -AC_SUBST(internal_api_docs) - # LTO is currently broken with clang for unknown reasons; ld segfaults in the llvm plugin AC_ARG_ENABLE(lto, AS_HELP_STRING([--enable-lto],[Enable LTO (only supported with GCC) [default=no]]), lto=$enableval, lto=no) @@ -310,8 +316,7 @@ if test "$gc" = yes; then AC_DEFINE(HAVE_BOEHMGC, 1, [Whether to use the Boehm garbage collector.]) fi - -if test "$ENABLE_TESTS" = yes; then +AS_IF([test "$ENABLE_TESTS" == "yes"],[ # Look for gtest. PKG_CHECK_MODULES([GTEST], [gtest_main]) @@ -338,12 +343,11 @@ AC_LINK_IFELSE([ [AC_MSG_ERROR([librapidcheck is not found.])]) AC_LANG_POP(C++) -fi +]) # Look for nlohmann/json. PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9]) - # documentation generation switch AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation generation]), doc_generate=$enableval, doc_generate=yes) @@ -388,6 +392,8 @@ if test "$embedded_sandbox_shell" = yes; then AC_DEFINE(HAVE_EMBEDDED_SANDBOX_SHELL, 1, [Include the sandbox shell in the Nix binary.]) fi +]) + # Expand all variables in config.status. test "$prefix" = NONE && prefix=$ac_default_prefix From 7a7ad7c84b4dd37331a8f8889b02c94540522dbc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:14:36 -0500 Subject: [PATCH 732/909] Remove uneeded file --- test-nix-versions.nix | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 test-nix-versions.nix diff --git a/test-nix-versions.nix b/test-nix-versions.nix deleted file mode 100644 index bda4621a1..000000000 --- a/test-nix-versions.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ lib -, fileset -, stdenv -, client -, daemon -}: - -stdenv.mkDerivation { - name = - "nix-tests" - + lib.optionalString - (lib.versionAtLeast daemon.version "2.4pre20211005" && - lib.versionAtLeast client.version "2.4pre20211005") - "-${client.version}-against-${daemon.version}"; -} From e275f0adfb6b3f360f10f5adcf140c17edc58cc6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:16:07 -0500 Subject: [PATCH 733/909] Move `binary-tarball.nix` to scripts dir --- flake.nix | 2 +- binary-tarball.nix => scripts/binary-tarball.nix | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) rename binary-tarball.nix => scripts/binary-tarball.nix (85%) diff --git a/flake.nix b/flake.nix index fab8c45be..5c6ad3bc7 100644 --- a/flake.nix +++ b/flake.nix @@ -139,7 +139,7 @@ doBuild = false; }; - binaryTarball = nix: pkgs: pkgs.callPackage ./binary-tarball.nix { + binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix { inherit nix; }; diff --git a/binary-tarball.nix b/scripts/binary-tarball.nix similarity index 85% rename from binary-tarball.nix rename to scripts/binary-tarball.nix index 0053abbca..32e811c94 100644 --- a/binary-tarball.nix +++ b/scripts/binary-tarball.nix @@ -21,18 +21,18 @@ in runCommand "nix-binary-tarball-${version}" env '' cp ${installerClosureInfo}/registration $TMPDIR/reginfo - cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh - substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ + cp ${./create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh + substitute ${./install-nix-from-closure.sh} $TMPDIR/install \ --subst-var-by nix ${nix} \ --subst-var-by cacert ${cacert} - substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ + substitute ${./install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ --subst-var-by nix ${nix} \ --subst-var-by cacert ${cacert} - substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ + substitute ${./install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ --subst-var-by nix ${nix} \ --subst-var-by cacert ${cacert} - substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ + substitute ${./install-multi-user.sh} $TMPDIR/install-multi-user \ --subst-var-by nix ${nix} \ --subst-var-by cacert ${cacert} From 60fe4ddaa1801b37a044a2c96071d96739bd26c0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:17:47 -0500 Subject: [PATCH 734/909] Expose `boehmgc-nix` in overlay --- flake.nix | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/flake.nix b/flake.nix index 5c6ad3bc7..78fc88bed 100644 --- a/flake.nix +++ b/flake.nix @@ -188,6 +188,17 @@ ++ [ "-DUSE_SSH=exec" ]; }); + boehmgc-nix = (final.boehmgc.override { + enableLargeConfig = true; + }).overrideAttrs(o: { + patches = (o.patches or []) ++ [ + ./boehmgc-coroutine-sp-fallback.diff + + # https://github.com/ivmai/bdwgc/pull/586 + ./boehmgc-traceable_allocator-public.diff + ]; + }); + nix = let officialRelease = false; @@ -196,25 +207,14 @@ then "" else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - boehmgc = (final.boehmgc.override { - enableLargeConfig = true; - }).overrideAttrs(o: { - patches = (o.patches or []) ++ [ - ./boehmgc-coroutine-sp-fallback.diff - - # https://github.com/ivmai/bdwgc/pull/586 - ./boehmgc-traceable_allocator-public.diff - ]; - }); - in final.callPackage ./package.nix { inherit - boehmgc fileset stdenv versionSuffix ; officialRelease = false; + boehmgc = final.boehmgc-nix; libgit2 = final.libgit2-nix; lowdown = final.lowdown-nix; busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; From 77003a4f0c380929f18b71476b9e7f9cd4009458 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:29:15 -0500 Subject: [PATCH 735/909] Factor out the installer script --- flake.nix | 43 +++++++++++-------------------------------- scripts/installer.nix | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 32 deletions(-) create mode 100644 scripts/installer.nix diff --git a/flake.nix b/flake.nix index 78fc88bed..ecd0381a2 100644 --- a/flake.nix +++ b/flake.nix @@ -89,38 +89,17 @@ }); installScriptFor = systems: - with nixpkgsFor.x86_64-linux.native; - runCommand "installer-script" - { buildInputs = [ nix ]; - } - '' - mkdir -p $out/nix-support - - # Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix. - tarballPath() { - # Remove the store prefix - local path=''${1#${builtins.storeDir}/} - # Get the path relative to the derivation root - local rest=''${path#*/} - # Get the derivation hash - local drvHash=''${path%%-*} - echo "$drvHash/$rest" - } - - substitute ${./scripts/install.in} $out/install \ - ${pkgs.lib.concatMapStrings - (system: let - tarball = if builtins.elem system crossSystems then self.hydraJobs.binaryTarballCross.x86_64-linux.${system} else self.hydraJobs.binaryTarball.${system}; - in '' \ - --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \ - --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \ - '' - ) - systems - } --replace '@nixVersion@' ${version} - - echo "file installer $out/install" >> $out/nix-support/hydra-build-products - ''; + nixpkgsFor.x86_64-linux.native.callPackage ./scripts/installer.nix { + systemTarballPairs = map + (system: { + inherit system; + tarball = + if builtins.elem system crossSystems + then self.hydraJobs.binaryTarballCross.x86_64-linux.${system} + else self.hydraJobs.binaryTarball.${system}; + }) + systems; + }; testNixVersions = pkgs: client: daemon: pkgs.callPackage ./package.nix { diff --git a/scripts/installer.nix b/scripts/installer.nix new file mode 100644 index 000000000..35d2d7fe6 --- /dev/null +++ b/scripts/installer.nix @@ -0,0 +1,35 @@ +{ lib +, runCommand +, nix +, systemTarballPairs +}: + +runCommand "installer-script" { + buildInputs = [ nix ]; +} '' + mkdir -p $out/nix-support + + # Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix. + tarballPath() { + # Remove the store prefix + local path=''${1#${builtins.storeDir}/} + # Get the path relative to the derivation root + local rest=''${path#*/} + # Get the derivation hash + local drvHash=''${path%%-*} + echo "$drvHash/$rest" + } + + substitute ${./install.in} $out/install \ + ${lib.concatMapStrings + ({ system, tarball }: + '' \ + --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \ + --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \ + '' + ) + systemTarballPairs + } --replace '@nixVersion@' ${nix.version} + + echo "file installer $out/install" >> $out/nix-support/hydra-build-products +'' From f58615518c1284d4dbe4655246d4d5d6e9b2befe Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:39:33 -0500 Subject: [PATCH 736/909] Add documenting comments to `package.nix` --- package.nix | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/package.nix b/package.nix index 0758f989e..7f0d78b5c 100644 --- a/package.nix +++ b/package.nix @@ -43,22 +43,37 @@ #: # This probably seems like too many degrees of freedom, but it # faithfully reflects how the underlying configure + make build system -# work. The top-level flake.nix will choose useful combinations. +# work. The top-level flake.nix will choose useful combinations of these +# options to CI. , pname ? "nix" , versionSuffix ? "" , officialRelease ? false +# Whether to build Nix. Useful to skip for tasks like (a) just +# generating API docs or (b) testing existing pre-built versions of Nix , doBuild ? true + +# Run the unit tests as part of the build. See `installUnitTests` for an +# alternative to this. , doCheck ? __forDefaults.canRunInstalled + +# Run the functional tests as part of the build. , doInstallCheck ? test-client != null || __forDefaults.canRunInstalled +# Check test coverage of Nix. Probably want to use with with at least +# one of `doCHeck` or `doInstallCheck` enabled. , withCoverageChecks ? false # Whether to build the regular manual , enableManual ? __forDefaults.canRunInstalled + +# Whether to compile `rl-next.md`, the release notes for the next +# not-yet-released version of Nix in the manul, from the individual +# change log entries in the directory. , buildUnreleasedNotes ? false + # Whether to build the internal API docs, can be done separately from # everything else. , enableInternalAPIDocs ? false @@ -350,8 +365,13 @@ in { platforms = lib.platforms.unix; mainProgram = "nix"; broken = !(lib.all (a: a) [ + # We cannot run or install unit tests if we don't build them or + # Nix proper (which they depend on). (installUnitTests -> doBuild) (doCheck -> doBuild) + # We have to build the manual to build unreleased notes, as those + # are part of the manual + (buildUnreleasedNotes -> enableManual) ]); }; From a5a45e64e18de3eb827ca83c7356dc8a088be125 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:45:15 -0500 Subject: [PATCH 737/909] Don't expose file sets anymore --- package.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/package.nix b/package.nix index 7f0d78b5c..52050496c 100644 --- a/package.nix +++ b/package.nix @@ -354,8 +354,6 @@ in { hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { - inherit filesets; - perl-bindings = callPackage ./perl { inherit fileset stdenv; }; From 7e2b1cce6abec48f85c8bc056da0ca991dfe7b32 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:47:54 -0500 Subject: [PATCH 738/909] Slap on `perl-bindings` in the caller The Perl bindings are not part of Nix, but a downstream package, so they don't belong in `package.nix`. They don't really belong as an attribute on `nix` either, but we can just leave that interface as is for now. --- flake.nix | 9 +++++++++ package.nix | 7 ------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/flake.nix b/flake.nix index ecd0381a2..c92f717d5 100644 --- a/flake.nix +++ b/flake.nix @@ -197,7 +197,16 @@ libgit2 = final.libgit2-nix; lowdown = final.lowdown-nix; busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; + } // { + # this is a proper separate downstream package, but put + # here also for back compat reasons. + perl-bindings = final.nix-perl-bindings; }; + + nix-perl-bindings = final.callPackage ./perl { + inherit fileset stdenv; + }; + }; in { diff --git a/package.nix b/package.nix index 52050496c..f6219e58a 100644 --- a/package.nix +++ b/package.nix @@ -1,5 +1,4 @@ { lib -, callPackage , stdenv , releaseTools , autoconf-archive @@ -353,12 +352,6 @@ in { hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - passthru = { - perl-bindings = callPackage ./perl { - inherit fileset stdenv; - }; - }; - meta = { platforms = lib.platforms.unix; mainProgram = "nix"; From 6e0656c66c1052bcbab204140c6b3dec81f3ab15 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:53:05 -0500 Subject: [PATCH 739/909] Add another configure flag assertion --- package.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/package.nix b/package.nix index f6219e58a..42f98a48c 100644 --- a/package.nix +++ b/package.nix @@ -363,6 +363,9 @@ in { # We have to build the manual to build unreleased notes, as those # are part of the manual (buildUnreleasedNotes -> enableManual) + # The build process for the manual currently requires extracting + # data from the Nix executable we are trying to document. + (enableManual -> doBuild) ]); }; From 14c26d642ebcff3fe45c8eb6719a213c63143fb3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:57:16 -0500 Subject: [PATCH 740/909] Clean up two comments --- package.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.nix b/package.nix index 42f98a48c..96b9111f8 100644 --- a/package.nix +++ b/package.nix @@ -329,10 +329,10 @@ in { doInstallCheck = attrs.doInstallCheck; installCheckFlags = "sysconfdir=$(out)/etc"; - # work around buggy detection in stdenv + # Work around buggy detection in stdenv. installCheckTarget = "installcheck"; - # work around weird bug where it doesn't want to do anything + # Work around weird bug where it doesn't think there is a Makefile. installCheckPhase = if (!doBuild && doInstallCheck) then '' mkdir -p src/nix-channel make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES From 2e5abc0fd0d5d45e125e1d981958149624268090 Mon Sep 17 00:00:00 2001 From: wh0 Date: Sun, 3 Dec 2023 17:18:58 -0800 Subject: [PATCH 741/909] tests: avoid a chroot store without sandbox support --- tests/functional/build-remote-trustless-should-fail-0.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/build-remote-trustless-should-fail-0.sh b/tests/functional/build-remote-trustless-should-fail-0.sh index fad1def59..3d4a4b097 100644 --- a/tests/functional/build-remote-trustless-should-fail-0.sh +++ b/tests/functional/build-remote-trustless-should-fail-0.sh @@ -4,6 +4,7 @@ enableFeatures "daemon-trust-override" restartDaemon +requireSandboxSupport [[ $busybox =~ busybox ]] || skipTest "no busybox" unset NIX_STORE_DIR From 3c310bde2e492c2dd8bdccdfd80076231905a429 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 14 Nov 2023 11:40:56 +0100 Subject: [PATCH 742/909] reword description for the `fetch-tree` experimental feature without knowing a lot of context, it's not clear who "we" are in that text. I'm also strongly opposed to adding procedural notes into a reference manual; it just won't age well. this change leaves a factual description of the experimental feature and its purpose. --- src/libutil/experimental-features.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index e4bdb8cb3..9b46fc5b0 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -80,12 +80,11 @@ constexpr std::array xpFeatureDetails .description = R"( Enable the use of the [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) built-in function in the Nix language. - `fetchTree` exposes a large suite of fetching functionality in a more systematic way. + `fetchTree` exposes a generic interface for fetching remote file system trees from different types of remote sources. The [`flakes`](#xp-feature-flakes) feature flag always enables `fetch-tree`. + This built-in was previously guarded by the `flakes` experimental feature because of that overlap. - This built-in was previously guarded by the `flakes` experimental feature because of that overlap, - but since the plan is to work on stabilizing this first (due 2024 Q1), we are putting it underneath a separate feature. - Once we've made the changes we want to make, enabling just this feature will serve as a "release candidate" --- allowing users to try out the functionality we want to stabilize and not any other functionality we don't yet want to, in isolation. + Enabling just this feature serves as a "release candidate", allowing users to try it out in isolation. )", }, { From 5fe2accb754249df6cb8f840330abfcf3bd26695 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 14 Nov 2023 11:44:34 +0100 Subject: [PATCH 743/909] fix up release note --- doc/manual/src/release-notes/rl-2.19.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md index 4eecaf929..ba6eb9c64 100644 --- a/doc/manual/src/release-notes/rl-2.19.md +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -18,7 +18,7 @@ - `nix-shell` shebang lines now support single-quoted arguments. - `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - As described in the documentation for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of flakes. + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - The interface for creating and updating lock files has been overhauled: From 823512c1e705d1fce8dfb8cde65228364c9a8045 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:09:10 +0000 Subject: [PATCH 744/909] Bump zeebe-io/backport-action from 2.1.1 to 2.2.0 Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 2.1.1 to 2.2.0. - [Release notes](https://github.com/zeebe-io/backport-action/releases) - [Commits](https://github.com/zeebe-io/backport-action/compare/v2.1.1...v2.2.0) --- updated-dependencies: - dependency-name: zeebe-io/backport-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 975c90b91..85ddcfad3 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v2.1.1 + uses: zeebe-io/backport-action@v2.2.0 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} From c446e5294dbc12729e7bc55ee10b40dbaeeaacf0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:09:14 +0000 Subject: [PATCH 745/909] Bump cachix/install-nix-action from 23 to 24 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 23 to 24. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v23...v24) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afe4dc2e3..34a23b5f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v23 + - uses: cachix/install-nix-action@v24 with: # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: "sandbox = true" @@ -62,7 +62,7 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v23 + - uses: cachix/install-nix-action@v24 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - uses: cachix/cachix-action@v12 @@ -84,7 +84,7 @@ jobs: steps: - uses: actions/checkout@v4 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v23 + - uses: cachix/install-nix-action@v24 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -114,7 +114,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v23 + - uses: cachix/install-nix-action@v24 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV From e6a3cbfceb66e06184b625a3913a786f68e71a1f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:09:18 +0000 Subject: [PATCH 746/909] Bump cachix/cachix-action from 12 to 13 Bumps [cachix/cachix-action](https://github.com/cachix/cachix-action) from 12 to 13. - [Release notes](https://github.com/cachix/cachix-action/releases) - [Commits](https://github.com/cachix/cachix-action/compare/v12...v13) --- updated-dependencies: - dependency-name: cachix/cachix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afe4dc2e3..033832c9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: "sandbox = true" - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v12 + - uses: cachix/cachix-action@v13 if: needs.check_secrets.outputs.cachix == 'true' with: name: '${{ env.CACHIX_NAME }}' @@ -65,7 +65,7 @@ jobs: - uses: cachix/install-nix-action@v23 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - - uses: cachix/cachix-action@v12 + - uses: cachix/cachix-action@v13 with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' @@ -119,7 +119,7 @@ jobs: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v12 + - uses: cachix/cachix-action@v13 if: needs.check_secrets.outputs.cachix == 'true' with: name: '${{ env.CACHIX_NAME }}' From e488a43f457f3ef9dba92184428bbe5381fe2634 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:09:25 +0000 Subject: [PATCH 747/909] Bump actions/labeler from 4 to 5 Bumps [actions/labeler](https://github.com/actions/labeler) from 4 to 5. - [Release notes](https://github.com/actions/labeler/releases) - [Commits](https://github.com/actions/labeler/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/labeler dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/labels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index d83cb4f18..34aa4e6bd 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest if: github.repository_owner == 'NixOS' steps: - - uses: actions/labeler@v4 + - uses: actions/labeler@v5 with: repo-token: ${{ secrets.GITHUB_TOKEN }} sync-labels: false From 345f79d01676680f2d4ef8803790896a190c855b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 Dec 2023 15:14:28 +0100 Subject: [PATCH 748/909] Check that we can't follow symlinks outside of the allowed paths --- tests/functional/restricted.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index b8deceacc..cb83c34b1 100644 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -39,6 +39,15 @@ nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT - [[ $(nix eval --raw --impure --restrict-eval -I . --expr 'builtins.readFile "${import ./simple.nix}/hello"') == 'Hello World!' ]] +# Check that we can't follow a symlink outside of the allowed paths. +mkdir -p $TEST_ROOT/tunnel.d +ln -sfn .. $TEST_ROOT/tunnel.d/tunnel +echo foo > $TEST_ROOT/bar + +expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readFile " -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode" + +expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir " -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode" + # Check whether we can leak symlink information through directory traversal. traverseDir="$(pwd)/restricted-traverse-me" ln -sfn "$(pwd)/restricted-secret" "$(pwd)/restricted-innocent" From 733333e87db391e4f832de65f0f49f60e50c45a4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 1 Dec 2023 17:38:34 -0500 Subject: [PATCH 749/909] Including `config.h` also needs `$(buildprefix)` Per the instruction in the manual, we want to run configure in a different directory so that we can configure + build for multiple platforms. That means `config.h` will be in the build directory. This is just like `Makefile.config`, which already is used with `$(buildprefix)`. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index eea297c89..41f14ac92 100644 --- a/Makefile +++ b/Makefile @@ -64,4 +64,4 @@ $(eval $(call include-sub-makefile, doc/manual/local.mk)) $(eval $(call include-sub-makefile, doc/internal-api/local.mk)) endif -GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src +GLOBAL_CXXFLAGS += -g -Wall -include $(buildprefix)config.h -std=c++2a -I src From 83c067c0fa0cc5a2dca440e5c986afe40b163802 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 Dec 2023 23:02:59 +0100 Subject: [PATCH 750/909] PosixSourceAccessor: Don't follow any symlinks All path components must not be symlinks now (so the user needs to call `resolveSymlinks()` when needed). --- src/libexpr/parser.y | 11 +++++----- src/libexpr/primops.cc | 30 ++++++++++++++-------------- src/libutil/posix-source-accessor.cc | 27 +++++++++++++++++++++---- src/libutil/posix-source-accessor.hh | 5 +++++ src/nix-env/nix-env.cc | 6 +++--- src/nix-env/user-env.cc | 2 +- tests/functional/restricted.sh | 7 +++++-- 7 files changed, 58 insertions(+), 30 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 58fc580fc..16ad8af2e 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -692,16 +692,17 @@ SourcePath resolveExprPath(SourcePath path) /* If `path' is a symlink, follow it. This is so that relative path references work. */ - while (true) { + while (!path.path.isRoot()) { // Basic cycle/depth limit to avoid infinite loops. if (++followCount >= maxFollow) throw Error("too many symbolic links encountered while traversing the path '%s'", path); - if (path.lstat().type != InputAccessor::tSymlink) break; - path = {path.accessor, CanonPath(path.readLink(), path.path.parent().value_or(CanonPath::root))}; + auto p = path.parent().resolveSymlinks() + path.baseName(); + if (p.lstat().type != InputAccessor::tSymlink) break; + path = {path.accessor, CanonPath(p.readLink(), path.path.parent().value_or(CanonPath::root))}; } /* If `path' refers to a directory, append `/default.nix'. */ - if (path.lstat().type == InputAccessor::tDirectory) + if (path.resolveSymlinks().lstat().type == InputAccessor::tDirectory) return path + "default.nix"; return path; @@ -716,7 +717,7 @@ Expr * EvalState::parseExprFromFile(const SourcePath & path) Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr & staticEnv) { - auto buffer = path.readFile(); + auto buffer = path.resolveSymlinks().readFile(); // readFile hopefully have left some extra space for terminators buffer.append("\0\0", 2); return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c442de986..f2d51f8f5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -110,7 +110,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context) return res; } -static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v) +static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, bool resolveSymlinks = true) { NixStringContext context; @@ -120,9 +120,9 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v) if (!context.empty() && path.accessor == state.rootFS) { auto rewrites = state.realiseContext(context); auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context); - return {path.accessor, CanonPath(realPath)}; - } else - return path; + path = {path.accessor, CanonPath(realPath)}; + } + return resolveSymlinks ? path.resolveSymlinks() : path; } catch (Error & e) { e.addTrace(state.positions[pos], "while realising the context of path '%s'", path); throw; @@ -162,7 +162,7 @@ static void mkOutputString( argument. */ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * vScope, Value & v) { - auto path = realisePath(state, pos, vPath); + auto path = realisePath(state, pos, vPath, false); auto path2 = path.path.abs(); // FIXME @@ -1525,16 +1525,16 @@ static RegisterPrimOp primop_storePath({ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto & arg = *args[0]; - - auto path = realisePath(state, pos, arg); - - /* SourcePath doesn't know about trailing slash. */ - auto mustBeDir = arg.type() == nString - && (arg.string_view().ends_with("/") - || arg.string_view().ends_with("/.")); - try { + auto & arg = *args[0]; + + auto path = realisePath(state, pos, arg); + + /* SourcePath doesn't know about trailing slash. */ + auto mustBeDir = arg.type() == nString + && (arg.string_view().ends_with("/") + || arg.string_view().ends_with("/.")); + auto st = path.maybeLstat(); auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory); v.mkBool(exists); @@ -1771,7 +1771,7 @@ static std::string_view fileTypeToString(InputAccessor::Type type) static void prim_readFileType(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto path = realisePath(state, pos, *args[0]); + auto path = realisePath(state, pos, *args[0], false); /* Retrieve the directory entry type and stringize it. */ v.mkString(fileTypeToString(path.lstat().type)); } diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index dc96f84e5..0601e6387 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -8,9 +8,9 @@ void PosixSourceAccessor::readFile( Sink & sink, std::function sizeCallback) { - // FIXME: add O_NOFOLLOW since symlinks should be resolved by the - // caller? - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + assertNoSymlinks(path); + + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW); if (!fd) throw SysError("opening file '%1%'", path); @@ -42,14 +42,16 @@ void PosixSourceAccessor::readFile( bool PosixSourceAccessor::pathExists(const CanonPath & path) { + if (auto parent = path.parent()) assertNoSymlinks(*parent); return nix::pathExists(path.abs()); } std::optional PosixSourceAccessor::maybeLstat(const CanonPath & path) { + if (auto parent = path.parent()) assertNoSymlinks(*parent); struct stat st; if (::lstat(path.c_str(), &st)) { - if (errno == ENOENT) return std::nullopt; + if (errno == ENOENT || errno == ENOTDIR) return std::nullopt; throw SysError("getting status of '%s'", showPath(path)); } mtime = std::max(mtime, st.st_mtime); @@ -66,6 +68,7 @@ std::optional PosixSourceAccessor::maybeLstat(const CanonP SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path) { + assertNoSymlinks(path); DirEntries res; for (auto & entry : nix::readDirectory(path.abs())) { std::optional type; @@ -81,6 +84,7 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & std::string PosixSourceAccessor::readLink(const CanonPath & path) { + if (auto parent = path.parent()) assertNoSymlinks(*parent); return nix::readLink(path.abs()); } @@ -89,4 +93,19 @@ std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & return path; } +void PosixSourceAccessor::assertNoSymlinks(CanonPath path) +{ + // FIXME: cache this since it potentially causes a lot of lstat calls. + while (!path.isRoot()) { + struct stat st; + if (::lstat(path.c_str(), &st)) { + if (errno != ENOENT) + throw SysError("getting status of '%s'", showPath(path)); + } + if (S_ISLNK(st.st_mode)) + throw Error("path '%s' is a symlink", showPath(path)); + path.pop(); + } +} + } diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh index a45d96bf8..7189a40e5 100644 --- a/src/libutil/posix-source-accessor.hh +++ b/src/libutil/posix-source-accessor.hh @@ -29,6 +29,11 @@ struct PosixSourceAccessor : virtual SourceAccessor std::string readLink(const CanonPath & path) override; std::optional getPhysicalPath(const CanonPath & path) override; + + /** + * Throw an error if `path` or any of its ancestors are symlinks. + */ + void assertNoSymlinks(CanonPath path); }; } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 86126c7ad..e2bbd9775 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -97,7 +97,7 @@ static bool isNixExpr(const SourcePath & path, struct InputAccessor::Stat & st) { return st.type == InputAccessor::tRegular - || (st.type == InputAccessor::tDirectory && (path + "default.nix").pathExists()); + || (st.type == InputAccessor::tDirectory && (path + "default.nix").resolveSymlinks().pathExists()); } @@ -116,11 +116,11 @@ static void getAllExprs(EvalState & state, are implemented using profiles). */ if (i == "manifest.nix") continue; - SourcePath path2 = path + i; + auto path2 = (path + i).resolveSymlinks(); InputAccessor::Stat st; try { - st = path2.resolveSymlinks().lstat(); + st = path2.lstat(); } catch (Error &) { continue; // ignore dangling symlinks in ~/.nix-defexpr } diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 250224e7d..34f6bd005 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -21,7 +21,7 @@ DrvInfos queryInstalled(EvalState & state, const Path & userEnv) auto manifestFile = userEnv + "/manifest.nix"; if (pathExists(manifestFile)) { Value v; - state.evalFile(state.rootPath(CanonPath(manifestFile)), v); + state.evalFile(state.rootPath(CanonPath(manifestFile)).resolveSymlinks(), v); Bindings & bindings(*state.allocBindings(0)); getDerivations(state, v, "", bindings, elems, false); } diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index cb83c34b1..2d6ab964b 100644 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -40,13 +40,16 @@ nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT - [[ $(nix eval --raw --impure --restrict-eval -I . --expr 'builtins.readFile "${import ./simple.nix}/hello"') == 'Hello World!' ]] # Check that we can't follow a symlink outside of the allowed paths. -mkdir -p $TEST_ROOT/tunnel.d +mkdir -p $TEST_ROOT/tunnel.d $TEST_ROOT/foo2 ln -sfn .. $TEST_ROOT/tunnel.d/tunnel echo foo > $TEST_ROOT/bar expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readFile " -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode" -expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir " -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode" +expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir " -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode" + +# Reading the parents of allowed paths should show only the ancestors of the allowed paths. +[[ $(nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir " -I $TEST_ROOT/tunnel.d) == '{ "tunnel.d" = "directory"; }' ]] # Check whether we can leak symlink information through directory traversal. traverseDir="$(pwd)/restricted-traverse-me" From b23273f6a29c725646b3523b1c35a0ae4a84ef61 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 5 Dec 2023 18:10:37 -0500 Subject: [PATCH 751/909] Add missing `-pthread` for test support libraries This is good in general (see how the other libraries also have long had it, since 49fe9592a47e7819179c2de4fd6068e897e944c7) but in particular needed to fix the NetBSD build. --- tests/unit/libexpr-support/local.mk | 2 +- tests/unit/libstore-support/local.mk | 2 +- tests/unit/libutil-support/local.mk | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk index 28e87b8f2..12a76206a 100644 --- a/tests/unit/libexpr-support/local.mk +++ b/tests/unit/libexpr-support/local.mk @@ -20,4 +20,4 @@ libexpr-test-support_LIBS = \ libstore-test-support libutil-test-support \ libexpr libstore libutil -libexpr-test-support_LDFLAGS := -lrapidcheck +libexpr-test-support_LDFLAGS := -pthread -lrapidcheck diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk index d5d657c91..ff075c96a 100644 --- a/tests/unit/libstore-support/local.mk +++ b/tests/unit/libstore-support/local.mk @@ -18,4 +18,4 @@ libstore-test-support_LIBS = \ libutil-test-support \ libstore libutil -libstore-test-support_LDFLAGS := -lrapidcheck +libstore-test-support_LDFLAGS := -pthread -lrapidcheck diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk index 43a1551e5..2ee2cdb6c 100644 --- a/tests/unit/libutil-support/local.mk +++ b/tests/unit/libutil-support/local.mk @@ -16,4 +16,4 @@ libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) libutil-test-support_LIBS = libutil -libutil-test-support_LDFLAGS := -lrapidcheck +libutil-test-support_LDFLAGS := -pthread -lrapidcheck From 504e4fc4576dc6a4cd5c083a3bf7b80dfb0ca220 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Dec 2023 13:45:59 +0100 Subject: [PATCH 752/909] CanonPath: Support std::hash --- src/libfetchers/git-utils.cc | 2 +- src/libutil/canon-path.hh | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 19eae0e1d..5f2a7a8bc 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -554,7 +554,7 @@ struct GitInputAccessor : InputAccessor return toHash(*git_tree_entry_id(entry)); } - std::map lookupCache; + std::unordered_map lookupCache; /* Recursively look up 'path' relative to the root. */ git_tree_entry * lookup(const CanonPath & path) diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh index 6d0519f4f..6aff4ec0d 100644 --- a/src/libutil/canon-path.hh +++ b/src/libutil/canon-path.hh @@ -205,8 +205,19 @@ public: * `CanonPath(this.makeRelative(x), this) == path`. */ std::string makeRelative(const CanonPath & path) const; + + friend class std::hash; }; std::ostream & operator << (std::ostream & stream, const CanonPath & path); } + +template<> +struct std::hash +{ + std::size_t operator ()(const nix::CanonPath & s) const noexcept + { + return std::hash{}(s.path); + } +}; From 57246c4c3802920e6167fd540dae2a0abca97f15 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Dec 2023 13:55:07 +0100 Subject: [PATCH 753/909] PosixSourceAccessor: Cache lstat() calls Since we're doing a lot of them in assertNoSymlinks(). --- src/libutil/posix-source-accessor.cc | 56 +++++++++++++++++++--------- src/libutil/posix-source-accessor.hh | 4 ++ 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 0601e6387..15ff76e59 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -1,5 +1,8 @@ #include "posix-source-accessor.hh" #include "signals.hh" +#include "sync.hh" + +#include namespace nix { @@ -46,23 +49,45 @@ bool PosixSourceAccessor::pathExists(const CanonPath & path) return nix::pathExists(path.abs()); } +std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path) +{ + static Sync>> _cache; + + { + auto cache(_cache.lock()); + auto i = cache->find(path); + if (i != cache->end()) return i->second; + } + + std::optional st{std::in_place}; + if (::lstat(path.c_str(), &*st)) { + if (errno == ENOENT || errno == ENOTDIR) + st.reset(); + else + throw SysError("getting status of '%s'", showPath(path)); + } + + auto cache(_cache.lock()); + if (cache->size() >= 16384) cache->clear(); + cache->emplace(path, st); + + return st; +} + std::optional PosixSourceAccessor::maybeLstat(const CanonPath & path) { if (auto parent = path.parent()) assertNoSymlinks(*parent); - struct stat st; - if (::lstat(path.c_str(), &st)) { - if (errno == ENOENT || errno == ENOTDIR) return std::nullopt; - throw SysError("getting status of '%s'", showPath(path)); - } - mtime = std::max(mtime, st.st_mtime); + auto st = cachedLstat(path); + if (!st) return std::nullopt; + mtime = std::max(mtime, st->st_mtime); return Stat { .type = - S_ISREG(st.st_mode) ? tRegular : - S_ISDIR(st.st_mode) ? tDirectory : - S_ISLNK(st.st_mode) ? tSymlink : + S_ISREG(st->st_mode) ? tRegular : + S_ISDIR(st->st_mode) ? tDirectory : + S_ISLNK(st->st_mode) ? tSymlink : tMisc, - .fileSize = S_ISREG(st.st_mode) ? std::optional(st.st_size) : std::nullopt, - .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR, + .fileSize = S_ISREG(st->st_mode) ? std::optional(st->st_size) : std::nullopt, + .isExecutable = S_ISREG(st->st_mode) && st->st_mode & S_IXUSR, }; } @@ -95,14 +120,9 @@ std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & void PosixSourceAccessor::assertNoSymlinks(CanonPath path) { - // FIXME: cache this since it potentially causes a lot of lstat calls. while (!path.isRoot()) { - struct stat st; - if (::lstat(path.c_str(), &st)) { - if (errno != ENOENT) - throw SysError("getting status of '%s'", showPath(path)); - } - if (S_ISLNK(st.st_mode)) + auto st = cachedLstat(path); + if (st && S_ISLNK(st->st_mode)) throw Error("path '%s' is a symlink", showPath(path)); path.pop(); } diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh index 7189a40e5..b2bd39805 100644 --- a/src/libutil/posix-source-accessor.hh +++ b/src/libutil/posix-source-accessor.hh @@ -30,10 +30,14 @@ struct PosixSourceAccessor : virtual SourceAccessor std::optional getPhysicalPath(const CanonPath & path) override; +private: + /** * Throw an error if `path` or any of its ancestors are symlinks. */ void assertNoSymlinks(CanonPath path); + + std::optional cachedLstat(const CanonPath & path); }; } From 53ab5d87c2eef72202bd76eb43e072636bbc72e8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Dec 2023 14:05:32 +0100 Subject: [PATCH 754/909] Use expectStderr --- tests/functional/restricted.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index 2d6ab964b..3de26eb36 100644 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -14,7 +14,7 @@ nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I sr (! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel') nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel' -I src=../../src -(! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ') +expectStderr 1 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' | grepQuiet "forbidden in restricted mode" nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' -I src=. p=$(nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)") From ee8540ae9055791cfec4cbf8cb6335368b867acc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Wed, 6 Dec 2023 14:07:08 +0100 Subject: [PATCH 755/909] Fix the labeler.yml config file labeler 5.0 changed the configuration file in a non-backwards-compatible way (https://github.com/actions/labeler/tree/main#breaking-changes-in-v5), so update our config file to match that (because all the CIs are red otherwise :grimacing: ). --- .github/labeler.yml | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index 7544f07a6..b1b18c488 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,23 +1,30 @@ "documentation": - - doc/manual/* - - src/nix/**/*.md + - changed-files: + - any-glob-to-any-file: "doc/manual/*" + - any-glob-to-any-file: "src/nix/**/*.md" "store": - - src/libstore/store-api.* - - src/libstore/*-store.* + - changed-files: + - any-glob-to-any-file: "src/libstore/store-api.*" + - any-glob-to-any-file: "src/libstore/*-store.*" "fetching": - - src/libfetchers/**/* + - changed-files: + - any-glob-to-any-file: "src/libfetchers/**/*" "repl": - - src/libcmd/repl.* - - src/nix/repl.* + - changed-files: + - any-glob-to-any-file: "src/libcmd/repl.*" + - any-glob-to-any-file: "src/nix/repl.*" "new-cli": - - src/nix/**/* + - changed-files: + - any-glob-to-any-file: "src/nix/**/*" "with-tests": - # Unit tests - - src/*/tests/**/* - # Functional and integration tests - - tests/functional/**/* + - changed-files: + # Unit tests + - any-glob-to-any-file: "src/*/tests/**/*" + # Functional and integration tests + - any-glob-to-any-file: "tests/functional/**/*" + From 2bd83225004012af97d2d5977dc1de952f60aa8d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Dec 2023 14:08:40 +0100 Subject: [PATCH 756/909] Update src/libfetchers/filtering-input-accessor.hh Co-authored-by: Robert Hensing --- src/libfetchers/filtering-input-accessor.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/filtering-input-accessor.hh b/src/libfetchers/filtering-input-accessor.hh index 209d26974..e1b83c929 100644 --- a/src/libfetchers/filtering-input-accessor.hh +++ b/src/libfetchers/filtering-input-accessor.hh @@ -13,8 +13,8 @@ typedef std::function MakeNotAllowe /** * An abstract wrapping `InputAccessor` that performs access - * control. Subclasses should override `checkAccess()` to implement an - * access control policy. + * control. Subclasses should override `isAllowed()` to implement an + * access control policy. The error message is customized at construction. */ struct FilteringInputAccessor : InputAccessor { From 7fff625e39fa6b11c4c61eeacadc70a0253bdab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Wed, 6 Dec 2023 14:13:45 +0100 Subject: [PATCH 757/909] =?UTF-8?q?Improve=20the=20error=20message=20for?= =?UTF-8?q?=20=E2=80=9Cmulticommands=E2=80=9D=20commands=20(#9510)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Factor out the default `MultiCommand` behavior All the `MultiCommand`s had (nearly) the same behavior when called without a subcommand. Factor out this behavior into the `NixMultiCommand` class. * Display the list of available subcommands when none is specified Whenever a user runs a command that excepts a subcommand, add the list of available subcommands to the error message. * Print the multi-command lists as Markdown lists This takes more screen real estate, but is also much more readable than a comma-separated list --- src/libcmd/command.cc | 14 ++++++++++++++ src/libcmd/command.hh | 6 +++++- src/libutil/args.cc | 5 +++-- src/libutil/args.hh | 9 ++++++--- src/nix/config.cc | 11 ++--------- src/nix/derivation.cc | 11 ++--------- src/nix/flake.cc | 8 ++++---- src/nix/hash.cc | 11 +++-------- src/nix/main.cc | 2 +- src/nix/nar.cc | 9 +-------- src/nix/profile.cc | 11 +++-------- src/nix/realisation.cc | 11 ++--------- src/nix/registry.cc | 14 ++++---------- src/nix/sigs.cc | 11 +++-------- src/nix/store.cc | 11 ++--------- 15 files changed, 55 insertions(+), 89 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index de9f546fc..369fa6004 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -1,4 +1,5 @@ #include "command.hh" +#include "markdown.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "derivations.hh" @@ -34,6 +35,19 @@ nlohmann::json NixMultiCommand::toJSON() return MultiCommand::toJSON(); } +void NixMultiCommand::run() +{ + if (!command) { + std::set subCommandTextLines; + for (auto & [name, _] : commands) + subCommandTextLines.insert(fmt("- `%s`", name)); + std::string markdownError = fmt("`nix %s` requires a sub-command. Available sub-commands:\n\n%s\n", + commandName, concatStringsSep("\n", subCommandTextLines)); + throw UsageError(renderMarkdownToTerminal(markdownError)); + } + command->second->run(); +} + StoreCommand::StoreCommand() { } diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 120c832ac..4a72627ed 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -26,9 +26,13 @@ static constexpr Command::Category catNixInstallation = 102; static constexpr auto installablesCategory = "Options that change the interpretation of [installables](@docroot@/command-ref/new-cli/nix.md#installables)"; -struct NixMultiCommand : virtual MultiCommand, virtual Command +struct NixMultiCommand : MultiCommand, virtual Command { nlohmann::json toJSON() override; + + using MultiCommand::MultiCommand; + + virtual void run() override; }; // For the overloaded run methods diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 4480a03f5..c4b2975ee 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -483,7 +483,7 @@ bool Args::processArgs(const Strings & args, bool finish) if (!anyCompleted) exp.handler.fun(ss); - /* Move the list element to the processedArgs. This is almost the same as + /* Move the list element to the processedArgs. This is almost the same as `processedArgs.push_back(expectedArgs.front()); expectedArgs.pop_front()`, except that it will only adjust the next and prev pointers of the list elements, meaning the actual contents don't move in memory. This is @@ -622,8 +622,9 @@ std::optional Command::experimentalFeature () return { Xp::NixCommand }; } -MultiCommand::MultiCommand(const Commands & commands_) +MultiCommand::MultiCommand(std::string_view commandName, const Commands & commands_) : commands(commands_) + , commandName(commandName) { expectArgs({ .label = "subcommand", diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 7af82b178..72278dccc 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -223,11 +223,11 @@ protected: std::list expectedArgs; /** * List of processed positional argument forms. - * + * * All items removed from `expectedArgs` are added here. After all * arguments were processed, this list should be exactly the same as * `expectedArgs` was before. - * + * * This list is used to extend the lifetime of the argument forms. * If this is not done, some closures that reference the command * itself will segfault. @@ -356,13 +356,16 @@ public: */ std::optional>> command; - MultiCommand(const Commands & commands); + MultiCommand(std::string_view commandName, const Commands & commands); bool processFlag(Strings::iterator & pos, Strings::iterator end) override; bool processArgs(const Strings & args, bool finish) override; nlohmann::json toJSON() override; + +protected: + std::string commandName = ""; }; Strings argvToStrings(int argc, char * * argv); diff --git a/src/nix/config.cc b/src/nix/config.cc index 5b280d11d..52706afcf 100644 --- a/src/nix/config.cc +++ b/src/nix/config.cc @@ -7,9 +7,9 @@ using namespace nix; -struct CmdConfig : virtual NixMultiCommand +struct CmdConfig : NixMultiCommand { - CmdConfig() : MultiCommand(RegisterCommand::getCommandsFor({"config"})) + CmdConfig() : NixMultiCommand("config", RegisterCommand::getCommandsFor({"config"})) { } std::string description() override @@ -18,13 +18,6 @@ struct CmdConfig : virtual NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix config' requires a sub-command."); - command->second->run(); - } }; struct CmdConfigShow : Command, MixJSON diff --git a/src/nix/derivation.cc b/src/nix/derivation.cc index cd3975a4f..59a78d378 100644 --- a/src/nix/derivation.cc +++ b/src/nix/derivation.cc @@ -2,9 +2,9 @@ using namespace nix; -struct CmdDerivation : virtual NixMultiCommand +struct CmdDerivation : NixMultiCommand { - CmdDerivation() : MultiCommand(RegisterCommand::getCommandsFor({"derivation"})) + CmdDerivation() : NixMultiCommand("derivation", RegisterCommand::getCommandsFor({"derivation"})) { } std::string description() override @@ -13,13 +13,6 @@ struct CmdDerivation : virtual NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix derivation' requires a sub-command."); - command->second->run(); - } }; static auto rCmdDerivation = registerCommand("derivation"); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index e0c67fdfa..2b6e56283 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1399,7 +1399,9 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON struct CmdFlake : NixMultiCommand { CmdFlake() - : MultiCommand({ + : NixMultiCommand( + "flake", + { {"update", []() { return make_ref(); }}, {"lock", []() { return make_ref(); }}, {"metadata", []() { return make_ref(); }}, @@ -1429,10 +1431,8 @@ struct CmdFlake : NixMultiCommand void run() override { - if (!command) - throw UsageError("'nix flake' requires a sub-command."); experimentalFeatureSettings.require(Xp::Flakes); - command->second->run(); + NixMultiCommand::run(); } }; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index d6595dcca..ededf6ef2 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -130,7 +130,9 @@ struct CmdToBase : Command struct CmdHash : NixMultiCommand { CmdHash() - : MultiCommand({ + : NixMultiCommand( + "hash", + { {"file", []() { return make_ref(FileIngestionMethod::Flat);; }}, {"path", []() { return make_ref(FileIngestionMethod::Recursive); }}, {"to-base16", []() { return make_ref(HashFormat::Base16); }}, @@ -146,13 +148,6 @@ struct CmdHash : NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix hash' requires a sub-command."); - command->second->run(); - } }; static auto rCmdHash = registerCommand("hash"); diff --git a/src/nix/main.cc b/src/nix/main.cc index 3d44e4a9d..109d2cc04 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -67,7 +67,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs bool helpRequested = false; bool showVersion = false; - NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix") + NixArgs() : MultiCommand("", RegisterCommand::getCommandsFor({})), MixCommonArgs("nix") { categories.clear(); categories[catHelp] = "Help commands"; diff --git a/src/nix/nar.cc b/src/nix/nar.cc index 9815410cf..8ad4f92a7 100644 --- a/src/nix/nar.cc +++ b/src/nix/nar.cc @@ -4,7 +4,7 @@ using namespace nix; struct CmdNar : NixMultiCommand { - CmdNar() : MultiCommand(RegisterCommand::getCommandsFor({"nar"})) + CmdNar() : NixMultiCommand("nar", RegisterCommand::getCommandsFor({"nar"})) { } std::string description() override @@ -20,13 +20,6 @@ struct CmdNar : NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix nar' requires a sub-command."); - command->second->run(); - } }; static auto rCmdNar = registerCommand("nar"); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 476ddcd60..147b4680b 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -825,7 +825,9 @@ struct CmdProfileWipeHistory : virtual StoreCommand, MixDefaultProfile, MixDryRu struct CmdProfile : NixMultiCommand { CmdProfile() - : MultiCommand({ + : NixMultiCommand( + "profile", + { {"install", []() { return make_ref(); }}, {"remove", []() { return make_ref(); }}, {"upgrade", []() { return make_ref(); }}, @@ -848,13 +850,6 @@ struct CmdProfile : NixMultiCommand #include "profile.md" ; } - - void run() override - { - if (!command) - throw UsageError("'nix profile' requires a sub-command."); - command->second->run(); - } }; static auto rCmdProfile = registerCommand("profile"); diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc index e19e93219..e1f231222 100644 --- a/src/nix/realisation.cc +++ b/src/nix/realisation.cc @@ -5,9 +5,9 @@ using namespace nix; -struct CmdRealisation : virtual NixMultiCommand +struct CmdRealisation : NixMultiCommand { - CmdRealisation() : MultiCommand(RegisterCommand::getCommandsFor({"realisation"})) + CmdRealisation() : NixMultiCommand("realisation", RegisterCommand::getCommandsFor({"realisation"})) { } std::string description() override @@ -16,13 +16,6 @@ struct CmdRealisation : virtual NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix realisation' requires a sub-command."); - command->second->run(); - } }; static auto rCmdRealisation = registerCommand("realisation"); diff --git a/src/nix/registry.cc b/src/nix/registry.cc index f509ccae8..0346ec1e0 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -196,10 +196,12 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand } }; -struct CmdRegistry : virtual NixMultiCommand +struct CmdRegistry : NixMultiCommand { CmdRegistry() - : MultiCommand({ + : NixMultiCommand( + "registry", + { {"list", []() { return make_ref(); }}, {"add", []() { return make_ref(); }}, {"remove", []() { return make_ref(); }}, @@ -221,14 +223,6 @@ struct CmdRegistry : virtual NixMultiCommand } Category category() override { return catSecondary; } - - void run() override - { - experimentalFeatureSettings.require(Xp::Flakes); - if (!command) - throw UsageError("'nix registry' requires a sub-command."); - command->second->run(); - } }; static auto rCmdRegistry = registerCommand("registry"); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 39555c9ea..a57a407e6 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -205,7 +205,9 @@ struct CmdKeyConvertSecretToPublic : Command struct CmdKey : NixMultiCommand { CmdKey() - : MultiCommand({ + : NixMultiCommand( + "key", + { {"generate-secret", []() { return make_ref(); }}, {"convert-secret-to-public", []() { return make_ref(); }}, }) @@ -218,13 +220,6 @@ struct CmdKey : NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix key' requires a sub-command."); - command->second->run(); - } }; static auto rCmdKey = registerCommand("key"); diff --git a/src/nix/store.cc b/src/nix/store.cc index 2879e03b3..79b41e096 100644 --- a/src/nix/store.cc +++ b/src/nix/store.cc @@ -2,9 +2,9 @@ using namespace nix; -struct CmdStore : virtual NixMultiCommand +struct CmdStore : NixMultiCommand { - CmdStore() : MultiCommand(RegisterCommand::getCommandsFor({"store"})) + CmdStore() : NixMultiCommand("store", RegisterCommand::getCommandsFor({"store"})) { } std::string description() override @@ -13,13 +13,6 @@ struct CmdStore : virtual NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix store' requires a sub-command."); - command->second->run(); - } }; static auto rCmdStore = registerCommand("store"); From e7abf60a0c8db19927e4fb195789b698c84e8d5a Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Sat, 25 Nov 2023 17:33:44 +0100 Subject: [PATCH 758/909] hash.cc/hash.h: Minor C++ improvements --- src/libutil/hash.hh | 14 +++++++------- src/nix/hash.cc | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 6ade6555c..0e5c91b79 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -52,7 +52,7 @@ struct Hash /** * Create a zero-filled hash object. */ - Hash(HashType type); + explicit Hash(HashType type); /** * Parse the hash from a string representation in the format @@ -103,7 +103,7 @@ public: /** * Returns the length of a base-16 representation of this hash. */ - size_t base16Len() const + [[nodiscard]] size_t base16Len() const { return hashSize * 2; } @@ -111,7 +111,7 @@ public: /** * Returns the length of a base-32 representation of this hash. */ - size_t base32Len() const + [[nodiscard]] size_t base32Len() const { return (hashSize * 8 - 1) / 5 + 1; } @@ -119,7 +119,7 @@ public: /** * Returns the length of a base-64 representation of this hash. */ - size_t base64Len() const + [[nodiscard]] size_t base64Len() const { return ((4 * hashSize / 3) + 3) & ~3; } @@ -129,14 +129,14 @@ public: * or base-64. By default, this is prefixed by the hash type * (e.g. "sha256:"). */ - std::string to_string(HashFormat hashFormat, bool includeType) const; + [[nodiscard]] std::string to_string(HashFormat hashFormat, bool includeType) const; - std::string gitRev() const + [[nodiscard]] std::string gitRev() const { return to_string(HashFormat::Base16, false); } - std::string gitShortRev() const + [[nodiscard]] std::string gitShortRev() const { return std::string(to_string(HashFormat::Base16, false), 0, 7); } diff --git a/src/nix/hash.cc b/src/nix/hash.cc index ededf6ef2..cac65006b 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -17,7 +17,7 @@ struct CmdHashBase : Command std::vector paths; std::optional modulus; - CmdHashBase(FileIngestionMethod mode) : mode(mode) + explicit CmdHashBase(FileIngestionMethod mode) : mode(mode) { addFlag({ .longName = "sri", From 156ea78d7402368e3816855800eb6e0ed33a1ecc Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Sat, 25 Nov 2023 17:34:16 +0100 Subject: [PATCH 759/909] CmdHashBase: doc comment --- src/nix/hash.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/nix/hash.cc b/src/nix/hash.cc index cac65006b..dfef44221 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -8,6 +8,11 @@ using namespace nix; +/** + * Base for `nix hash file` (deprecated), `nix hash path` and `nix-hash` (legacy). + * + * Deprecation Issue: https://github.com/NixOS/nix/issues/8876 + */ struct CmdHashBase : Command { FileIngestionMethod mode; From 6bbd900d4f9983f74dcd9a0f85ab899331f661c7 Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Sat, 25 Nov 2023 17:35:24 +0100 Subject: [PATCH 760/909] nix hash convert: added This deviated from the proposal! See comments on the issue. https://github.com/NixOS/nix/issues/8876 --- src/nix/hash.cc | 63 ++++++++++++++++++++++++++++++++++++++++ tests/functional/hash.sh | 19 +++++++++++- 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/src/nix/hash.cc b/src/nix/hash.cc index dfef44221..2b32ac03c 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -132,12 +132,75 @@ struct CmdToBase : Command } }; +/** + * `nix hash convert` + */ +struct CmdHashConvert : Command +{ + std::optional from; + HashFormat to; + std::optional type; + std::vector hashStrings; + + CmdHashConvert(): to(HashFormat::SRI) { + addFlag({ + .longName = "from", + // TODO: List format choices. Maybe introduce a constant? + .description = "The format of the input hash.", + .labels = {"hash format"}, + .handler = {[this](std::string str) { + from = parseHashFormat(str); + }}, + }); + addFlag({ + .longName = "to", + // TODO: List format choices. Maybe introduce a constant? + .description = "The format of the output hash.", + .labels = {"hash format"}, + .handler = {[this](std::string str) { + to = parseHashFormat(str); + }}, + }); + addFlag({ + .longName = "type", + .description = "Specify the type if it can't be auto-detected.", + .labels = {"hash type"}, + .handler = {[this](std::string str) { + type = parseHashType(str); + }}, + }); + expectArgs({ + .label = "hashes", + .handler = {&hashStrings}, + }); + } + + std::string description() override + { + return "convert between different hash formats, e.g. base16 and sri."; + } + + Category category() override { return catUtility; } + + void run() override { + for (const auto& s: hashStrings) { + Hash h = Hash::parseAny(s, type); + if (from && h.to_string(*from, from == HashFormat::SRI) != s) { + auto from_as_string = printHashFormat(*from); + throw BadHash("input hash '%s' does not have the expected format '--from %s'", s, from_as_string); + } + logger->cout(h.to_string(to, to == HashFormat::SRI)); + } + } +}; + struct CmdHash : NixMultiCommand { CmdHash() : NixMultiCommand( "hash", { + {"convert", []() { return make_ref();}}, {"file", []() { return make_ref(FileIngestionMethod::Flat);; }}, {"path", []() { return make_ref(FileIngestionMethod::Recursive); }}, {"to-base16", []() { return make_ref(HashFormat::Base16); }}, diff --git a/tests/functional/hash.sh b/tests/functional/hash.sh index 34c1bb38a..d66b27a26 100644 --- a/tests/functional/hash.sh +++ b/tests/functional/hash.sh @@ -81,24 +81,41 @@ rm $TEST_ROOT/hash-path/hello ln -s x $TEST_ROOT/hash-path/hello try2 md5 "f78b733a68f5edbdf9413899339eaa4a" -# Conversion. +# Conversion with `nix hash` `nix-hash` and `nix hash convert` try3() { + # $1 = hash type + # $2 = expected hash in base16 + # $3 = expected hash in base32 + # $4 = expected hash in base64 + h64=$(nix hash convert --type "$1" --to base64 "$2") + [ "$h64" = "$4" ] h64=$(nix-hash --type "$1" --to-base64 "$2") [ "$h64" = "$4" ] + # Deprecated experiment h64=$(nix hash to-base64 --type "$1" "$2") [ "$h64" = "$4" ] + + sri=$(nix hash convert --type "$1" --to sri "$2") + [ "$sri" = "$1-$4" ] sri=$(nix-hash --type "$1" --to-sri "$2") [ "$sri" = "$1-$4" ] sri=$(nix hash to-sri --type "$1" "$2") [ "$sri" = "$1-$4" ] + h32=$(nix hash convert --type "$1" --to base32 "$2") + [ "$h32" = "$3" ] h32=$(nix-hash --type "$1" --to-base32 "$2") [ "$h32" = "$3" ] h32=$(nix hash to-base32 --type "$1" "$2") [ "$h32" = "$3" ] h16=$(nix-hash --type "$1" --to-base16 "$h32") [ "$h16" = "$2" ] + + h16=$(nix hash convert --type "$1" --to base16 "$h64") + [ "$h16" = "$2" ] h16=$(nix hash to-base16 --type "$1" "$h64") [ "$h16" = "$2" ] + h16=$(nix hash convert --to base16 "$sri") + [ "$h16" = "$2" ] h16=$(nix hash to-base16 "$sri") [ "$h16" = "$2" ] } From 0c2d5f7673ae0196b660c39b59941755103c23d0 Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Tue, 28 Nov 2023 11:42:52 +0100 Subject: [PATCH 761/909] nix hash convert: s/--type/--algo/ + more functional tests https://github.com/NixOS/nix/issues/8876 --- src/libutil/hash.hh | 1 - src/nix/hash.cc | 8 ++--- tests/functional/hash.sh | 72 +++++++++++++++++++++++++++++++++++++--- 3 files changed, 71 insertions(+), 10 deletions(-) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 0e5c91b79..820154e7a 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -40,7 +40,6 @@ enum struct HashFormat : int { SRI }; - struct Hash { constexpr static size_t maxHashSize = 64; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 2b32ac03c..62f96ef1d 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -162,9 +162,9 @@ struct CmdHashConvert : Command }}, }); addFlag({ - .longName = "type", - .description = "Specify the type if it can't be auto-detected.", - .labels = {"hash type"}, + .longName = "algo", + .description = "Specify the algorithm if it can't be auto-detected.", + .labels = {"hash algorithm"}, .handler = {[this](std::string str) { type = parseHashType(str); }}, @@ -177,7 +177,7 @@ struct CmdHashConvert : Command std::string description() override { - return "convert between different hash formats, e.g. base16 and sri."; + return "convert between different hash formats, e.g. base16, nix32, base64 and sri."; } Category category() override { return catUtility; } diff --git a/tests/functional/hash.sh b/tests/functional/hash.sh index d66b27a26..031e33adf 100644 --- a/tests/functional/hash.sh +++ b/tests/functional/hash.sh @@ -83,11 +83,11 @@ try2 md5 "f78b733a68f5edbdf9413899339eaa4a" # Conversion with `nix hash` `nix-hash` and `nix hash convert` try3() { - # $1 = hash type + # $1 = hash algo # $2 = expected hash in base16 # $3 = expected hash in base32 # $4 = expected hash in base64 - h64=$(nix hash convert --type "$1" --to base64 "$2") + h64=$(nix hash convert --algo "$1" --to base64 "$2") [ "$h64" = "$4" ] h64=$(nix-hash --type "$1" --to-base64 "$2") [ "$h64" = "$4" ] @@ -95,13 +95,13 @@ try3() { h64=$(nix hash to-base64 --type "$1" "$2") [ "$h64" = "$4" ] - sri=$(nix hash convert --type "$1" --to sri "$2") + sri=$(nix hash convert --algo "$1" --to sri "$2") [ "$sri" = "$1-$4" ] sri=$(nix-hash --type "$1" --to-sri "$2") [ "$sri" = "$1-$4" ] sri=$(nix hash to-sri --type "$1" "$2") [ "$sri" = "$1-$4" ] - h32=$(nix hash convert --type "$1" --to base32 "$2") + h32=$(nix hash convert --algo "$1" --to base32 "$2") [ "$h32" = "$3" ] h32=$(nix-hash --type "$1" --to-base32 "$2") [ "$h32" = "$3" ] @@ -110,7 +110,7 @@ try3() { h16=$(nix-hash --type "$1" --to-base16 "$h32") [ "$h16" = "$2" ] - h16=$(nix hash convert --type "$1" --to base16 "$h64") + h16=$(nix hash convert --algo "$1" --to base16 "$h64") [ "$h16" = "$2" ] h16=$(nix hash to-base16 --type "$1" "$h64") [ "$h16" = "$2" ] @@ -118,7 +118,69 @@ try3() { [ "$h16" = "$2" ] h16=$(nix hash to-base16 "$sri") [ "$h16" = "$2" ] + + # + # Converting from SRI + # + + # Input hash algo auto-detected from SRI and output defaults to SRI as well. + sri=$(nix hash convert "$1-$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --from sri "$1-$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --to sri "$1-$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --from sri --to sri "$1-$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --to base64 "$1-$4") + [ "$sri" = "$4" ] + + # + # Auto-detecting the input from algo and length. + # + + sri=$(nix hash convert --algo "$1" "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --algo "$1" "$3") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --algo "$1" "$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --algo "$1" "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --algo "$1" "$3") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --algo "$1" "$4") + [ "$sri" = "$1-$4" ] + + # + # Asserting input format succeeds. + # + + sri=$(nix hash convert --algo "$1" --from base16 "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --algo "$1" --from base32 "$3") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --algo "$1" --from base64 "$4") + [ "$sri" = "$1-$4" ] + + # + # Asserting input format fails. + # + + fail=$(nix hash convert --algo "$1" --from base32 "$2" 2>&1 || echo "exit: $?") + [[ "$fail" == "error: input hash"*"exit: 1" ]] + fail=$(nix hash convert --algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?") + [[ "$fail" == "error: input hash"*"exit: 1" ]] + fail=$(nix hash convert --algo "$1" --from base32 "$4" 2>&1 || echo "exit: $?") + [[ "$fail" == "error: input hash"*"exit: 1" ]] + } + try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" From 5334c9c792a208db4d3824e88019a626ded1b65d Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Tue, 28 Nov 2023 14:20:27 +0100 Subject: [PATCH 762/909] HashType: Rename to HashAlgorithm To be consistent with CLI, nix API and many other references. As part of this, we also converted it to a scoped enum. https://github.com/NixOS/nix/issues/8876 --- perl/lib/Nix/Store.xs | 12 +- src/libexpr/flake/flake.cc | 2 +- src/libexpr/primops.cc | 36 ++--- src/libexpr/primops/fetchMercurial.cc | 4 +- src/libexpr/primops/fetchTree.cc | 8 +- src/libfetchers/fetchers.cc | 6 +- src/libfetchers/git-utils.cc | 4 +- src/libfetchers/git.cc | 6 +- src/libfetchers/github.cc | 10 +- src/libfetchers/indirect.cc | 4 +- src/libfetchers/input-accessor.cc | 4 +- src/libfetchers/mercurial.cc | 10 +- src/libfetchers/tarball.cc | 6 +- src/libstore/binary-cache-store.cc | 24 +-- src/libstore/binary-cache-store.hh | 16 +- src/libstore/build/local-derivation-goal.cc | 42 +++--- src/libstore/build/worker.cc | 4 +- src/libstore/builtins/fetchurl.cc | 4 +- src/libstore/content-address.cc | 28 ++-- src/libstore/content-address.hh | 4 +- src/libstore/daemon.cc | 16 +- src/libstore/derivations.cc | 64 ++++---- src/libstore/derivations.hh | 8 +- src/libstore/downstream-placeholder.cc | 4 +- src/libstore/export-import.cc | 6 +- src/libstore/gc.cc | 2 +- src/libstore/legacy-ssh-store.cc | 14 +- src/libstore/local-store.cc | 30 ++-- src/libstore/local-store.hh | 6 +- src/libstore/make-content-addressed.cc | 4 +- src/libstore/nar-info.cc | 4 +- src/libstore/optimise-store.cc | 4 +- src/libstore/path-references.cc | 2 +- src/libstore/path.cc | 2 +- src/libstore/remote-store.cc | 28 ++-- src/libstore/remote-store.hh | 14 +- src/libstore/store-api.cc | 46 +++--- src/libstore/store-api.hh | 22 +-- src/libstore/store-dir-config.hh | 2 +- src/libstore/worker-protocol.cc | 2 +- src/libutil/args.cc | 14 +- src/libutil/args.hh | 6 +- src/libutil/git.cc | 8 +- src/libutil/git.hh | 6 +- src/libutil/hash.cc | 158 ++++++++++---------- src/libutil/hash.hh | 38 ++--- src/libutil/references.cc | 4 +- src/libutil/references.hh | 2 +- src/libutil/source-accessor.cc | 8 +- src/libutil/source-accessor.hh | 6 +- src/nix-store/nix-store.cc | 12 +- src/nix/add-to-store.cc | 4 +- src/nix/hash.cc | 24 +-- src/nix/prefetch.cc | 34 ++--- src/nix/profile.cc | 2 +- src/nix/verify.cc | 2 +- tests/unit/libstore/common-protocol.cc | 8 +- tests/unit/libstore/derivation.cc | 4 +- tests/unit/libstore/nar-info.cc | 2 +- tests/unit/libstore/path-info.cc | 2 +- tests/unit/libstore/serve-protocol.cc | 8 +- tests/unit/libstore/worker-protocol.cc | 10 +- tests/unit/libutil/git.cc | 8 +- tests/unit/libutil/hash.cc | 16 +- 64 files changed, 450 insertions(+), 450 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 40257ed74..50148141b 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -205,7 +205,7 @@ void importPaths(int fd, int dontCheckSigs) SV * hashPath(char * algo, int base32, char * path) PPCODE: try { - Hash h = hashPath(parseHashType(algo), path).first; + Hash h = hashPath(parseHashAlgo(algo), path).first; auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { @@ -216,7 +216,7 @@ SV * hashPath(char * algo, int base32, char * path) SV * hashFile(char * algo, int base32, char * path) PPCODE: try { - Hash h = hashFile(parseHashType(algo), path); + Hash h = hashFile(parseHashAlgo(algo), path); auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { @@ -227,7 +227,7 @@ SV * hashFile(char * algo, int base32, char * path) SV * hashString(char * algo, int base32, char * s) PPCODE: try { - Hash h = hashString(parseHashType(algo), s); + Hash h = hashString(parseHashAlgo(algo), s); auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { @@ -238,7 +238,7 @@ SV * hashString(char * algo, int base32, char * s) SV * convertHash(char * algo, char * s, int toBase32) PPCODE: try { - auto h = Hash::parseAny(s, parseHashType(algo)); + auto h = Hash::parseAny(s, parseHashAlgo(algo)); auto s = h.to_string(toBase32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { @@ -281,7 +281,7 @@ SV * addToStore(char * srcPath, int recursive, char * algo) PPCODE: try { auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashType(algo)); + auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashAlgo(algo)); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -291,7 +291,7 @@ SV * addToStore(char * srcPath, int recursive, char * algo) SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) PPCODE: try { - auto h = Hash::parseAny(hash, parseHashType(algo)); + auto h = Hash::parseAny(hash, parseHashAlgo(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { .method = method, diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 8f8fc64f0..fee58792b 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -904,7 +904,7 @@ Fingerprint LockedFlake::getFingerprint() const // FIXME: as an optimization, if the flake contains a lock file // and we haven't changed it, then it's sufficient to use // flake.sourceInfo.storePath for the fingerprint. - return hashString(htSHA256, + return hashString(HashAlgorithm::SHA256, fmt("%s;%s;%d;%d;%s", flake.storePath.to_string(), flake.lockedRef.subdir, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c2499bdae..7831f3803 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1317,7 +1317,7 @@ drvName, Bindings * attrs, Value & v) .errPos = state.positions[noPos] })); - auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo)); + auto h = newHashAllowEmpty(*outputHash, parseHashAlgoOpt(outputHashAlgo)); auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); @@ -1339,7 +1339,7 @@ drvName, Bindings * attrs, Value & v) .errPos = state.positions[noPos] }); - auto ht = parseHashTypeOpt(outputHashAlgo).value_or(htSHA256); + auto ht = parseHashAlgoOpt(outputHashAlgo).value_or(HashAlgorithm::SHA256); auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive); for (auto & i : outputs) { @@ -1348,13 +1348,13 @@ drvName, Bindings * attrs, Value & v) drv.outputs.insert_or_assign(i, DerivationOutput::Impure { .method = method, - .hashType = ht, + .hashAlgo = ht, }); else drv.outputs.insert_or_assign(i, DerivationOutput::CAFloating { .method = method, - .hashType = ht, + .hashAlgo = ht, }); } } @@ -1754,17 +1754,17 @@ static RegisterPrimOp primop_findFile(PrimOp { /* Return the cryptographic hash of a file in base-16. */ static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto type = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashFile"); - std::optional ht = parseHashType(type); - if (!ht) + auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashFile"); + std::optional ha = parseHashAlgo(algo); + if (!ha) state.debugThrowLastTrace(Error({ - .msg = hintfmt("unknown hash type '%1%'", type), + .msg = hintfmt("unknown hash algo '%1%'", algo), .errPos = state.positions[pos] })); auto path = realisePath(state, pos, *args[1]); - v.mkString(hashString(*ht, path.readFile()).to_string(HashFormat::Base16, false)); + v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false)); } static RegisterPrimOp primop_hashFile({ @@ -2341,7 +2341,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value else if (n == "recursive") method = FileIngestionMethod { state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") }; else if (n == "sha256") - expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), htSHA256); + expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256); else state.debugThrowLastTrace(EvalError({ .msg = hintfmt("unsupported argument '%1%' to 'addPath'", state.symbols[attr.name]), @@ -3766,18 +3766,18 @@ static RegisterPrimOp primop_stringLength({ /* Return the cryptographic hash of a string in base-16. */ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto type = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashString"); - std::optional ht = parseHashType(type); - if (!ht) + auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashString"); + std::optional ha = parseHashAlgo(algo); + if (!ha) state.debugThrowLastTrace(Error({ - .msg = hintfmt("unknown hash type '%1%'", type), + .msg = hintfmt("unknown hash algo '%1%'", algo), .errPos = state.positions[pos] })); NixStringContext context; // discarded auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); - v.mkString(hashString(*ht, s).to_string(HashFormat::Base16, false)); + v.mkString(hashString(*ha, s).to_string(HashFormat::Base16, false)); } static RegisterPrimOp primop_hashString({ @@ -3800,15 +3800,15 @@ static void prim_convertHash(EvalState & state, const PosIdx pos, Value * * args auto hash = state.forceStringNoCtx(*iteratorHash->value, pos, "while evaluating the attribute 'hash'"); Bindings::iterator iteratorHashAlgo = inputAttrs->find(state.symbols.create("hashAlgo")); - std::optional ht = std::nullopt; + std::optional ha = std::nullopt; if (iteratorHashAlgo != inputAttrs->end()) { - ht = parseHashType(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'")); + ha = parseHashAlgo(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'")); } Bindings::iterator iteratorToHashFormat = getAttr(state, state.symbols.create("toHashFormat"), args[0]->attrs, "while locating the attribute 'toHashFormat'"); HashFormat hf = parseHashFormat(state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'")); - v.mkString(Hash::parseAny(hash, ht).to_string(hf, hf == HashFormat::SRI)); + v.mkString(Hash::parseAny(hash, ha).to_string(hf, hf == HashFormat::SRI)); } static RegisterPrimOp primop_convertHash({ diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index e76ce455d..58fe6f173 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -31,7 +31,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a // be both a revision or a branch/tag name. auto value = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `rev` attribute passed to builtins.fetchMercurial"); if (std::regex_match(value.begin(), value.end(), revRegex)) - rev = Hash::parseAny(value, htSHA1); + rev = Hash::parseAny(value, HashAlgorithm::SHA1); else ref = value; } @@ -79,7 +79,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a attrs2.alloc("branch").mkString(*input2.getRef()); // Backward compatibility: set 'rev' to // 0000000000000000000000000000000000000000 for a dirty tree. - auto rev2 = input2.getRev().value_or(Hash(htSHA1)); + auto rev2 = input2.getRev().value_or(Hash(HashAlgorithm::SHA1)); attrs2.alloc("rev").mkString(rev2.gitRev()); attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12)); if (auto revCount = input2.getRevCount()) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 383ec7c58..ef80c634f 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -46,7 +46,7 @@ void emitTreeAttrs( attrs.alloc("shortRev").mkString(rev->gitShortRev()); } else if (emptyRevFallback) { // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev - auto emptyHash = Hash(htSHA1); + auto emptyHash = Hash(HashAlgorithm::SHA1); attrs.alloc("rev").mkString(emptyHash.gitRev()); attrs.alloc("shortRev").mkString(emptyHash.gitShortRev()); } @@ -246,7 +246,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v if (n == "url") url = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the url we should fetch"); else if (n == "sha256") - expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), htSHA256); + expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), HashAlgorithm::SHA256); else if (n == "name") name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch"); else @@ -276,7 +276,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v state.debugThrowLastTrace(EvalError("in pure evaluation mode, '%s' requires a 'sha256' argument", who)); // early exit if pinned and already in the store - if (expectedHash && expectedHash->type == htSHA256) { + if (expectedHash && expectedHash->algo == HashAlgorithm::SHA256) { auto expectedPath = state.store->makeFixedOutputPath( name, FixedOutputInfo { @@ -301,7 +301,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v if (expectedHash) { auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash - : hashFile(htSHA256, state.store->toRealPath(storePath)); + : hashFile(HashAlgorithm::SHA256, state.store->toRealPath(storePath)); if (hash != *expectedHash) state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", *url, expectedHash->to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true))); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 60208619e..573341a3d 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -289,8 +289,8 @@ std::string Input::getType() const std::optional Input::getNarHash() const { if (auto s = maybeGetStrAttr(attrs, "narHash")) { - auto hash = s->empty() ? Hash(htSHA256) : Hash::parseSRI(*s); - if (hash.type != htSHA256) + auto hash = s->empty() ? Hash(HashAlgorithm::SHA256) : Hash::parseSRI(*s); + if (hash.algo != HashAlgorithm::SHA256) throw UsageError("narHash must use SHA-256"); return hash; } @@ -314,7 +314,7 @@ std::optional Input::getRev() const } catch (BadHash &e) { // Default to sha1 for backwards compatibility with existing // usages (e.g. `builtins.fetchTree` calls or flake inputs). - hash = Hash::parseAny(*s, htSHA1); + hash = Hash::parseAny(*s, HashAlgorithm::SHA1); } } diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 19eae0e1d..9356e5817 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -91,7 +91,7 @@ Hash toHash(const git_oid & oid) #ifdef GIT_EXPERIMENTAL_SHA256 assert(oid.type == GIT_OID_SHA1); #endif - Hash hash(htSHA1); + Hash hash(HashAlgorithm::SHA1); memcpy(hash.hash, oid.id, hash.hashSize); return hash; } @@ -439,7 +439,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this std::string re = R"(Good "git" signature for \* with .* key SHA256:[)"; for (const fetchers::PublicKey & k : publicKeys){ // Calculate sha256 fingerprint from public key and escape the regex symbol '+' to match the key literally - auto fingerprint = trim(hashString(htSHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "="); + auto fingerprint = trim(hashString(HashAlgorithm::SHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "="); auto escaped_fingerprint = std::regex_replace(fingerprint, std::regex("\\+"), "\\+" ); re += "(" + escaped_fingerprint + ")"; } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 8cd74057c..a89acc1c0 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -52,7 +52,7 @@ bool touchCacheFile(const Path & path, time_t touch_time) Path getCachePath(std::string_view key) { return getCacheDir() + "/nix/gitv3/" + - hashString(htSHA256, key).to_string(HashFormat::Base32, false); + hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Base32, false); } // Returns the name of the HEAD branch. @@ -369,7 +369,7 @@ struct GitInputScheme : InputScheme { auto checkHashType = [&](const std::optional & hash) { - if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256)) + if (hash.has_value() && !(hash->algo == HashAlgorithm::SHA1 || hash->algo == HashAlgorithm::SHA256)) throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true)); }; @@ -559,7 +559,7 @@ struct GitInputScheme : InputScheme repoInfo.url ); } else - input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev()); + input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), HashAlgorithm::SHA1).gitRev()); // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 661ad4884..70acb9354 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -42,7 +42,7 @@ struct GitArchiveInputScheme : InputScheme auto size = path.size(); if (size == 3) { if (std::regex_match(path[2], revRegex)) - rev = Hash::parseAny(path[2], htSHA1); + rev = Hash::parseAny(path[2], HashAlgorithm::SHA1); else if (std::regex_match(path[2], refRegex)) ref = path[2]; else @@ -68,7 +68,7 @@ struct GitArchiveInputScheme : InputScheme if (name == "rev") { if (rev) throw BadURL("URL '%s' contains multiple commit hashes", url.url); - rev = Hash::parseAny(value, htSHA1); + rev = Hash::parseAny(value, HashAlgorithm::SHA1); } else if (name == "ref") { if (!std::regex_match(value, refRegex)) @@ -284,7 +284,7 @@ struct GitHubInputScheme : GitArchiveInputScheme readFile( store->toRealPath( downloadFile(store, url, "source", false, headers).storePath))); - auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1); + auto rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1); debug("HEAD revision for '%s' is %s", url, rev.gitRev()); return rev; } @@ -356,7 +356,7 @@ struct GitLabInputScheme : GitArchiveInputScheme readFile( store->toRealPath( downloadFile(store, url, "source", false, headers).storePath))); - auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1); + auto rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1); debug("HEAD revision for '%s' is %s", url, rev.gitRev()); return rev; } @@ -448,7 +448,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme if(!id) throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref); - auto rev = Hash::parseAny(*id, htSHA1); + auto rev = Hash::parseAny(*id, HashAlgorithm::SHA1); debug("HEAD revision for '%s' is %s", fmt("%s/%s", base_url, ref), rev.gitRev()); return rev; } diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 8e30284c6..002c0c292 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -20,7 +20,7 @@ struct IndirectInputScheme : InputScheme if (path.size() == 1) { } else if (path.size() == 2) { if (std::regex_match(path[1], revRegex)) - rev = Hash::parseAny(path[1], htSHA1); + rev = Hash::parseAny(path[1], HashAlgorithm::SHA1); else if (std::regex_match(path[1], refRegex)) ref = path[1]; else @@ -31,7 +31,7 @@ struct IndirectInputScheme : InputScheme ref = path[1]; if (!std::regex_match(path[2], revRegex)) throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]); - rev = Hash::parseAny(path[2], htSHA1); + rev = Hash::parseAny(path[2], HashAlgorithm::SHA1); } else throw BadURL("GitHub URL '%s' is invalid", url.url); diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index 85dc4609f..eabef55d8 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -44,8 +44,8 @@ StorePath InputAccessor::fetchToStore( auto storePath = settings.readOnlyMode - ? store->computeStorePathFromDump(*source, name, method, htSHA256).first - : store->addToStoreFromDump(*source, name, method, htSHA256, repair); + ? store->computeStorePathFromDump(*source, name, method, HashAlgorithm::SHA256).first + : store->addToStoreFromDump(*source, name, method, HashAlgorithm::SHA256, repair); if (cacheKey) fetchers::getCache()->add(store, *cacheKey, {}, storePath, true); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index aa991a75d..713f24bbb 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -210,7 +210,7 @@ struct MercurialInputScheme : InputScheme return files.count(file); }; - auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, filter); return {std::move(storePath), input}; } @@ -220,7 +220,7 @@ struct MercurialInputScheme : InputScheme auto checkHashType = [&](const std::optional & hash) { - if (hash.has_value() && hash->type != htSHA1) + if (hash.has_value() && hash->algo != HashAlgorithm::SHA1) throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(HashFormat::Base16, true)); }; @@ -260,14 +260,14 @@ struct MercurialInputScheme : InputScheme }); if (auto res = getCache()->lookup(store, unlockedAttrs)) { - auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); + auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), HashAlgorithm::SHA1); if (!input.getRev() || input.getRev() == rev2) { input.attrs.insert_or_assign("rev", rev2.gitRev()); return makeResult(res->first, std::move(res->second)); } } - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(HashFormat::Base32, false)); + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(HashAlgorithm::SHA256, actualUrl).to_string(HashFormat::Base32, false)); /* If this is a commit hash that we already have, we don't have to pull again. */ @@ -301,7 +301,7 @@ struct MercurialInputScheme : InputScheme runHg({ "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); assert(tokens.size() == 3); - input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev()); + input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], HashAlgorithm::SHA1).gitRev()); auto revCount = std::stoull(tokens[1]); input.attrs.insert_or_assign("ref", tokens[2]); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 0062878a9..086366180 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -73,7 +73,7 @@ DownloadFileResult downloadFile( } else { StringSink sink; dumpString(res.data, sink); - auto hash = hashString(htSHA256, res.data); + auto hash = hashString(HashAlgorithm::SHA256, res.data); ValidPathInfo info { *store, name, @@ -82,7 +82,7 @@ DownloadFileResult downloadFile( .hash = hash, .references = {}, }, - hashString(htSHA256, sink.s), + hashString(HashAlgorithm::SHA256, sink.s), }; info.narSize = sink.s.size(); auto source = StringSource { sink.s }; @@ -156,7 +156,7 @@ DownloadTarballResult downloadTarball( throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); auto topDir = tmpDir + "/" + members.begin()->name; lastModified = lstat(topDir).st_mtime; - unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair); + unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, defaultPathFilter, NoRepair); } Attrs infoAttrs({ diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index ae483c95e..f287d72a8 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -143,9 +143,9 @@ ref BinaryCacheStore::addToStoreCommon( /* Read the NAR simultaneously into a CompressionSink+FileSink (to write the compressed NAR to disk), into a HashSink (to get the NAR hash), and into a NarAccessor (to get the NAR listing). */ - HashSink fileHashSink { htSHA256 }; + HashSink fileHashSink { HashAlgorithm::SHA256 }; std::shared_ptr narAccessor; - HashSink narHashSink { htSHA256 }; + HashSink narHashSink { HashAlgorithm::SHA256 }; { FdSink fileSink(fdTemp.get()); TeeSink teeSinkCompressed { fileSink, fileHashSink }; @@ -301,9 +301,9 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource } StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) + FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) { - if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) + if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) unsupported("addToStoreFromDump"); return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { @@ -399,13 +399,13 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, } StorePath BinaryCacheStore::addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashType hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) + std::string_view name, + const Path & srcPath, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) { /* FIXME: Make BinaryCacheStore::addToStoreCommon support non-recursive+sha256 so we can just use the default @@ -448,7 +448,7 @@ StorePath BinaryCacheStore::addTextToStore( const StorePathSet & references, RepairFlag repair) { - auto textHash = hashString(htSHA256, s); + auto textHash = hashString(HashAlgorithm::SHA256, s); auto path = makeTextPath(name, TextInfo { { textHash }, references }); if (!repair && isValidPath(path)) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index cea2a571f..395e1b479 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -124,16 +124,16 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; + FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) override; StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashType hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override; + std::string_view name, + const Path & srcPath, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) override; StorePath addTextToStore( std::string_view name, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 198402ff7..4c3dc1f5c 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1066,7 +1066,7 @@ void LocalDerivationGoal::initTmpDir() { if (passAsFile.find(i.first) == passAsFile.end()) { env[i.first] = i.second; } else { - auto hash = hashString(htSHA256, i.first); + auto hash = hashString(HashAlgorithm::SHA256, i.first); std::string fn = ".attr-" + hash.to_string(HashFormat::Base32, false); Path p = tmpDir + "/" + fn; writeFile(p, rewriteStrings(i.second, inputRewrites)); @@ -1290,13 +1290,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In { throw Error("queryPathFromHashPart"); } StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashType hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override + std::string_view name, + const Path & srcPath, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) override { throw Error("addToStore"); } void addToStore(const ValidPathInfo & info, Source & narSource, @@ -1318,12 +1318,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In } StorePath addToStoreFromDump( - Source & dump, - std::string_view name, - FileIngestionMethod method, - HashType hashAlgo, - RepairFlag repair, - const StorePathSet & references) override + Source & dump, + std::string_view name, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + RepairFlag repair, + const StorePathSet & references) override { auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references); goal.addDependency(path); @@ -2466,7 +2466,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() rewriteOutput(outputRewrites); /* FIXME optimize and deduplicate with addToStore */ std::string oldHashPart { scratchPath->hashPart() }; - HashModuloSink caSink { outputHash.hashType, oldHashPart }; + HashModuloSink caSink {outputHash.hashAlgo, oldHashPart }; std::visit(overloaded { [&](const TextIngestionMethod &) { readFile(actualPath, caSink); @@ -2511,7 +2511,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() std::string(newInfo0.path.hashPart())}}); } - HashResult narHashAndSize = hashPath(htSHA256, actualPath); + HashResult narHashAndSize = hashPath(HashAlgorithm::SHA256, actualPath); newInfo0.narHash = narHashAndSize.first; newInfo0.narSize = narHashAndSize.second; @@ -2531,7 +2531,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() std::string { scratchPath->hashPart() }, std::string { requiredFinalPath.hashPart() }); rewriteOutput(outputRewrites); - auto narHashAndSize = hashPath(htSHA256, actualPath); + auto narHashAndSize = hashPath(HashAlgorithm::SHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; auto refs = rewriteRefs(); @@ -2546,7 +2546,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { .method = dof.ca.method, - .hashType = wanted.type, + .hashAlgo = wanted.algo, }); /* Check wanted hash */ @@ -2583,7 +2583,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() [&](const DerivationOutput::Impure & doi) { return newInfoFromCA(DerivationOutput::CAFloating { .method = doi.method, - .hashType = doi.hashType, + .hashAlgo = doi.hashAlgo, }); }, @@ -2945,7 +2945,7 @@ StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName) { return worker.store.makeStorePath( "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), - Hash(htSHA256), outputPathName(drv->name, outputName)); + Hash(HashAlgorithm::SHA256), outputPathName(drv->name, outputName)); } @@ -2953,7 +2953,7 @@ StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path) { return worker.store.makeStorePath( "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), - Hash(htSHA256), path.name()); + Hash(HashAlgorithm::SHA256), path.name()); } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 01f52e7ab..9b8c36286 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -519,8 +519,8 @@ bool Worker::pathContentsGood(const StorePath & path) if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); - Hash nullHash(htSHA256); + HashResult current = hashPath(info->narHash.algo, store.printStorePath(path)); + Hash nullHash(HashAlgorithm::SHA256); res = info->narHash == nullHash || info->narHash == current.first; } pathContentsGoodCache.insert_or_assign(path, res); diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 357800333..2086bd0b9 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -63,9 +63,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) for (auto hashedMirror : settings.hashedMirrors.get()) try { if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; - std::optional ht = parseHashTypeOpt(getAttr("outputHashAlgo")); + std::optional ht = parseHashAlgoOpt(getAttr("outputHashAlgo")); Hash h = newHashAllowEmpty(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(HashFormat::Base16, false)); + fetch(hashedMirror + printHashAlgo(h.algo) + "/" + h.to_string(HashFormat::Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a5f7cdf81..de8194f73 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -38,14 +38,14 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) return FileIngestionMethod::Flat; } -std::string ContentAddressMethod::render(HashType ht) const +std::string ContentAddressMethod::render(HashAlgorithm ha) const { return std::visit(overloaded { [&](const TextIngestionMethod & th) { - return std::string{"text:"} + printHashType(ht); + return std::string{"text:"} + printHashAlgo(ha); }, [&](const FileIngestionMethod & fim) { - return "fixed:" + makeFileIngestionPrefix(fim) + printHashType(ht); + return "fixed:" + makeFileIngestionPrefix(fim) + printHashAlgo(ha); } }, raw); } @@ -67,7 +67,7 @@ std::string ContentAddress::render() const /** * Parses content address strings up to the hash. */ -static std::pair parseContentAddressMethodPrefix(std::string_view & rest) +static std::pair parseContentAddressMethodPrefix(std::string_view & rest) { std::string_view wholeInput { rest }; @@ -83,27 +83,27 @@ static std::pair parseContentAddressMethodPrefix auto hashTypeRaw = splitPrefixTo(rest, ':'); if (!hashTypeRaw) throw UsageError("content address hash must be in form ':', but found: %s", wholeInput); - HashType hashType = parseHashType(*hashTypeRaw); - return hashType; + HashAlgorithm hashAlgo = parseHashAlgo(*hashTypeRaw); + return hashAlgo; }; // Switch on prefix if (prefix == "text") { // No parsing of the ingestion method, "text" only support flat. - HashType hashType = parseHashType_(); + HashAlgorithm hashAlgo = parseHashType_(); return { TextIngestionMethod {}, - std::move(hashType), + std::move(hashAlgo), }; } else if (prefix == "fixed") { // Parse method auto method = FileIngestionMethod::Flat; if (splitPrefix(rest, "r:")) method = FileIngestionMethod::Recursive; - HashType hashType = parseHashType_(); + HashAlgorithm hashAlgo = parseHashType_(); return { std::move(method), - std::move(hashType), + std::move(hashAlgo), }; } else throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); @@ -113,15 +113,15 @@ ContentAddress ContentAddress::parse(std::string_view rawCa) { auto rest = rawCa; - auto [caMethod, hashType] = parseContentAddressMethodPrefix(rest); + auto [caMethod, hashAlgo] = parseContentAddressMethodPrefix(rest); return ContentAddress { .method = std::move(caMethod), - .hash = Hash::parseNonSRIUnprefixed(rest, hashType), + .hash = Hash::parseNonSRIUnprefixed(rest, hashAlgo), }; } -std::pair ContentAddressMethod::parse(std::string_view caMethod) +std::pair ContentAddressMethod::parse(std::string_view caMethod) { std::string asPrefix = std::string{caMethod} + ":"; // parseContentAddressMethodPrefix takes its argument by reference @@ -144,7 +144,7 @@ std::string renderContentAddress(std::optional ca) std::string ContentAddress::printMethodAlgo() const { return method.renderPrefix() - + printHashType(hash.type); + + printHashAlgo(hash.algo); } bool StoreReferences::empty() const diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index bdb558907..05234da38 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -94,7 +94,7 @@ struct ContentAddressMethod /** * Parse a content addressing method and hash type. */ - static std::pair parse(std::string_view rawCaMethod); + static std::pair parse(std::string_view rawCaMethod); /** * Render a content addressing method and hash type in a @@ -102,7 +102,7 @@ struct ContentAddressMethod * * The rough inverse of `parse()`. */ - std::string render(HashType ht) const; + std::string render(HashAlgorithm ha) const; }; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index be9b0b0d3..530b1a178 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -400,22 +400,22 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto pathInfo = [&]() { // NB: FramedSource must be out of scope before logger->stopWork(); - auto [contentAddressMethod, hashType_] = ContentAddressMethod::parse(camStr); - auto hashType = hashType_; // work around clang bug + auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parse(camStr); + auto hashAlgo = hashAlgo_; // work around clang bug FramedSource source(from); // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. return std::visit(overloaded { [&](const TextIngestionMethod &) { - if (hashType != htSHA256) + if (hashAlgo != HashAlgorithm::SHA256) throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", - name, printHashType(hashType)); + name, printHashAlgo(hashAlgo)); // We could stream this by changing Store std::string contents = source.drain(); auto path = store->addTextToStore(name, contents, refs, repair); return store->queryPathInfo(path); }, [&](const FileIngestionMethod & fim) { - auto path = store->addToStoreFromDump(source, name, fim, hashType, repair, refs); + auto path = store->addToStoreFromDump(source, name, fim, hashAlgo, repair, refs); return store->queryPathInfo(path); }, }, contentAddressMethod.raw); @@ -424,7 +424,7 @@ static void performOp(TunnelLogger * logger, ref store, WorkerProto::Serialise::write(*store, wconn, *pathInfo); } else { - HashType hashAlgo; + HashAlgorithm hashAlgo; std::string baseName; FileIngestionMethod method; { @@ -440,7 +440,7 @@ static void performOp(TunnelLogger * logger, ref store, hashAlgoRaw = "sha256"; method = FileIngestionMethod::Recursive; } - hashAlgo = parseHashType(hashAlgoRaw); + hashAlgo = parseHashAlgo(hashAlgoRaw); } auto dumpSource = sinkToSource([&](Sink & saved) { @@ -883,7 +883,7 @@ static void performOp(TunnelLogger * logger, ref store, bool repair, dontCheckSigs; auto path = store->parseStorePath(readString(from)); auto deriver = readString(from); - auto narHash = Hash::parseAny(readString(from), htSHA256); + auto narHash = Hash::parseAny(readString(from), HashAlgorithm::SHA256); ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index dd87203b8..c68631c1a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -215,25 +215,25 @@ static StringSet parseStrings(std::istream & str, bool arePaths) static DerivationOutput parseDerivationOutput( const StoreDirConfig & store, - std::string_view pathS, std::string_view hashAlgo, std::string_view hashS, + std::string_view pathS, std::string_view hashAlgoStr, std::string_view hashS, const ExperimentalFeatureSettings & xpSettings) { - if (hashAlgo != "") { - ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo); + if (hashAlgoStr != "") { + ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgoStr); if (method == TextIngestionMethod {}) xpSettings.require(Xp::DynamicDerivations); - const auto hashType = parseHashType(hashAlgo); + const auto hashAlgo = parseHashAlgo(hashAlgoStr); if (hashS == "impure") { xpSettings.require(Xp::ImpureDerivations); if (pathS != "") throw FormatError("impure derivation output should not specify output path"); return DerivationOutput::Impure { .method = std::move(method), - .hashType = std::move(hashType), + .hashAlgo = std::move(hashAlgo), }; } else if (hashS != "") { validatePath(pathS); - auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType); + auto hash = Hash::parseNonSRIUnprefixed(hashS, hashAlgo); return DerivationOutput::CAFixed { .ca = ContentAddress { .method = std::move(method), @@ -246,7 +246,7 @@ static DerivationOutput parseDerivationOutput( throw FormatError("content-addressed derivation output should not specify output path"); return DerivationOutput::CAFloating { .method = std::move(method), - .hashType = std::move(hashType), + .hashAlgo = std::move(hashAlgo), }; } } else { @@ -547,7 +547,7 @@ std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs, }, [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); - s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashType(dof.hashType)); + s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo)); s += ','; printUnquotedString(s, ""); }, [&](const DerivationOutput::Deferred &) { @@ -558,7 +558,7 @@ std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs, [&](const DerivationOutput::Impure & doi) { // FIXME s += ','; printUnquotedString(s, ""); - s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashType(doi.hashType)); + s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo)); s += ','; printUnquotedString(s, "impure"); } }, i.second.raw); @@ -631,7 +631,7 @@ DerivationType BasicDerivation::type() const floatingCAOutputs, deferredIAOutputs, impureOutputs; - std::optional floatingHashType; + std::optional floatingHashAlgo; for (auto & i : outputs) { std::visit(overloaded { @@ -643,10 +643,10 @@ DerivationType BasicDerivation::type() const }, [&](const DerivationOutput::CAFloating & dof) { floatingCAOutputs.insert(i.first); - if (!floatingHashType) { - floatingHashType = dof.hashType; + if (!floatingHashAlgo) { + floatingHashAlgo = dof.hashAlgo; } else { - if (*floatingHashType != dof.hashType) + if (*floatingHashAlgo != dof.hashAlgo) throw Error("all floating outputs must use the same hash type"); } }, @@ -774,7 +774,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut std::map outputHashes; for (const auto & i : drv.outputs) { auto & dof = std::get(i.second.raw); - auto hash = hashString(htSHA256, "fixed:out:" + auto hash = hashString(HashAlgorithm::SHA256, "fixed:out:" + dof.ca.printMethodAlgo() + ":" + dof.ca.hash.to_string(HashFormat::Base16, false) + ":" + store.printStorePath(dof.path(store, drv.name, i.first))); @@ -825,7 +825,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut } } - auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); + auto hash = hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); std::map outputHashes; for (const auto & [outputName, _] : drv.outputs) { @@ -930,7 +930,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva }, [&](const DerivationOutput::CAFloating & dof) { out << "" - << (dof.method.renderPrefix() + printHashType(dof.hashType)) + << (dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo)) << ""; }, [&](const DerivationOutput::Deferred &) { @@ -940,7 +940,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva }, [&](const DerivationOutput::Impure & doi) { out << "" - << (doi.method.renderPrefix() + printHashType(doi.hashType)) + << (doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo)) << "impure"; }, }, i.second.raw); @@ -958,7 +958,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva std::string hashPlaceholder(const OutputNameView outputName) { // FIXME: memoize? - return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false); + return "/" + hashString(HashAlgorithm::SHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false); } @@ -1150,7 +1150,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const } -const Hash impureOutputHash = hashString(htSHA256, "impure"); +const Hash impureOutputHash = hashString(HashAlgorithm::SHA256, "impure"); nlohmann::json DerivationOutput::toJSON( const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const @@ -1167,11 +1167,11 @@ nlohmann::json DerivationOutput::toJSON( // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { - res["hashAlgo"] = dof.method.renderPrefix() + printHashType(dof.hashType); + res["hashAlgo"] = dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo); }, [&](const DerivationOutput::Deferred &) {}, [&](const DerivationOutput::Impure & doi) { - res["hashAlgo"] = doi.method.renderPrefix() + printHashType(doi.hashType); + res["hashAlgo"] = doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo); res["impure"] = true; }, }, raw); @@ -1191,15 +1191,15 @@ DerivationOutput DerivationOutput::fromJSON( for (const auto & [key, _] : json) keys.insert(key); - auto methodAlgo = [&]() -> std::pair { - std::string hashAlgo = json["hashAlgo"]; + auto methodAlgo = [&]() -> std::pair { + std::string hashAlgoStr = json["hashAlgo"]; // remaining to parse, will be mutated by parsers - std::string_view s = hashAlgo; + std::string_view s = hashAlgoStr; ContentAddressMethod method = ContentAddressMethod::parsePrefix(s); if (method == TextIngestionMethod {}) xpSettings.require(Xp::DynamicDerivations); - auto hashType = parseHashType(s); - return { std::move(method), std::move(hashType) }; + auto hashAlgo = parseHashAlgo(s); + return { std::move(method), std::move(hashAlgo) }; }; if (keys == (std::set { "path" })) { @@ -1209,11 +1209,11 @@ DerivationOutput DerivationOutput::fromJSON( } else if (keys == (std::set { "path", "hashAlgo", "hash" })) { - auto [method, hashType] = methodAlgo(); + auto [method, hashAlgo] = methodAlgo(); auto dof = DerivationOutput::CAFixed { .ca = ContentAddress { .method = std::move(method), - .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType), + .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashAlgo), }, }; if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"])) @@ -1223,10 +1223,10 @@ DerivationOutput DerivationOutput::fromJSON( else if (keys == (std::set { "hashAlgo" })) { xpSettings.require(Xp::CaDerivations); - auto [method, hashType] = methodAlgo(); + auto [method, hashAlgo] = methodAlgo(); return DerivationOutput::CAFloating { .method = std::move(method), - .hashType = std::move(hashType), + .hashAlgo = std::move(hashAlgo), }; } @@ -1236,10 +1236,10 @@ DerivationOutput DerivationOutput::fromJSON( else if (keys == (std::set { "hashAlgo", "impure" })) { xpSettings.require(Xp::ImpureDerivations); - auto [method, hashType] = methodAlgo(); + auto [method, hashAlgo] = methodAlgo(); return DerivationOutput::Impure { .method = std::move(method), - .hashType = hashType, + .hashAlgo = hashAlgo, }; } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 219e8e7d7..290abedcf 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -75,9 +75,9 @@ struct DerivationOutput /** * How the serialization will be hashed */ - HashType hashType; + HashAlgorithm hashAlgo; - GENERATE_CMP(CAFloating, me->method, me->hashType); + GENERATE_CMP(CAFloating, me->method, me->hashAlgo); }; /** @@ -102,9 +102,9 @@ struct DerivationOutput /** * How the serialization will be hashed */ - HashType hashType; + HashAlgorithm hashAlgo; - GENERATE_CMP(Impure, me->method, me->hashType); + GENERATE_CMP(Impure, me->method, me->hashAlgo); }; typedef std::variant< diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index ca9f7476e..10df37fa4 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -19,7 +19,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput( auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4); auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName); return DownstreamPlaceholder { - hashString(htSHA256, clearText) + hashString(HashAlgorithm::SHA256, clearText) }; } @@ -34,7 +34,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( + compressed.to_string(HashFormat::Base32, false) + ":" + std::string { outputName }; return DownstreamPlaceholder { - hashString(htSHA256, clearText) + hashString(HashAlgorithm::SHA256, clearText) }; } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 52130f8f6..48718ef84 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -30,7 +30,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) { auto info = queryPathInfo(path); - HashSink hashSink(htSHA256); + HashSink hashSink(HashAlgorithm::SHA256); TeeSink teeSink(sink, hashSink); narFromPath(path, teeSink); @@ -39,7 +39,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) filesystem corruption from spreading to other machines. Don't complain if the stored hash is zero (unknown). */ Hash hash = hashSink.currentHash().first; - if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) + if (hash != info->narHash && info->narHash != Hash(info->narHash.algo)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", printStorePath(path), info->narHash.to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true)); @@ -79,7 +79,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) auto references = CommonProto::Serialise::read(*this, CommonProto::ReadConn { .from = source }); auto deriver = readString(source); - auto narHash = hashString(htSHA256, saved.s); + auto narHash = hashString(HashAlgorithm::SHA256, saved.s); ValidPathInfo info { path, narHash }; if (deriver != "") diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 93fa60682..5c413aa77 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -50,7 +50,7 @@ static void makeSymlink(const Path & link, const Path & target) void LocalStore::addIndirectRoot(const Path & path) { - std::string hash = hashString(htSHA1, path).to_string(HashFormat::Base32, false); + std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Base32, false); Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash)); makeSymlink(realRoot, path); } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 731457354..fb1580dd6 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -267,13 +267,13 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor { unsupported("queryPathFromHashPart"); } StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashType hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override + std::string_view name, + const Path & srcPath, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) override { unsupported("addToStore"); } StorePath addTextToStore( diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c8962f574..ef7dd7985 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -955,7 +955,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) StorePathSet paths; for (auto & [_, i] : infos) { - assert(i.narHash.type == htSHA256); + assert(i.narHash.algo == HashAlgorithm::SHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else @@ -1069,7 +1069,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, /* While restoring the path from the NAR, compute the hash of the NAR. */ - HashSink hashSink(htSHA256); + HashSink hashSink(HashAlgorithm::SHA256); TeeSource wrapperSource { source, hashSink }; @@ -1090,7 +1090,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, auto & specified = *info.ca; auto actualHash = hashCAPath( specified.method, - specified.hash.type, + specified.hash.algo, info.path ); if (specified.hash != actualHash.hash) { @@ -1116,7 +1116,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) + FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) { /* For computing the store path. */ auto hashSink = std::make_unique(hashAlgo); @@ -1220,8 +1220,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name /* For computing the nar hash. In recursive SHA-256 mode, this is the same as the store hash, so no need to do it again. */ auto narHash = std::pair { hash, size }; - if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) { - HashSink narSink { htSHA256 }; + if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) { + HashSink narSink { HashAlgorithm::SHA256 }; dumpPath(realPath, narSink); narHash = narSink.finish(); } @@ -1252,7 +1252,7 @@ StorePath LocalStore::addTextToStore( std::string_view s, const StorePathSet & references, RepairFlag repair) { - auto hash = hashString(htSHA256, s); + auto hash = hashString(HashAlgorithm::SHA256, s); auto dstPath = makeTextPath(name, TextInfo { .hash = hash, .references = references, @@ -1278,7 +1278,7 @@ StorePath LocalStore::addTextToStore( StringSink sink; dumpString(s, sink); - auto narHash = hashString(htSHA256, sink.s); + auto narHash = hashString(HashAlgorithm::SHA256, sink.s); optimisePath(realPath, repair); @@ -1389,7 +1389,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) for (auto & link : readDirectory(linksDir)) { printMsg(lvlTalkative, "checking contents of '%s'", link.name); Path linkPath = linksDir + "/" + link.name; - std::string hash = hashPath(htSHA256, linkPath).first.to_string(HashFormat::Base32, false); + std::string hash = hashPath(HashAlgorithm::SHA256, linkPath).first.to_string(HashFormat::Base32, false); if (hash != link.name) { printError("link '%s' was modified! expected hash '%s', got '%s'", linkPath, link.name, hash); @@ -1406,7 +1406,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printInfo("checking store hashes..."); - Hash nullHash(htSHA256); + Hash nullHash(HashAlgorithm::SHA256); for (auto & i : validPaths) { try { @@ -1415,7 +1415,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Check the content hash (optionally - slow). */ printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i)); - auto hashSink = HashSink(info->narHash.type); + auto hashSink = HashSink(info->narHash.algo); dumpPath(Store::toRealPath(i), hashSink); auto current = hashSink.finish(); @@ -1697,20 +1697,20 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id, } ContentAddress LocalStore::hashCAPath( - const ContentAddressMethod & method, const HashType & hashType, + const ContentAddressMethod & method, const HashAlgorithm & hashAlgo, const StorePath & path) { - return hashCAPath(method, hashType, Store::toRealPath(path), path.hashPart()); + return hashCAPath(method, hashAlgo, Store::toRealPath(path), path.hashPart()); } ContentAddress LocalStore::hashCAPath( const ContentAddressMethod & method, - const HashType & hashType, + const HashAlgorithm & hashAlgo, const Path & path, const std::string_view pathHash ) { - HashModuloSink caSink ( hashType, std::string(pathHash) ); + HashModuloSink caSink ( hashAlgo, std::string(pathHash) ); std::visit(overloaded { [&](const TextIngestionMethod &) { readFile(path, caSink); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 8f0ffd2a2..ee605b5a2 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -178,7 +178,7 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; + FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) override; StorePath addTextToStore( std::string_view name, @@ -353,12 +353,12 @@ private: // XXX: Make a generic `Store` method ContentAddress hashCAPath( const ContentAddressMethod & method, - const HashType & hashType, + const HashAlgorithm & hashAlgo, const StorePath & path); ContentAddress hashCAPath( const ContentAddressMethod & method, - const HashType & hashType, + const HashAlgorithm & hashAlgo, const Path & path, const std::string_view pathHash ); diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 253609ed2..170fe67b9 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -43,7 +43,7 @@ std::map makeContentAddressed( sink.s = rewriteStrings(sink.s, rewrites); - HashModuloSink hashModuloSink(htSHA256, oldHashPart); + HashModuloSink hashModuloSink(HashAlgorithm::SHA256, oldHashPart); hashModuloSink(sink.s); auto narModuloHash = hashModuloSink.finish().first; @@ -66,7 +66,7 @@ std::map makeContentAddressed( rsink2(sink.s); rsink2.flush(); - info.narHash = hashString(htSHA256, sink2.s); + info.narHash = hashString(HashAlgorithm::SHA256, sink2.s); info.narSize = sink.s.size(); StringSource source(sink2.s); diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 1060a6c8b..25e2a7d7b 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -113,10 +113,10 @@ std::string NarInfo::to_string(const Store & store) const res += "URL: " + url + "\n"; assert(compression != ""); res += "Compression: " + compression + "\n"; - assert(fileHash && fileHash->type == htSHA256); + assert(fileHash && fileHash->algo == HashAlgorithm::SHA256); res += "FileHash: " + fileHash->to_string(HashFormat::Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; - assert(narHash.type == htSHA256); + assert(narHash.algo == HashAlgorithm::SHA256); res += "NarHash: " + narHash.to_string(HashFormat::Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 0fa977545..cadf88347 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -146,7 +146,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, Also note that if `path' is a symlink, then we're hashing the contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ - Hash hash = hashPath(htSHA256, path).first; + Hash hash = hashPath(HashAlgorithm::SHA256, path).first; debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Base32, true)); /* Check if this is a known hash. */ @@ -156,7 +156,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, if (pathExists(linkPath)) { auto stLink = lstat(linkPath); if (st.st_size != stLink.st_size - || (repair && hash != hashPath(htSHA256, linkPath).first)) + || (repair && hash != hashPath(HashAlgorithm::SHA256, linkPath).first)) { // XXX: Consider overwriting linkPath with our valid version. warn("removing corrupted link '%s'", linkPath); diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc index 274b596c0..15f52ec9d 100644 --- a/src/libstore/path-references.cc +++ b/src/libstore/path-references.cc @@ -49,7 +49,7 @@ std::pair scanForReferences( const std::string & path, const StorePathSet & refs) { - HashSink hashSink { htSHA256 }; + HashSink hashSink { HashAlgorithm::SHA256 }; auto found = scanForReferences(hashSink, path, refs); auto hash = hashSink.finish(); return std::pair(found, hash); diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 69f6d7356..d5257c939 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -49,7 +49,7 @@ StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x"); StorePath StorePath::random(std::string_view name) { - Hash hash(htSHA1); + Hash hash(HashAlgorithm::SHA1); randombytes_buf(hash.hash, hash.hashSize); return StorePath(hash, name); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 3d3919882..cc26c2a94 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -417,12 +417,12 @@ std::optional RemoteStore::queryPathFromHashPart(const std::string & ref RemoteStore::addCAToStore( - Source & dump, - std::string_view name, - ContentAddressMethod caMethod, - HashType hashType, - const StorePathSet & references, - RepairFlag repair) + Source & dump, + std::string_view name, + ContentAddressMethod caMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) { std::optional conn_(getConnection()); auto & conn = *conn_; @@ -432,7 +432,7 @@ ref RemoteStore::addCAToStore( conn->to << WorkerProto::Op::AddToStore << name - << caMethod.render(hashType); + << caMethod.render(hashAlgo); WorkerProto::write(*this, *conn, references); conn->to << repair; @@ -453,9 +453,9 @@ ref RemoteStore::addCAToStore( std::visit(overloaded { [&](const TextIngestionMethod & thm) -> void { - if (hashType != htSHA256) + if (hashAlgo != HashAlgorithm::SHA256) throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", - name, printHashType(hashType)); + name, printHashAlgo(hashAlgo)); std::string s = dump.drain(); conn->to << WorkerProto::Op::AddTextToStore << name << s; WorkerProto::write(*this, *conn, references); @@ -465,9 +465,9 @@ ref RemoteStore::addCAToStore( conn->to << WorkerProto::Op::AddToStore << name - << ((hashType == htSHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << ((hashAlgo == HashAlgorithm::SHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ << (fim == FileIngestionMethod::Recursive ? 1 : 0) - << printHashType(hashType); + << printHashAlgo(hashAlgo); try { conn->to.written = 0; @@ -503,9 +503,9 @@ ref RemoteStore::addCAToStore( StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references) + FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) { - return addCAToStore(dump, name, method, hashType, references, repair)->path; + return addCAToStore(dump, name, method, hashAlgo, references, repair)->path; } @@ -610,7 +610,7 @@ StorePath RemoteStore::addTextToStore( RepairFlag repair) { StringSource source(s); - return addCAToStore(source, name, TextIngestionMethod {}, htSHA256, references, repair)->path; + return addCAToStore(source, name, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair)->path; } void RemoteStore::registerDrvOutput(const Realisation & info) diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 1cc11af86..f2e34c1a3 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -74,18 +74,18 @@ public: * Add a content-addressable store path. `dump` will be drained. */ ref addCAToStore( - Source & dump, - std::string_view name, - ContentAddressMethod caMethod, - HashType hashType, - const StorePathSet & references, - RepairFlag repair); + Source & dump, + std::string_view name, + ContentAddressMethod caMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair); /** * Add a content-addressable store path. Does not support references. `dump` will be drained. */ StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; + FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8601e0857..800df7fa0 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -153,7 +153,7 @@ StorePath StoreDirConfig::makeStorePath(std::string_view type, /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ auto s = std::string(type) + ":" + std::string(hash) + ":" + storeDir + ":" + std::string(name); - auto h = compressHash(hashString(htSHA256, s), 20); + auto h = compressHash(hashString(HashAlgorithm::SHA256, s), 20); return StorePath(h, name); } @@ -191,12 +191,12 @@ static std::string makeType( StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { + if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { assert(info.references.size() == 0); return makeStorePath("output:out", - hashString(htSHA256, + hashString(HashAlgorithm::SHA256, "fixed:out:" + makeFileIngestionPrefix(info.method) + info.hash.to_string(HashFormat::Base16, true) + ":"), @@ -207,7 +207,7 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed StorePath StoreDirConfig::makeTextPath(std::string_view name, const TextInfo & info) const { - assert(info.hash.type == htSHA256); + assert(info.hash.algo == HashAlgorithm::SHA256); return makeStorePath( makeType(*this, "text", StoreReferences { .others = info.references, @@ -233,11 +233,11 @@ StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const std::pair StoreDirConfig::computeStorePathFromDump( - Source & dump, - std::string_view name, - FileIngestionMethod method, - HashType hashAlgo, - const StorePathSet & references) const + Source & dump, + std::string_view name, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references) const { HashSink sink(hashAlgo); dump.drainInto(sink); @@ -257,20 +257,20 @@ StorePath StoreDirConfig::computeStorePathForText( const StorePathSet & references) const { return makeTextPath(name, TextInfo { - .hash = hashString(htSHA256, s), + .hash = hashString(HashAlgorithm::SHA256, s), .references = references, }); } StorePath Store::addToStore( - std::string_view name, - const Path & _srcPath, - FileIngestionMethod method, - HashType hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) + std::string_view name, + const Path & _srcPath, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) { Path srcPath(absPath(_srcPath)); auto source = sinkToSource([&](Sink & sink) { @@ -405,10 +405,10 @@ digraph graphname { } */ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, - std::optional expectedCAHash) + FileIngestionMethod method, HashAlgorithm hashAlgo, + std::optional expectedCAHash) { - HashSink narHashSink { htSHA256 }; + HashSink narHashSink { HashAlgorithm::SHA256 }; HashSink caHashSink { hashAlgo }; /* Note that fileSink and unusualHashTee must be mutually exclusive, since @@ -417,7 +417,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, RegularFileSink fileSink { caHashSink }; TeeSink unusualHashTee { narHashSink, caHashSink }; - auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256 + auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != HashAlgorithm::SHA256 ? static_cast(unusualHashTee) : narHashSink; @@ -445,7 +445,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, finish. */ auto [narHash, narSize] = narHashSink.finish(); - auto hash = method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 + auto hash = method == FileIngestionMethod::Recursive && hashAlgo == HashAlgorithm::SHA256 ? narHash : caHashSink.finish().first; @@ -1205,7 +1205,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre if (!hashGiven) { std::string s; getline(str, s); - auto narHash = Hash::parseAny(s, htSHA256); + auto narHash = Hash::parseAny(s, HashAlgorithm::SHA256); getline(str, s); auto narSize = string2Int(s); if (!narSize) throw Error("number expected"); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 5860d0ea6..ada6699d5 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -427,13 +427,13 @@ public: * libutil/archive.hh). */ virtual StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, - HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, - RepairFlag repair = NoRepair, - const StorePathSet & references = StorePathSet()); + std::string_view name, + const Path & srcPath, + FileIngestionMethod method = FileIngestionMethod::Recursive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + PathFilter & filter = defaultPathFilter, + RepairFlag repair = NoRepair, + const StorePathSet & references = StorePathSet()); /** * Copy the contents of a path to the store and register the @@ -441,8 +441,8 @@ public: * memory. */ ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - std::optional expectedCAHash = {}); + FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + std::optional expectedCAHash = {}); /** * Like addToStore(), but the contents of the path are contained @@ -454,8 +454,8 @@ public: * \todo remove? */ virtual StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, - const StorePathSet & references = StorePathSet()) + FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, RepairFlag repair = NoRepair, + const StorePathSet & references = StorePathSet()) { unsupported("addToStoreFromDump"); } /** diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index 53843d663..8dafca096 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -98,7 +98,7 @@ struct StoreDirConfig : public Config Source & dump, std::string_view name, FileIngestionMethod method = FileIngestionMethod::Recursive, - HashType hashAlgo = htSHA256, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = {}) const; /** diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 43654d7e8..2a379e75e 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -160,7 +160,7 @@ void WorkerProto::Serialise::write(const StoreDirConfig & store, UnkeyedValidPathInfo WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) { auto deriver = readString(conn.from); - auto narHash = Hash::parseAny(readString(conn.from), htSHA256); + auto narHash = Hash::parseAny(readString(conn.from), HashAlgorithm::SHA256); UnkeyedValidPathInfo info(narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); info.references = WorkerProto::Serialise::read(store, conn); diff --git a/src/libutil/args.cc b/src/libutil/args.cc index c4b2975ee..ac3727d11 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -546,32 +546,32 @@ nlohmann::json Args::toJSON() static void hashTypeCompleter(AddCompletions & completions, size_t index, std::string_view prefix) { - for (auto & type : hashTypes) + for (auto & type : hashAlgorithms) if (hasPrefix(type, prefix)) completions.add(type); } -Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) +Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashAlgorithm * ha) { return Flag { .longName = std::move(longName), .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')", .labels = {"hash-algo"}, - .handler = {[ht](std::string s) { - *ht = parseHashType(s); + .handler = {[ha](std::string s) { + *ha = parseHashAlgo(s); }}, .completer = hashTypeCompleter, }; } -Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional * oht) +Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional * oha) { return Flag { .longName = std::move(longName), .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.", .labels = {"hash-algo"}, - .handler = {[oht](std::string s) { - *oht = std::optional { parseHashType(s) }; + .handler = {[oha](std::string s) { + *oha = std::optional {parseHashAlgo(s) }; }}, .completer = hashTypeCompleter, }; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 72278dccc..0cff76158 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -14,7 +14,7 @@ namespace nix { -enum HashType : char; +enum struct HashAlgorithm : char; class MultiCommand; @@ -175,8 +175,8 @@ protected: std::optional experimentalFeature; - static Flag mkHashTypeFlag(std::string && longName, HashType * ht); - static Flag mkHashTypeOptFlag(std::string && longName, std::optional * oht); + static Flag mkHashTypeFlag(std::string && longName, HashAlgorithm * ha); + static Flag mkHashTypeOptFlag(std::string && longName, std::optional * oha); }; /** diff --git a/src/libutil/git.cc b/src/libutil/git.cc index a4bd60096..296b75628 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -106,7 +106,7 @@ void parse( std::string hashs = getString(source, 20); left -= 20; - Hash hash(htSHA1); + Hash hash(HashAlgorithm::SHA1); std::copy(hashs.begin(), hashs.end(), hash.hash); hook(name, TreeEntry { @@ -241,12 +241,12 @@ Mode dump( TreeEntry dumpHash( - HashType ht, - SourceAccessor & accessor, const CanonPath & path, PathFilter & filter) + HashAlgorithm ha, + SourceAccessor & accessor, const CanonPath & path, PathFilter & filter) { std::function hook; hook = [&](const CanonPath & path) -> TreeEntry { - auto hashSink = HashSink(ht); + auto hashSink = HashSink(ha); auto mode = dump(accessor, path, hashSink, hook, filter); auto hash = hashSink.finish().first; return { diff --git a/src/libutil/git.hh b/src/libutil/git.hh index 303460072..b24b25dd3 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -123,9 +123,9 @@ Mode dump( * A smaller wrapper around `dump`. */ TreeEntry dumpHash( - HashType ht, - SourceAccessor & accessor, const CanonPath & path, - PathFilter & filter = defaultPathFilter); + HashAlgorithm ha, + SourceAccessor & accessor, const CanonPath & path, + PathFilter & filter = defaultPathFilter); /** * A line from the output of `git ls-remote --symref`. diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 144f7ae7e..38a29c459 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -16,23 +16,23 @@ namespace nix { -static size_t regularHashSize(HashType type) { +static size_t regularHashSize(HashAlgorithm type) { switch (type) { - case htMD5: return md5HashSize; - case htSHA1: return sha1HashSize; - case htSHA256: return sha256HashSize; - case htSHA512: return sha512HashSize; + case HashAlgorithm::MD5: return md5HashSize; + case HashAlgorithm::SHA1: return sha1HashSize; + case HashAlgorithm::SHA256: return sha256HashSize; + case HashAlgorithm::SHA512: return sha512HashSize; } abort(); } -std::set hashTypes = { "md5", "sha1", "sha256", "sha512" }; +std::set hashAlgorithms = {"md5", "sha1", "sha256", "sha512" }; -Hash::Hash(HashType type) : type(type) +Hash::Hash(HashAlgorithm algo) : algo(algo) { - hashSize = regularHashSize(type); + hashSize = regularHashSize(algo); assert(hashSize <= maxHashSize); memset(hash, 0, maxHashSize); } @@ -109,16 +109,16 @@ static std::string printHash32(const Hash & hash) std::string printHash16or32(const Hash & hash) { - assert(hash.type); - return hash.to_string(hash.type == htMD5 ? HashFormat::Base16 : HashFormat::Base32, false); + assert(static_cast(hash.algo)); + return hash.to_string(hash.algo == HashAlgorithm::MD5 ? HashFormat::Base16 : HashFormat::Base32, false); } -std::string Hash::to_string(HashFormat hashFormat, bool includeType) const +std::string Hash::to_string(HashFormat hashFormat, bool includeAlgo) const { std::string s; - if (hashFormat == HashFormat::SRI || includeType) { - s += printHashType(type); + if (hashFormat == HashFormat::SRI || includeAlgo) { + s += printHashAlgo(algo); s += hashFormat == HashFormat::SRI ? '-' : ':'; } switch (hashFormat) { @@ -136,7 +136,7 @@ std::string Hash::to_string(HashFormat hashFormat, bool includeType) const return s; } -Hash Hash::dummy(htSHA256); +Hash Hash::dummy(HashAlgorithm::SHA256); Hash Hash::parseSRI(std::string_view original) { auto rest = original; @@ -145,18 +145,18 @@ Hash Hash::parseSRI(std::string_view original) { auto hashRaw = splitPrefixTo(rest, '-'); if (!hashRaw) throw BadHash("hash '%s' is not SRI", original); - HashType parsedType = parseHashType(*hashRaw); + HashAlgorithm parsedType = parseHashAlgo(*hashRaw); return Hash(rest, parsedType, true); } // Mutates the string to eliminate the prefixes when found -static std::pair, bool> getParsedTypeAndSRI(std::string_view & rest) +static std::pair, bool> getParsedTypeAndSRI(std::string_view & rest) { bool isSRI = false; // Parse the hash type before the separator, if there was one. - std::optional optParsedType; + std::optional optParsedType; { auto hashRaw = splitPrefixTo(rest, ':'); @@ -166,7 +166,7 @@ static std::pair, bool> getParsedTypeAndSRI(std::string_ isSRI = true; } if (hashRaw) - optParsedType = parseHashType(*hashRaw); + optParsedType = parseHashAlgo(*hashRaw); } return {optParsedType, isSRI}; @@ -185,29 +185,29 @@ Hash Hash::parseAnyPrefixed(std::string_view original) return Hash(rest, *optParsedType, isSRI); } -Hash Hash::parseAny(std::string_view original, std::optional optType) +Hash Hash::parseAny(std::string_view original, std::optional optAlgo) { auto rest = original; auto [optParsedType, isSRI] = getParsedTypeAndSRI(rest); // Either the string or user must provide the type, if they both do they // must agree. - if (!optParsedType && !optType) + if (!optParsedType && !optAlgo) throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context", rest); - else if (optParsedType && optType && *optParsedType != *optType) - throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType)); + else if (optParsedType && optAlgo && *optParsedType != *optAlgo) + throw BadHash("hash '%s' should have type '%s'", original, printHashAlgo(*optAlgo)); - HashType hashType = optParsedType ? *optParsedType : *optType; - return Hash(rest, hashType, isSRI); + HashAlgorithm hashAlgo = optParsedType ? *optParsedType : *optAlgo; + return Hash(rest, hashAlgo, isSRI); } -Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashType type) +Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo) { - return Hash(s, type, false); + return Hash(s, algo, false); } -Hash::Hash(std::string_view rest, HashType type, bool isSRI) - : Hash(type) +Hash::Hash(std::string_view rest, HashAlgorithm algo, bool isSRI) + : Hash(algo) { if (!isSRI && rest.size() == base16Len()) { @@ -257,19 +257,19 @@ Hash::Hash(std::string_view rest, HashType type, bool isSRI) } else - throw BadHash("hash '%s' has wrong length for hash type '%s'", rest, printHashType(this->type)); + throw BadHash("hash '%s' has wrong length for hash algorithm '%s'", rest, printHashAlgo(this->algo)); } -Hash newHashAllowEmpty(std::string_view hashStr, std::optional ht) +Hash newHashAllowEmpty(std::string_view hashStr, std::optional ha) { if (hashStr.empty()) { - if (!ht) + if (!ha) throw BadHash("empty hash requires explicit hash type"); - Hash h(*ht); + Hash h(*ha); warn("found empty hash, assuming '%s'", h.to_string(HashFormat::SRI, true)); return h; } else - return Hash::parseAny(hashStr, ht); + return Hash::parseAny(hashStr, ha); } @@ -282,58 +282,58 @@ union Ctx }; -static void start(HashType ht, Ctx & ctx) +static void start(HashAlgorithm ha, Ctx & ctx) { - if (ht == htMD5) MD5_Init(&ctx.md5); - else if (ht == htSHA1) SHA1_Init(&ctx.sha1); - else if (ht == htSHA256) SHA256_Init(&ctx.sha256); - else if (ht == htSHA512) SHA512_Init(&ctx.sha512); + if (ha == HashAlgorithm::MD5) MD5_Init(&ctx.md5); + else if (ha == HashAlgorithm::SHA1) SHA1_Init(&ctx.sha1); + else if (ha == HashAlgorithm::SHA256) SHA256_Init(&ctx.sha256); + else if (ha == HashAlgorithm::SHA512) SHA512_Init(&ctx.sha512); } -static void update(HashType ht, Ctx & ctx, - std::string_view data) +static void update(HashAlgorithm ha, Ctx & ctx, + std::string_view data) { - if (ht == htMD5) MD5_Update(&ctx.md5, data.data(), data.size()); - else if (ht == htSHA1) SHA1_Update(&ctx.sha1, data.data(), data.size()); - else if (ht == htSHA256) SHA256_Update(&ctx.sha256, data.data(), data.size()); - else if (ht == htSHA512) SHA512_Update(&ctx.sha512, data.data(), data.size()); + if (ha == HashAlgorithm::MD5) MD5_Update(&ctx.md5, data.data(), data.size()); + else if (ha == HashAlgorithm::SHA1) SHA1_Update(&ctx.sha1, data.data(), data.size()); + else if (ha == HashAlgorithm::SHA256) SHA256_Update(&ctx.sha256, data.data(), data.size()); + else if (ha == HashAlgorithm::SHA512) SHA512_Update(&ctx.sha512, data.data(), data.size()); } -static void finish(HashType ht, Ctx & ctx, unsigned char * hash) +static void finish(HashAlgorithm ha, Ctx & ctx, unsigned char * hash) { - if (ht == htMD5) MD5_Final(hash, &ctx.md5); - else if (ht == htSHA1) SHA1_Final(hash, &ctx.sha1); - else if (ht == htSHA256) SHA256_Final(hash, &ctx.sha256); - else if (ht == htSHA512) SHA512_Final(hash, &ctx.sha512); + if (ha == HashAlgorithm::MD5) MD5_Final(hash, &ctx.md5); + else if (ha == HashAlgorithm::SHA1) SHA1_Final(hash, &ctx.sha1); + else if (ha == HashAlgorithm::SHA256) SHA256_Final(hash, &ctx.sha256); + else if (ha == HashAlgorithm::SHA512) SHA512_Final(hash, &ctx.sha512); } -Hash hashString(HashType ht, std::string_view s) +Hash hashString(HashAlgorithm ha, std::string_view s) { Ctx ctx; - Hash hash(ht); - start(ht, ctx); - update(ht, ctx, s); - finish(ht, ctx, hash.hash); + Hash hash(ha); + start(ha, ctx); + update(ha, ctx, s); + finish(ha, ctx, hash.hash); return hash; } -Hash hashFile(HashType ht, const Path & path) +Hash hashFile(HashAlgorithm ha, const Path & path) { - HashSink sink(ht); + HashSink sink(ha); readFile(path, sink); return sink.finish().first; } -HashSink::HashSink(HashType ht) : ht(ht) +HashSink::HashSink(HashAlgorithm ha) : ha(ha) { ctx = new Ctx; bytes = 0; - start(ht, *ctx); + start(ha, *ctx); } HashSink::~HashSink() @@ -345,14 +345,14 @@ HashSink::~HashSink() void HashSink::writeUnbuffered(std::string_view data) { bytes += data.size(); - update(ht, *ctx, data); + update(ha, *ctx, data); } HashResult HashSink::finish() { flush(); - Hash hash(ht); - nix::finish(ht, *ctx, hash.hash); + Hash hash(ha); + nix::finish(ha, *ctx, hash.hash); return HashResult(hash, bytes); } @@ -360,16 +360,16 @@ HashResult HashSink::currentHash() { flush(); Ctx ctx2 = *ctx; - Hash hash(ht); - nix::finish(ht, ctx2, hash.hash); + Hash hash(ha); + nix::finish(ha, ctx2, hash.hash); return HashResult(hash, bytes); } HashResult hashPath( - HashType ht, const Path & path, PathFilter & filter) + HashAlgorithm ha, const Path & path, PathFilter & filter) { - HashSink sink(ht); + HashSink sink(ha); dumpPath(path, sink, filter); return sink.finish(); } @@ -377,7 +377,7 @@ HashResult hashPath( Hash compressHash(const Hash & hash, unsigned int newSize) { - Hash h(hash.type); + Hash h(hash.algo); h.hashSize = newSize; for (unsigned int i = 0; i < hash.hashSize; ++i) h.hash[i % newSize] ^= hash.hash[i]; @@ -420,31 +420,31 @@ std::string_view printHashFormat(HashFormat HashFormat) } } -std::optional parseHashTypeOpt(std::string_view s) +std::optional parseHashAlgoOpt(std::string_view s) { - if (s == "md5") return htMD5; - if (s == "sha1") return htSHA1; - if (s == "sha256") return htSHA256; - if (s == "sha512") return htSHA512; + if (s == "md5") return HashAlgorithm::MD5; + if (s == "sha1") return HashAlgorithm::SHA1; + if (s == "sha256") return HashAlgorithm::SHA256; + if (s == "sha512") return HashAlgorithm::SHA512; return std::nullopt; } -HashType parseHashType(std::string_view s) +HashAlgorithm parseHashAlgo(std::string_view s) { - auto opt_h = parseHashTypeOpt(s); + auto opt_h = parseHashAlgoOpt(s); if (opt_h) return *opt_h; else throw UsageError("unknown hash algorithm '%1%', expect 'md5', 'sha1', 'sha256', or 'sha512'", s); } -std::string_view printHashType(HashType ht) +std::string_view printHashAlgo(HashAlgorithm ha) { - switch (ht) { - case htMD5: return "md5"; - case htSHA1: return "sha1"; - case htSHA256: return "sha256"; - case htSHA512: return "sha512"; + switch (ha) { + case HashAlgorithm::MD5: return "md5"; + case HashAlgorithm::SHA1: return "sha1"; + case HashAlgorithm::SHA256: return "sha256"; + case HashAlgorithm::SHA512: return "sha512"; default: // illegal hash type enum value internally, as opposed to external input // which should be validated with nice error message. diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 820154e7a..3c97ed4b1 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -12,7 +12,7 @@ namespace nix { MakeError(BadHash, Error); -enum HashType : char { htMD5 = 42, htSHA1, htSHA256, htSHA512 }; +enum struct HashAlgorithm : char { MD5 = 42, SHA1, SHA256, SHA512 }; const int md5HashSize = 16; @@ -20,7 +20,7 @@ const int sha1HashSize = 20; const int sha256HashSize = 32; const int sha512HashSize = 64; -extern std::set hashTypes; +extern std::set hashAlgorithms; extern const std::string base32Chars; @@ -46,12 +46,12 @@ struct Hash size_t hashSize = 0; uint8_t hash[maxHashSize] = {}; - HashType type; + HashAlgorithm algo; /** * Create a zero-filled hash object. */ - explicit Hash(HashType type); + explicit Hash(HashAlgorithm algo); /** * Parse the hash from a string representation in the format @@ -60,7 +60,7 @@ struct Hash * is not present, then the hash type must be specified in the * string. */ - static Hash parseAny(std::string_view s, std::optional type); + static Hash parseAny(std::string_view s, std::optional optAlgo); /** * Parse a hash from a string representation like the above, except the @@ -72,7 +72,7 @@ struct Hash * Parse a plain hash that musst not have any prefix indicating the type. * The type is passed in to disambiguate. */ - static Hash parseNonSRIUnprefixed(std::string_view s, HashType type); + static Hash parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo); static Hash parseSRI(std::string_view original); @@ -81,7 +81,7 @@ private: * The type must be provided, the string view must not include * prefix. `isSRI` helps disambigate the various base-* encodings. */ - Hash(std::string_view s, HashType type, bool isSRI); + Hash(std::string_view s, HashAlgorithm algo, bool isSRI); public: /** @@ -125,10 +125,10 @@ public: /** * Return a string representation of the hash, in base-16, base-32 - * or base-64. By default, this is prefixed by the hash type + * or base-64. By default, this is prefixed by the hash algo * (e.g. "sha256:"). */ - [[nodiscard]] std::string to_string(HashFormat hashFormat, bool includeType) const; + [[nodiscard]] std::string to_string(HashFormat hashFormat, bool includeAlgo) const; [[nodiscard]] std::string gitRev() const { @@ -146,7 +146,7 @@ public: /** * Helper that defaults empty hashes to the 0 hash. */ -Hash newHashAllowEmpty(std::string_view hashStr, std::optional ht); +Hash newHashAllowEmpty(std::string_view hashStr, std::optional ha); /** * Print a hash in base-16 if it's MD5, or base-32 otherwise. @@ -156,14 +156,14 @@ std::string printHash16or32(const Hash & hash); /** * Compute the hash of the given string. */ -Hash hashString(HashType ht, std::string_view s); +Hash hashString(HashAlgorithm ha, std::string_view s); /** * Compute the hash of the given file, hashing its contents directly. * * (Metadata, such as the executable permission bit, is ignored.) */ -Hash hashFile(HashType ht, const Path & path); +Hash hashFile(HashAlgorithm ha, const Path & path); /** * Compute the hash of the given path, serializing as a Nix Archive and @@ -172,8 +172,8 @@ Hash hashFile(HashType ht, const Path & path); * The hash is defined as (essentially) hashString(ht, dumpPath(path)). */ typedef std::pair HashResult; -HashResult hashPath(HashType ht, const Path & path, - PathFilter & filter = defaultPathFilter); +HashResult hashPath(HashAlgorithm ha, const Path & path, + PathFilter & filter = defaultPathFilter); /** * Compress a hash to the specified number of bytes by cyclically @@ -199,17 +199,17 @@ std::string_view printHashFormat(HashFormat hashFormat); /** * Parse a string representing a hash type. */ -HashType parseHashType(std::string_view s); +HashAlgorithm parseHashAlgo(std::string_view s); /** * Will return nothing on parse error */ -std::optional parseHashTypeOpt(std::string_view s); +std::optional parseHashAlgoOpt(std::string_view s); /** * And the reverse. */ -std::string_view printHashType(HashType ht); +std::string_view printHashAlgo(HashAlgorithm ha); union Ctx; @@ -222,12 +222,12 @@ struct AbstractHashSink : virtual Sink class HashSink : public BufferedSink, public AbstractHashSink { private: - HashType ht; + HashAlgorithm ha; Ctx * ctx; uint64_t bytes; public: - HashSink(HashType ht); + HashSink(HashAlgorithm ha); HashSink(const HashSink & h); ~HashSink(); void writeUnbuffered(std::string_view data) override; diff --git a/src/libutil/references.cc b/src/libutil/references.cc index 9d75606ef..d82d51945 100644 --- a/src/libutil/references.cc +++ b/src/libutil/references.cc @@ -110,8 +110,8 @@ void RewritingSink::flush() prev.clear(); } -HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus) - : hashSink(ht) +HashModuloSink::HashModuloSink(HashAlgorithm ha, const std::string & modulus) + : hashSink(ha) , rewritingSink(modulus, std::string(modulus.size(), 0), hashSink) { } diff --git a/src/libutil/references.hh b/src/libutil/references.hh index f0baeffe1..8bc9f7ec9 100644 --- a/src/libutil/references.hh +++ b/src/libutil/references.hh @@ -46,7 +46,7 @@ struct HashModuloSink : AbstractHashSink HashSink hashSink; RewritingSink rewritingSink; - HashModuloSink(HashType ht, const std::string & modulus); + HashModuloSink(HashAlgorithm ha, const std::string & modulus); void operator () (std::string_view data) override; diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index 7813433a7..afbbbe1a9 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -39,11 +39,11 @@ void SourceAccessor::readFile( } Hash SourceAccessor::hashPath( - const CanonPath & path, - PathFilter & filter, - HashType ht) + const CanonPath & path, + PathFilter & filter, + HashAlgorithm ha) { - HashSink sink(ht); + HashSink sink(ha); dumpPath(path, sink, filter); return sink.finish().first; } diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index 264caab16..3ca12d624 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -97,9 +97,9 @@ struct SourceAccessor PathFilter & filter = defaultPathFilter); Hash hashPath( - const CanonPath & path, - PathFilter & filter = defaultPathFilter, - HashType ht = htSHA256); + const CanonPath & path, + PathFilter & filter = defaultPathFilter, + HashAlgorithm ha = HashAlgorithm::SHA256); /** * Return a corresponding path in the root filesystem, if diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 25f0107bc..75ad4e75f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -193,7 +193,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs) if (opArgs.empty()) throw UsageError("first argument must be hash algorithm"); - HashType hashAlgo = parseHashType(opArgs.front()); + HashAlgorithm hashAlgo = parseHashAlgo(opArgs.front()); opArgs.pop_front(); for (auto & i : opArgs) @@ -214,7 +214,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) throw UsageError("'--print-fixed-path' requires three arguments"); Strings::iterator i = opArgs.begin(); - HashType hashAlgo = parseHashType(*i++); + HashAlgorithm hashAlgo = parseHashAlgo(*i++); std::string hash = *i++; std::string name = *i++; @@ -405,7 +405,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) { auto info = store->queryPathInfo(j); if (query == qHash) { - assert(info->narHash.type == htSHA256); + assert(info->narHash.algo == HashAlgorithm::SHA256); cout << fmt("%s\n", info->narHash.to_string(HashFormat::Base32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); @@ -541,7 +541,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) if (canonicalise) canonicalisePathMetaData(store->printStorePath(info->path), {}); if (!hashGiven) { - HashResult hash = hashPath(htSHA256, store->printStorePath(info->path)); + HashResult hash = hashPath(HashAlgorithm::SHA256, store->printStorePath(info->path)); info->narHash = hash.first; info->narSize = hash.second; } @@ -763,7 +763,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) auto path = store->followLinksToStorePath(i); printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path)); auto info = store->queryPathInfo(path); - HashSink sink(info->narHash.type); + HashSink sink(info->narHash.algo); store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info->narHash) { @@ -979,7 +979,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto deriver = readString(in); ValidPathInfo info { store->parseStorePath(path), - Hash::parseAny(readString(in), htSHA256), + Hash::parseAny(readString(in), HashAlgorithm::SHA256), }; if (deriver != "") info.deriver = store->parseStorePath(deriver); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index f9d487ada..02de796b5 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -60,11 +60,11 @@ struct CmdAddToStore : MixDryRun, StoreCommand StringSink sink; dumpPath(path, sink); - auto narHash = hashString(htSHA256, sink.s); + auto narHash = hashString(HashAlgorithm::SHA256, sink.s); Hash hash = narHash; if (ingestionMethod == FileIngestionMethod::Flat) { - HashSink hsink(htSHA256); + HashSink hsink(HashAlgorithm::SHA256); readFile(path, hsink); hash = hsink.finish().first; } diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 62f96ef1d..638178afa 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -18,7 +18,7 @@ struct CmdHashBase : Command FileIngestionMethod mode; HashFormat hashFormat = HashFormat::SRI; bool truncate = false; - HashType ht = htSHA256; + HashAlgorithm ha = HashAlgorithm::SHA256; std::vector paths; std::optional modulus; @@ -48,7 +48,7 @@ struct CmdHashBase : Command .handler = {&hashFormat, HashFormat::Base16}, }); - addFlag(Flag::mkHashTypeFlag("type", &ht)); + addFlag(Flag::mkHashTypeFlag("type", &ha)); #if 0 addFlag({ @@ -84,9 +84,9 @@ struct CmdHashBase : Command std::unique_ptr hashSink; if (modulus) - hashSink = std::make_unique(ht, *modulus); + hashSink = std::make_unique(ha, *modulus); else - hashSink = std::make_unique(ht); + hashSink = std::make_unique(ha); switch (mode) { case FileIngestionMethod::Flat: @@ -107,7 +107,7 @@ struct CmdHashBase : Command struct CmdToBase : Command { HashFormat hashFormat; - std::optional ht; + std::optional ht; std::vector args; CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat) @@ -139,7 +139,7 @@ struct CmdHashConvert : Command { std::optional from; HashFormat to; - std::optional type; + std::optional type; std::vector hashStrings; CmdHashConvert(): to(HashFormat::SRI) { @@ -166,7 +166,7 @@ struct CmdHashConvert : Command .description = "Specify the algorithm if it can't be auto-detected.", .labels = {"hash algorithm"}, .handler = {[this](std::string str) { - type = parseHashType(str); + type = parseHashAlgo(str); }}, }); expectArgs({ @@ -223,7 +223,7 @@ static auto rCmdHash = registerCommand("hash"); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) { - std::optional ht; + std::optional ha; bool flat = false; HashFormat hashFormat = HashFormat::Base16; bool truncate = false; @@ -243,7 +243,7 @@ static int compatNixHash(int argc, char * * argv) else if (*arg == "--truncate") truncate = true; else if (*arg == "--type") { std::string s = getArg(*arg, arg, end); - ht = parseHashType(s); + ha = parseHashAlgo(s); } else if (*arg == "--to-base16") { op = opTo; @@ -270,8 +270,8 @@ static int compatNixHash(int argc, char * * argv) if (op == opHash) { CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); - if (!ht.has_value()) ht = htMD5; - cmd.ht = ht.value(); + if (!ha.has_value()) ha = HashAlgorithm::MD5; + cmd.ha = ha.value(); cmd.hashFormat = hashFormat; cmd.truncate = truncate; cmd.paths = ss; @@ -281,7 +281,7 @@ static int compatNixHash(int argc, char * * argv) else { CmdToBase cmd(hashFormat); cmd.args = ss; - if (ht.has_value()) cmd.ht = ht; + if (ha.has_value()) cmd.ht = ha; cmd.run(); } diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 3ed7946a8..09f33a51e 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -46,13 +46,13 @@ std::string resolveMirrorUrl(EvalState & state, const std::string & url) } std::tuple prefetchFile( - ref store, - std::string_view url, - std::optional name, - HashType hashType, - std::optional expectedHash, - bool unpack, - bool executable) + ref store, + std::string_view url, + std::optional name, + HashAlgorithm hashAlgo, + std::optional expectedHash, + bool unpack, + bool executable) { auto ingestionMethod = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; @@ -69,7 +69,7 @@ std::tuple prefetchFile( /* If an expected hash is given, the file may already exist in the store. */ if (expectedHash) { - hashType = expectedHash->type; + hashAlgo = expectedHash->algo; storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { .method = ingestionMethod, .hash = *expectedHash, @@ -122,7 +122,7 @@ std::tuple prefetchFile( Activity act(*logger, lvlChatty, actUnknown, fmt("adding '%s' to the store", url)); - auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); + auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashAlgo, expectedHash); storePath = info.path; assert(info.ca); hash = info.ca->hash; @@ -134,7 +134,7 @@ std::tuple prefetchFile( static int main_nix_prefetch_url(int argc, char * * argv) { { - HashType ht = htSHA256; + HashAlgorithm ha = HashAlgorithm::SHA256; std::vector args; bool printPath = getEnv("PRINT_PATH") == "1"; bool fromExpr = false; @@ -155,7 +155,7 @@ static int main_nix_prefetch_url(int argc, char * * argv) printVersion("nix-prefetch-url"); else if (*arg == "--type") { auto s = getArg(*arg, arg, end); - ht = parseHashType(s); + ha = parseHashAlgo(s); } else if (*arg == "--print-path") printPath = true; @@ -233,10 +233,10 @@ static int main_nix_prefetch_url(int argc, char * * argv) std::optional expectedHash; if (args.size() == 2) - expectedHash = Hash::parseAny(args[1], ht); + expectedHash = Hash::parseAny(args[1], ha); auto [storePath, hash] = prefetchFile( - store, resolveMirrorUrl(*state, url), name, ht, expectedHash, unpack, executable); + store, resolveMirrorUrl(*state, url), name, ha, expectedHash, unpack, executable); stopProgressBar(); @@ -258,7 +258,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON std::string url; bool executable = false; std::optional name; - HashType hashType = htSHA256; + HashAlgorithm hashAlgo = HashAlgorithm::SHA256; std::optional expectedHash; CmdStorePrefetchFile() @@ -275,11 +275,11 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON .description = "The expected hash of the file.", .labels = {"hash"}, .handler = {[&](std::string s) { - expectedHash = Hash::parseAny(s, hashType); + expectedHash = Hash::parseAny(s, hashAlgo); }} }); - addFlag(Flag::mkHashTypeFlag("hash-type", &hashType)); + addFlag(Flag::mkHashTypeFlag("hash-type", &hashAlgo)); addFlag({ .longName = "executable", @@ -305,7 +305,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON } void run(ref store) override { - auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, false, executable); + auto [storePath, hash] = prefetchFile(store, url, name, hashAlgo, expectedHash, false, executable); if (json) { auto res = nlohmann::json::object(); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 147b4680b..9d9492da9 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -216,7 +216,7 @@ struct ProfileManifest StringSink sink; dumpPath(tempDir, sink); - auto narHash = hashString(htSHA256, sink.s); + auto narHash = hashString(HashAlgorithm::SHA256, sink.s); ValidPathInfo info { *store, diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 78cb765ce..cd0f6d95f 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -98,7 +98,7 @@ struct CmdVerify : StorePathsCommand if (!noContents) { - auto hashSink = HashSink(info->narHash.type); + auto hashSink = HashSink(info->narHash.algo); store->narFromPath(info->path, hashSink); diff --git a/tests/unit/libstore/common-protocol.cc b/tests/unit/libstore/common-protocol.cc index c09ac6a3e..d23805fc3 100644 --- a/tests/unit/libstore/common-protocol.cc +++ b/tests/unit/libstore/common-protocol.cc @@ -84,15 +84,15 @@ CHARACTERIZATION_TEST( (std::tuple { ContentAddress { .method = TextIngestionMethod {}, - .hash = hashString(HashType::htSHA256, "Derive(...)"), + .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -179,7 +179,7 @@ CHARACTERIZATION_TEST( std::optional { ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, })) diff --git a/tests/unit/libstore/derivation.cc b/tests/unit/libstore/derivation.cc index a7f4488fa..7a4b1403a 100644 --- a/tests/unit/libstore/derivation.cc +++ b/tests/unit/libstore/derivation.cc @@ -134,7 +134,7 @@ TEST_JSON(DynDerivationTest, caFixedText, TEST_JSON(CaDerivationTest, caFloating, (DerivationOutput::CAFloating { .method = FileIngestionMethod::Recursive, - .hashType = htSHA256, + .hashAlgo = HashAlgorithm::SHA256, }), "drv-name", "output-name") @@ -145,7 +145,7 @@ TEST_JSON(DerivationTest, deferred, TEST_JSON(ImpureDerivationTest, impure, (DerivationOutput::Impure { .method = FileIngestionMethod::Recursive, - .hashType = htSHA256, + .hashAlgo = HashAlgorithm::SHA256, }), "drv-name", "output-name") diff --git a/tests/unit/libstore/nar-info.cc b/tests/unit/libstore/nar-info.cc index 4f124e89e..bd10602e7 100644 --- a/tests/unit/libstore/nar-info.cc +++ b/tests/unit/libstore/nar-info.cc @@ -26,7 +26,7 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) { "foo", FixedOutputInfo { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { .others = { diff --git a/tests/unit/libstore/path-info.cc b/tests/unit/libstore/path-info.cc index 18f00ca19..80d6fcfed 100644 --- a/tests/unit/libstore/path-info.cc +++ b/tests/unit/libstore/path-info.cc @@ -25,7 +25,7 @@ static UnkeyedValidPathInfo makePathInfo(const Store & store, bool includeImpure "foo", FixedOutputInfo { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { .others = { diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc index c8ac87a04..6d2054f7d 100644 --- a/tests/unit/libstore/serve-protocol.cc +++ b/tests/unit/libstore/serve-protocol.cc @@ -53,15 +53,15 @@ VERSIONED_CHARACTERIZATION_TEST( (std::tuple { ContentAddress { .method = TextIngestionMethod {}, - .hash = hashString(HashType::htSHA256, "Derive(...)"), + .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -271,7 +271,7 @@ VERSIONED_CHARACTERIZATION_TEST( std::optional { ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, })) diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc index ad5943c69..91f804f0c 100644 --- a/tests/unit/libstore/worker-protocol.cc +++ b/tests/unit/libstore/worker-protocol.cc @@ -55,15 +55,15 @@ VERSIONED_CHARACTERIZATION_TEST( (std::tuple { ContentAddress { .method = TextIngestionMethod {}, - .hash = hashString(HashType::htSHA256, "Derive(...)"), + .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -464,7 +464,7 @@ VERSIONED_CHARACTERIZATION_TEST( "foo", FixedOutputInfo { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { .others = { StorePath { @@ -539,7 +539,7 @@ VERSIONED_CHARACTERIZATION_TEST( std::optional { ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, })) diff --git a/tests/unit/libutil/git.cc b/tests/unit/libutil/git.cc index 551a2d105..141a55816 100644 --- a/tests/unit/libutil/git.cc +++ b/tests/unit/libutil/git.cc @@ -95,7 +95,7 @@ const static Tree tree = { { .mode = Mode::Regular, // hello world with special chars from above - .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", htSHA1), + .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", HashAlgorithm::SHA1), }, }, { @@ -103,7 +103,7 @@ const static Tree tree = { { .mode = Mode::Executable, // ditto - .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", htSHA1), + .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", HashAlgorithm::SHA1), }, }, { @@ -111,7 +111,7 @@ const static Tree tree = { { .mode = Mode::Directory, // Empty directory hash - .hash = Hash::parseAny("4b825dc642cb6eb9a060e54bf8d69288fbee4904", htSHA1), + .hash = Hash::parseAny("4b825dc642cb6eb9a060e54bf8d69288fbee4904", HashAlgorithm::SHA1), }, }, }; @@ -174,7 +174,7 @@ TEST_F(GitTest, both_roundrip) { std::function dumpHook; dumpHook = [&](const CanonPath & path) { StringSink s; - HashSink hashSink { htSHA1 }; + HashSink hashSink { HashAlgorithm::SHA1 }; TeeSink s2 { s, hashSink }; auto mode = dump( files, path, s2, dumpHook, diff --git a/tests/unit/libutil/hash.cc b/tests/unit/libutil/hash.cc index 92291afce..4d82c7f09 100644 --- a/tests/unit/libutil/hash.cc +++ b/tests/unit/libutil/hash.cc @@ -13,28 +13,28 @@ namespace nix { TEST(hashString, testKnownMD5Hashes1) { // values taken from: https://tools.ietf.org/html/rfc1321 auto s1 = ""; - auto hash = hashString(HashType::htMD5, s1); + auto hash = hashString(HashAlgorithm::MD5, s1); ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); } TEST(hashString, testKnownMD5Hashes2) { // values taken from: https://tools.ietf.org/html/rfc1321 auto s2 = "abc"; - auto hash = hashString(HashType::htMD5, s2); + auto hash = hashString(HashAlgorithm::MD5, s2); ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); } TEST(hashString, testKnownSHA1Hashes1) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abc"; - auto hash = hashString(HashType::htSHA1, s); + auto hash = hashString(HashAlgorithm::SHA1, s); ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); } TEST(hashString, testKnownSHA1Hashes2) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; - auto hash = hashString(HashType::htSHA1, s); + auto hash = hashString(HashAlgorithm::SHA1, s); ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); } @@ -42,7 +42,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abc"; - auto hash = hashString(HashType::htSHA256, s); + auto hash = hashString(HashAlgorithm::SHA256, s); ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); } @@ -50,7 +50,7 @@ namespace nix { TEST(hashString, testKnownSHA256Hashes2) { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; - auto hash = hashString(HashType::htSHA256, s); + auto hash = hashString(HashAlgorithm::SHA256, s); ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); } @@ -58,7 +58,7 @@ namespace nix { TEST(hashString, testKnownSHA512Hashes1) { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abc"; - auto hash = hashString(HashType::htSHA512, s); + auto hash = hashString(HashAlgorithm::SHA512, s); ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9" "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" @@ -68,7 +68,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; - auto hash = hashString(HashType::htSHA512, s); + auto hash = hashString(HashAlgorithm::SHA512, s); ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1" "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" From 837b889c41543b32154ceade2363ec6ad6dff15d Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Tue, 28 Nov 2023 14:41:44 +0100 Subject: [PATCH 763/909] Further HashType renaming + using mkHashAlgoOptFlag for new conversion https://github.com/NixOS/nix/issues/8876 --- src/libutil/args.cc | 38 +++++++++++++++++++------------------- src/libutil/args.hh | 4 ++-- src/nix/hash.cc | 17 +++++------------ src/nix/prefetch.cc | 2 +- 4 files changed, 27 insertions(+), 34 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index ac3727d11..7ea1647d9 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -544,36 +544,36 @@ nlohmann::json Args::toJSON() return res; } -static void hashTypeCompleter(AddCompletions & completions, size_t index, std::string_view prefix) +static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix) { for (auto & type : hashAlgorithms) if (hasPrefix(type, prefix)) completions.add(type); } -Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashAlgorithm * ha) +Args::Flag Args::Flag::mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha) { - return Flag { - .longName = std::move(longName), - .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')", - .labels = {"hash-algo"}, - .handler = {[ha](std::string s) { - *ha = parseHashAlgo(s); - }}, - .completer = hashTypeCompleter, + return Flag{ + .longName = std::move(longName), + .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')", + .labels = {"hash-algo"}, + .handler = {[ha](std::string s) { + *ha = parseHashAlgo(s); + }}, + .completer = hashAlgoCompleter, }; } -Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional * oha) +Args::Flag Args::Flag::mkHashAlgoOptFlag(std::string && longName, std::optional * oha) { - return Flag { - .longName = std::move(longName), - .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.", - .labels = {"hash-algo"}, - .handler = {[oha](std::string s) { - *oha = std::optional {parseHashAlgo(s) }; - }}, - .completer = hashTypeCompleter, + return Flag{ + .longName = std::move(longName), + .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.", + .labels = {"hash-algo"}, + .handler = {[oha](std::string s) { + *oha = std::optional{parseHashAlgo(s)}; + }}, + .completer = hashAlgoCompleter, }; } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 0cff76158..653a9bbd6 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -175,8 +175,8 @@ protected: std::optional experimentalFeature; - static Flag mkHashTypeFlag(std::string && longName, HashAlgorithm * ha); - static Flag mkHashTypeOptFlag(std::string && longName, std::optional * oha); + static Flag mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha); + static Flag mkHashAlgoOptFlag(std::string && longName, std::optional * oha); }; /** diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 638178afa..173043c8a 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -48,7 +48,7 @@ struct CmdHashBase : Command .handler = {&hashFormat, HashFormat::Base16}, }); - addFlag(Flag::mkHashTypeFlag("type", &ha)); + addFlag(Flag::mkHashAlgoFlag("type", &ha)); #if 0 addFlag({ @@ -112,7 +112,7 @@ struct CmdToBase : Command CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat) { - addFlag(Flag::mkHashTypeOptFlag("type", &ht)); + addFlag(Flag::mkHashAlgoOptFlag("type", &ht)); expectArgs("strings", &args); } @@ -139,7 +139,7 @@ struct CmdHashConvert : Command { std::optional from; HashFormat to; - std::optional type; + std::optional algo; std::vector hashStrings; CmdHashConvert(): to(HashFormat::SRI) { @@ -161,14 +161,7 @@ struct CmdHashConvert : Command to = parseHashFormat(str); }}, }); - addFlag({ - .longName = "algo", - .description = "Specify the algorithm if it can't be auto-detected.", - .labels = {"hash algorithm"}, - .handler = {[this](std::string str) { - type = parseHashAlgo(str); - }}, - }); + addFlag(Args::Flag::mkHashAlgoOptFlag("algo", &algo)); expectArgs({ .label = "hashes", .handler = {&hashStrings}, @@ -184,7 +177,7 @@ struct CmdHashConvert : Command void run() override { for (const auto& s: hashStrings) { - Hash h = Hash::parseAny(s, type); + Hash h = Hash::parseAny(s, algo); if (from && h.to_string(*from, from == HashFormat::SRI) != s) { auto from_as_string = printHashFormat(*from); throw BadHash("input hash '%s' does not have the expected format '--from %s'", s, from_as_string); diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 09f33a51e..bbfeb8aa4 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -279,7 +279,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON }} }); - addFlag(Flag::mkHashTypeFlag("hash-type", &hashAlgo)); + addFlag(Flag::mkHashAlgoFlag("hash-type", &hashAlgo)); addFlag({ .longName = "executable", From fc6f29053aa69b6b14bcad93cb273b1c266e74fe Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Tue, 28 Nov 2023 15:38:15 +0100 Subject: [PATCH 764/909] Renamed HashFormat::Base32 to HashFormat::Nix32 ...and also adjusted parsing accordingly. Also added CLI completion for HashFormats. https://github.com/NixOS/nix/issues/8876 --- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/git.cc | 2 +- src/libfetchers/mercurial.cc | 2 +- src/libstore/binary-cache-store.cc | 4 +- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/content-address.cc | 2 +- src/libstore/derivations.cc | 2 +- src/libstore/downstream-placeholder.cc | 4 +- src/libstore/export-import.cc | 2 +- src/libstore/gc.cc | 2 +- src/libstore/local-store.cc | 10 +- src/libstore/nar-info-disk-cache.cc | 4 +- src/libstore/nar-info.cc | 4 +- src/libstore/optimise-store.cc | 4 +- src/libstore/parsed-derivations.cc | 2 +- src/libstore/path-info.cc | 6 +- src/libstore/path.cc | 2 +- src/libutil/args.cc | 40 ++++++- src/libutil/args.hh | 3 + src/libutil/hash.cc | 25 ++-- src/libutil/hash.hh | 10 +- src/libutil/references.cc | 4 +- src/nix-store/nix-store.cc | 8 +- src/nix/hash.cc | 40 +++---- src/nix/verify.cc | 4 +- tests/functional/hash.sh | 6 +- .../lang/eval-okay-convertHash.err.exp | 108 ++++++++++++++++++ .../functional/lang/eval-okay-convertHash.exp | 2 +- .../functional/lang/eval-okay-convertHash.nix | 2 + tests/unit/libutil/hash.cc | 2 +- 30 files changed, 228 insertions(+), 82 deletions(-) create mode 100644 tests/functional/lang/eval-okay-convertHash.err.exp diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index ef80c634f..15f870a95 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -304,7 +304,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v : hashFile(HashAlgorithm::SHA256, state.store->toRealPath(storePath)); if (hash != *expectedHash) state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", - *url, expectedHash->to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true))); + *url, expectedHash->to_string(HashFormat::Nix32, true), hash.to_string(HashFormat::Nix32, true))); } state.allowAndSetStorePathString(storePath, v); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index a89acc1c0..9e6ba8963 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -52,7 +52,7 @@ bool touchCacheFile(const Path & path, time_t touch_time) Path getCachePath(std::string_view key) { return getCacheDir() + "/nix/gitv3/" + - hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Base32, false); + hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false); } // Returns the name of the HEAD branch. diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 713f24bbb..6056b9a3c 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -267,7 +267,7 @@ struct MercurialInputScheme : InputScheme } } - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(HashAlgorithm::SHA256, actualUrl).to_string(HashFormat::Base32, false)); + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(HashAlgorithm::SHA256, actualUrl).to_string(HashFormat::Nix32, false)); /* If this is a commit hash that we already have, we don't have to pull again. */ diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index f287d72a8..2837e8934 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -165,8 +165,8 @@ ref BinaryCacheStore::addToStoreCommon( auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; - narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Base32, false) + ".nar" - + (compression == "xz" ? ".xz" : + narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Nix32, false) + ".nar" + + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "zstd" ? ".zst" : compression == "lzip" ? ".lzip" : diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 4c3dc1f5c..802b39f84 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1067,7 +1067,7 @@ void LocalDerivationGoal::initTmpDir() { env[i.first] = i.second; } else { auto hash = hashString(HashAlgorithm::SHA256, i.first); - std::string fn = ".attr-" + hash.to_string(HashFormat::Base32, false); + std::string fn = ".attr-" + hash.to_string(HashFormat::Nix32, false); Path p = tmpDir + "/" + fn; writeFile(p, rewriteStrings(i.second, inputRewrites)); chownToBuilder(p); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index de8194f73..f42a13126 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -61,7 +61,7 @@ std::string ContentAddress::render() const + makeFileIngestionPrefix(method); }, }, method.raw) - + this->hash.to_string(HashFormat::Base32, true); + + this->hash.to_string(HashFormat::Nix32, true); } /** diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index c68631c1a..664ab7556 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -958,7 +958,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva std::string hashPlaceholder(const OutputNameView outputName) { // FIXME: memoize? - return "/" + hashString(HashAlgorithm::SHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false); + return "/" + hashString(HashAlgorithm::SHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Nix32, false); } diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index 10df37fa4..91d47f946 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -5,7 +5,7 @@ namespace nix { std::string DownstreamPlaceholder::render() const { - return "/" + hash.to_string(HashFormat::Base32, false); + return "/" + hash.to_string(HashFormat::Nix32, false); } @@ -31,7 +31,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( xpSettings.require(Xp::DynamicDerivations); auto compressed = compressHash(placeholder.hash, 20); auto clearText = "nix-computed-output:" - + compressed.to_string(HashFormat::Base32, false) + + compressed.to_string(HashFormat::Nix32, false) + ":" + std::string { outputName }; return DownstreamPlaceholder { hashString(HashAlgorithm::SHA256, clearText) diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 48718ef84..d57b25bd7 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -41,7 +41,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) Hash hash = hashSink.currentHash().first; if (hash != info->narHash && info->narHash != Hash(info->narHash.algo)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true)); + printStorePath(path), info->narHash.to_string(HashFormat::Nix32, true), hash.to_string(HashFormat::Nix32, true)); teeSink << exportMagic diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 5c413aa77..2bd3a2edc 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -50,7 +50,7 @@ static void makeSymlink(const Path & link, const Path & target) void LocalStore::addIndirectRoot(const Path & path) { - std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Base32, false); + std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Nix32, false); Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash)); makeSymlink(realRoot, path); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ef7dd7985..7e82bae28 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1080,7 +1080,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(HashFormat::Base32, true), hashResult.first.to_string(HashFormat::Base32, true)); + printStorePath(info.path), info.narHash.to_string(HashFormat::Nix32, true), hashResult.first.to_string(HashFormat::Nix32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s", @@ -1096,8 +1096,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (specified.hash != actualHash.hash) { throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), - specified.hash.to_string(HashFormat::Base32, true), - actualHash.hash.to_string(HashFormat::Base32, true)); + specified.hash.to_string(HashFormat::Nix32, true), + actualHash.hash.to_string(HashFormat::Nix32, true)); } } @@ -1389,7 +1389,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) for (auto & link : readDirectory(linksDir)) { printMsg(lvlTalkative, "checking contents of '%s'", link.name); Path linkPath = linksDir + "/" + link.name; - std::string hash = hashPath(HashAlgorithm::SHA256, linkPath).first.to_string(HashFormat::Base32, false); + std::string hash = hashPath(HashAlgorithm::SHA256, linkPath).first.to_string(HashFormat::Nix32, false); if (hash != link.name) { printError("link '%s' was modified! expected hash '%s', got '%s'", linkPath, link.name, hash); @@ -1422,7 +1422,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) if (info->narHash != nullHash && info->narHash != current.first) { printError("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(HashFormat::Base32, true), current.first.to_string(HashFormat::Base32, true)); + printStorePath(i), info->narHash.to_string(HashFormat::Nix32, true), current.first.to_string(HashFormat::Nix32, true)); if (repair) repairPath(i); else errors = true; } else { diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index e50c15939..310105c75 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -333,9 +333,9 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(HashFormat::Base32, true) : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(HashFormat::Nix32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string(HashFormat::Base32, true)) + (info->narHash.to_string(HashFormat::Nix32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 25e2a7d7b..d9618d04c 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -114,10 +114,10 @@ std::string NarInfo::to_string(const Store & store) const assert(compression != ""); res += "Compression: " + compression + "\n"; assert(fileHash && fileHash->algo == HashAlgorithm::SHA256); - res += "FileHash: " + fileHash->to_string(HashFormat::Base32, true) + "\n"; + res += "FileHash: " + fileHash->to_string(HashFormat::Nix32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; assert(narHash.algo == HashAlgorithm::SHA256); - res += "NarHash: " + narHash.to_string(HashFormat::Base32, true) + "\n"; + res += "NarHash: " + narHash.to_string(HashFormat::Nix32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index cadf88347..b395453d1 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -147,10 +147,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ Hash hash = hashPath(HashAlgorithm::SHA256, path).first; - debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Base32, true)); + debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true)); /* Check if this is a known hash. */ - Path linkPath = linksDir + "/" + hash.to_string(HashFormat::Base32, false); + Path linkPath = linksDir + "/" + hash.to_string(HashFormat::Nix32, false); /* Maybe delete the link, if it has been corrupted. */ if (pathExists(linkPath)) { diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 73e55a96c..72f45143d 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -146,7 +146,7 @@ static nlohmann::json pathInfoToJSON( auto info = store.queryPathInfo(storePath); auto & jsonPath = jsonList.emplace_back( - info->toJSON(store, false, HashFormat::Base32)); + info->toJSON(store, false, HashFormat::Nix32)); // Add the path to the object whose metadata we are including. jsonPath["path"] = store.printStorePath(storePath); diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 2d7dc972f..f58e31bfd 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -31,9 +31,9 @@ std::string ValidPathInfo::fingerprint(const Store & store) const throw Error("cannot calculate fingerprint of path '%s' because its size is not known", store.printStorePath(path)); return - "1;" + store.printStorePath(path) + ";" - + narHash.to_string(HashFormat::Base32, true) + ";" - + std::to_string(narSize) + ";" + "1;" + store.printStorePath(path) + ";" + + narHash.to_string(HashFormat::Nix32, true) + ";" + + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } diff --git a/src/libstore/path.cc b/src/libstore/path.cc index d5257c939..1afd10af7 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -35,7 +35,7 @@ StorePath::StorePath(std::string_view _baseName) } StorePath::StorePath(const Hash & hash, std::string_view _name) - : baseName((hash.to_string(HashFormat::Base32, false) + "-").append(std::string(_name))) + : baseName((hash.to_string(HashFormat::Nix32, false) + "-").append(std::string(_name))) { checkName(baseName, name()); } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 7ea1647d9..e2668c673 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -544,11 +544,45 @@ nlohmann::json Args::toJSON() return res; } +static void hashFormatCompleter(AddCompletions & completions, size_t index, std::string_view prefix) +{ + for (auto & format : hashFormats) { + if (hasPrefix(format, prefix)) { + completions.add(format); + } + } +} + +Args::Flag Args::Flag::mkHashFormatFlagWithDefault(std::string &&longName, HashFormat * hf) { + assert(*hf == nix::HashFormat::SRI); + return Flag{ + .longName = std::move(longName), + .description = "hash format ('base16', 'nix32', 'base64', 'sri'). Default: 'sri'", + .labels = {"hash-format"}, + .handler = {[hf](std::string s) { + *hf = parseHashFormat(s); + }}, + .completer = hashFormatCompleter, + }; +} + +Args::Flag Args::Flag::mkHashFormatOptFlag(std::string && longName, std::optional * ohf) { + return Flag{ + .longName = std::move(longName), + .description = "hash format ('base16', 'nix32', 'base64', 'sri').", + .labels = {"hash-format"}, + .handler = {[ohf](std::string s) { + *ohf = std::optional{parseHashFormat(s)}; + }}, + .completer = hashFormatCompleter, + }; +} + static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix) { - for (auto & type : hashAlgorithms) - if (hasPrefix(type, prefix)) - completions.add(type); + for (auto & algo : hashAlgorithms) + if (hasPrefix(algo, prefix)) + completions.add(algo); } Args::Flag Args::Flag::mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 653a9bbd6..18b0ae583 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -15,6 +15,7 @@ namespace nix { enum struct HashAlgorithm : char; +enum struct HashFormat : int; class MultiCommand; @@ -177,6 +178,8 @@ protected: static Flag mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha); static Flag mkHashAlgoOptFlag(std::string && longName, std::optional * oha); + static Flag mkHashFormatFlagWithDefault(std::string && longName, HashFormat * hf); + static Flag mkHashFormatOptFlag(std::string && longName, std::optional * ohf); }; /** diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 38a29c459..30456ae5c 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -27,8 +27,9 @@ static size_t regularHashSize(HashAlgorithm type) { } -std::set hashAlgorithms = {"md5", "sha1", "sha256", "sha512" }; +const std::set hashAlgorithms = {"md5", "sha1", "sha256", "sha512" }; +const std::set hashFormats = {"base64", "nix32", "base16", "sri" }; Hash::Hash(HashAlgorithm algo) : algo(algo) { @@ -81,7 +82,7 @@ static std::string printHash16(const Hash & hash) // omitted: E O U T -const std::string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; +const std::string nix32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; static std::string printHash32(const Hash & hash) @@ -100,7 +101,7 @@ static std::string printHash32(const Hash & hash) unsigned char c = (hash.hash[i] >> j) | (i >= hash.hashSize - 1 ? 0 : hash.hash[i + 1] << (8 - j)); - s.push_back(base32Chars[c & 0x1f]); + s.push_back(nix32Chars[c & 0x1f]); } return s; @@ -110,7 +111,7 @@ static std::string printHash32(const Hash & hash) std::string printHash16or32(const Hash & hash) { assert(static_cast(hash.algo)); - return hash.to_string(hash.algo == HashAlgorithm::MD5 ? HashFormat::Base16 : HashFormat::Base32, false); + return hash.to_string(hash.algo == HashAlgorithm::MD5 ? HashFormat::Base16 : HashFormat::Nix32, false); } @@ -125,7 +126,7 @@ std::string Hash::to_string(HashFormat hashFormat, bool includeAlgo) const case HashFormat::Base16: s += printHash16(*this); break; - case HashFormat::Base32: + case HashFormat::Nix32: s += printHash32(*this); break; case HashFormat::Base64: @@ -230,8 +231,8 @@ Hash::Hash(std::string_view rest, HashAlgorithm algo, bool isSRI) for (unsigned int n = 0; n < rest.size(); ++n) { char c = rest[rest.size() - n - 1]; unsigned char digit; - for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ - if (base32Chars[digit] == c) break; + for (digit = 0; digit < nix32Chars.size(); ++digit) /* !!! slow */ + if (nix32Chars[digit] == c) break; if (digit >= 32) throw BadHash("invalid base-32 hash '%s'", rest); unsigned int b = n * 5; @@ -388,7 +389,11 @@ Hash compressHash(const Hash & hash, unsigned int newSize) std::optional parseHashFormatOpt(std::string_view hashFormatName) { if (hashFormatName == "base16") return HashFormat::Base16; - if (hashFormatName == "base32") return HashFormat::Base32; + if (hashFormatName == "nix32") return HashFormat::Nix32; + if (hashFormatName == "base32") { + warn(R"("base32" is a deprecated alias for hash format "nix32".)"); + return HashFormat::Nix32; + } if (hashFormatName == "base64") return HashFormat::Base64; if (hashFormatName == "sri") return HashFormat::SRI; return std::nullopt; @@ -407,8 +412,8 @@ std::string_view printHashFormat(HashFormat HashFormat) switch (HashFormat) { case HashFormat::Base64: return "base64"; - case HashFormat::Base32: - return "base32"; + case HashFormat::Nix32: + return "nix32"; case HashFormat::Base16: return "base16"; case HashFormat::SRI: diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 3c97ed4b1..7bed9e2bd 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -20,9 +20,9 @@ const int sha1HashSize = 20; const int sha256HashSize = 32; const int sha512HashSize = 64; -extern std::set hashAlgorithms; +extern const std::set hashAlgorithms; -extern const std::string base32Chars; +extern const std::string nix32Chars; /** * @brief Enumeration representing the hash formats. @@ -31,8 +31,8 @@ enum struct HashFormat : int { /// @brief Base 64 encoding. /// @see [IETF RFC 4648, section 4](https://datatracker.ietf.org/doc/html/rfc4648#section-4). Base64, - /// @brief Nix-specific base-32 encoding. @see base32Chars - Base32, + /// @brief Nix-specific base-32 encoding. @see nix32Chars + Nix32, /// @brief Lowercase hexadecimal encoding. @see base16Chars Base16, /// @brief ":", format of the SRI integrity attribute. @@ -40,6 +40,8 @@ enum struct HashFormat : int { SRI }; +extern const std::set hashFormats; + struct Hash { constexpr static size_t maxHashSize = 64; diff --git a/src/libutil/references.cc b/src/libutil/references.cc index d82d51945..b30e62c7b 100644 --- a/src/libutil/references.cc +++ b/src/libutil/references.cc @@ -23,8 +23,8 @@ static void search( static bool isBase32[256]; std::call_once(initialised, [](){ for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false; - for (unsigned int i = 0; i < base32Chars.size(); ++i) - isBase32[(unsigned char) base32Chars[i]] = true; + for (unsigned int i = 0; i < nix32Chars.size(); ++i) + isBase32[(unsigned char) nix32Chars[i]] = true; }); for (size_t i = 0; i + refLength <= s.size(); ) { diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 75ad4e75f..db45be2a8 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -406,7 +406,7 @@ static void opQuery(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(j); if (query == qHash) { assert(info->narHash.algo == HashAlgorithm::SHA256); - cout << fmt("%s\n", info->narHash.to_string(HashFormat::Base32, true)); + cout << fmt("%s\n", info->narHash.to_string(HashFormat::Nix32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); } @@ -769,8 +769,8 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) if (current.first != info->narHash) { printError("path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(path), - info->narHash.to_string(HashFormat::Base32, true), - current.first.to_string(HashFormat::Base32, true)); + info->narHash.to_string(HashFormat::Nix32, true), + current.first.to_string(HashFormat::Nix32, true)); status = 1; } } @@ -898,7 +898,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << info->narHash.to_string(HashFormat::Base32, true) + out << info->narHash.to_string(HashFormat::Nix32, true) << renderContentAddress(info->ca) << info->sigs; } catch (InvalidPath &) { diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 173043c8a..f9c7592a3 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -39,7 +39,7 @@ struct CmdHashBase : Command addFlag({ .longName = "base32", .description = "Print the hash in base-32 (Nix-specific) format.", - .handler = {&hashFormat, HashFormat::Base32}, + .handler = {&hashFormat, HashFormat::Nix32}, }); addFlag({ @@ -120,7 +120,7 @@ struct CmdToBase : Command { return fmt("convert a hash to %s representation", hashFormat == HashFormat::Base16 ? "base-16" : - hashFormat == HashFormat::Base32 ? "base-32" : + hashFormat == HashFormat::Nix32 ? "base-32" : hashFormat == HashFormat::Base64 ? "base-64" : "SRI"); } @@ -143,24 +143,8 @@ struct CmdHashConvert : Command std::vector hashStrings; CmdHashConvert(): to(HashFormat::SRI) { - addFlag({ - .longName = "from", - // TODO: List format choices. Maybe introduce a constant? - .description = "The format of the input hash.", - .labels = {"hash format"}, - .handler = {[this](std::string str) { - from = parseHashFormat(str); - }}, - }); - addFlag({ - .longName = "to", - // TODO: List format choices. Maybe introduce a constant? - .description = "The format of the output hash.", - .labels = {"hash format"}, - .handler = {[this](std::string str) { - to = parseHashFormat(str); - }}, - }); + addFlag(Args::Flag::mkHashFormatOptFlag("from", &from)); + addFlag(Args::Flag::mkHashFormatFlagWithDefault("to", &to)); addFlag(Args::Flag::mkHashAlgoOptFlag("algo", &algo)); expectArgs({ .label = "hashes", @@ -170,7 +154,15 @@ struct CmdHashConvert : Command std::string description() override { - return "convert between different hash formats, e.g. base16, nix32, base64 and sri."; + std::string descr( "convert between different hash formats. Choose from: "); + auto iter = hashFormats.begin(); + assert(iter != hashFormats.end()); + descr += *iter++; + while (iter != hashFormats.end()) { + descr += ", " + *iter++; + } + + return descr; } Category category() override { return catUtility; } @@ -197,7 +189,7 @@ struct CmdHash : NixMultiCommand {"file", []() { return make_ref(FileIngestionMethod::Flat);; }}, {"path", []() { return make_ref(FileIngestionMethod::Recursive); }}, {"to-base16", []() { return make_ref(HashFormat::Base16); }}, - {"to-base32", []() { return make_ref(HashFormat::Base32); }}, + {"to-base32", []() { return make_ref(HashFormat::Nix32); }}, {"to-base64", []() { return make_ref(HashFormat::Base64); }}, {"to-sri", []() { return make_ref(HashFormat::SRI); }}, }) @@ -230,7 +222,7 @@ static int compatNixHash(int argc, char * * argv) printVersion("nix-hash"); else if (*arg == "--flat") flat = true; else if (*arg == "--base16") hashFormat = HashFormat::Base16; - else if (*arg == "--base32") hashFormat = HashFormat::Base32; + else if (*arg == "--base32") hashFormat = HashFormat::Nix32; else if (*arg == "--base64") hashFormat = HashFormat::Base64; else if (*arg == "--sri") hashFormat = HashFormat::SRI; else if (*arg == "--truncate") truncate = true; @@ -244,7 +236,7 @@ static int compatNixHash(int argc, char * * argv) } else if (*arg == "--to-base32") { op = opTo; - hashFormat = HashFormat::Base32; + hashFormat = HashFormat::Nix32; } else if (*arg == "--to-base64") { op = opTo; diff --git a/src/nix/verify.cc b/src/nix/verify.cc index cd0f6d95f..f0234f7be 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -109,8 +109,8 @@ struct CmdVerify : StorePathsCommand act2.result(resCorruptedPath, store->printStorePath(info->path)); printError("path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(info->path), - info->narHash.to_string(HashFormat::Base32, true), - hash.first.to_string(HashFormat::Base32, true)); + info->narHash.to_string(HashFormat::Nix32, true), + hash.first.to_string(HashFormat::Nix32, true)); } } diff --git a/tests/functional/hash.sh b/tests/functional/hash.sh index 031e33adf..278ed83b9 100644 --- a/tests/functional/hash.sh +++ b/tests/functional/hash.sh @@ -163,7 +163,7 @@ try3() { sri=$(nix hash convert --algo "$1" --from base16 "$2") [ "$sri" = "$1-$4" ] - sri=$(nix hash convert --algo "$1" --from base32 "$3") + sri=$(nix hash convert --algo "$1" --from nix32 "$3") [ "$sri" = "$1-$4" ] sri=$(nix hash convert --algo "$1" --from base64 "$4") [ "$sri" = "$1-$4" ] @@ -172,11 +172,11 @@ try3() { # Asserting input format fails. # - fail=$(nix hash convert --algo "$1" --from base32 "$2" 2>&1 || echo "exit: $?") + fail=$(nix hash convert --algo "$1" --from nix32 "$2" 2>&1 || echo "exit: $?") [[ "$fail" == "error: input hash"*"exit: 1" ]] fail=$(nix hash convert --algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?") [[ "$fail" == "error: input hash"*"exit: 1" ]] - fail=$(nix hash convert --algo "$1" --from base32 "$4" 2>&1 || echo "exit: $?") + fail=$(nix hash convert --algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?") [[ "$fail" == "error: input hash"*"exit: 1" ]] } diff --git a/tests/functional/lang/eval-okay-convertHash.err.exp b/tests/functional/lang/eval-okay-convertHash.err.exp new file mode 100644 index 000000000..41d746725 --- /dev/null +++ b/tests/functional/lang/eval-okay-convertHash.err.exp @@ -0,0 +1,108 @@ +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". diff --git a/tests/functional/lang/eval-okay-convertHash.exp b/tests/functional/lang/eval-okay-convertHash.exp index 60e0a3c49..16b0240e5 100644 --- a/tests/functional/lang/eval-okay-convertHash.exp +++ b/tests/functional/lang/eval-okay-convertHash.exp @@ -1 +1 @@ -{ hashesBase16 = [ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]; hashesBase32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesBase64 = [ "1B2M2Y8AsgTpgAmY7PhCfg==" "bGnufyEcZAQZ1TZswHauRg==" "uzQ4+6vUYOptvSfRU+IjOw==" "2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "zVToVowbN88eW62wd5vL84IhIYk=" "bRLhCx0zHa0hDkf9JdTyYIArfnc=" "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; hashesSRI = [ "md5-1B2M2Y8AsgTpgAmY7PhCfg==" "md5-bGnufyEcZAQZ1TZswHauRg==" "md5-uzQ4+6vUYOptvSfRU+IjOw==" "sha1-2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "sha1-zVToVowbN88eW62wd5vL84IhIYk=" "sha1-bRLhCx0zHa0hDkf9JdTyYIArfnc=" "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "sha256-kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "sha256-rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "sha512-nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "sha512-IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; } +{ hashesBase16 = [ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]; hashesBase32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesBase64 = [ "1B2M2Y8AsgTpgAmY7PhCfg==" "bGnufyEcZAQZ1TZswHauRg==" "uzQ4+6vUYOptvSfRU+IjOw==" "2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "zVToVowbN88eW62wd5vL84IhIYk=" "bRLhCx0zHa0hDkf9JdTyYIArfnc=" "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; hashesNix32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesSRI = [ "md5-1B2M2Y8AsgTpgAmY7PhCfg==" "md5-bGnufyEcZAQZ1TZswHauRg==" "md5-uzQ4+6vUYOptvSfRU+IjOw==" "sha1-2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "sha1-zVToVowbN88eW62wd5vL84IhIYk=" "sha1-bRLhCx0zHa0hDkf9JdTyYIArfnc=" "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "sha256-kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "sha256-rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "sha512-nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "sha512-IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; } diff --git a/tests/functional/lang/eval-okay-convertHash.nix b/tests/functional/lang/eval-okay-convertHash.nix index cf4909aaf..a0191ee8d 100644 --- a/tests/functional/lang/eval-okay-convertHash.nix +++ b/tests/functional/lang/eval-okay-convertHash.nix @@ -5,12 +5,14 @@ let map2' = f: fsts: snds: map2 f { inherit fsts snds; }; getOutputHashes = hashes: { hashesBase16 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base16";}) hashAlgos hashes; + hashesNix32 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";}) hashAlgos hashes; hashesBase32 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";}) hashAlgos hashes; hashesBase64 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base64";}) hashAlgos hashes; hashesSRI = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "sri" ;}) hashAlgos hashes; }; getOutputHashesColon = hashes: { hashesBase16 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base16";}) hashAlgos hashes; + hashesNix32 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "nix32";}) hashAlgos hashes; hashesBase32 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base32";}) hashAlgos hashes; hashesBase64 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base64";}) hashAlgos hashes; hashesSRI = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "sri" ;}) hashAlgos hashes; diff --git a/tests/unit/libutil/hash.cc b/tests/unit/libutil/hash.cc index 4d82c7f09..a88994d0b 100644 --- a/tests/unit/libutil/hash.cc +++ b/tests/unit/libutil/hash.cc @@ -80,7 +80,7 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(hashFormat, testRoundTripPrintParse) { - for (const HashFormat hashFormat: { HashFormat::Base64, HashFormat::Base32, HashFormat::Base16, HashFormat::SRI}) { + for (const HashFormat hashFormat: { HashFormat::Base64, HashFormat::Nix32, HashFormat::Base16, HashFormat::SRI}) { ASSERT_EQ(parseHashFormat(printHashFormat(hashFormat)), hashFormat); ASSERT_EQ(*parseHashFormatOpt(printHashFormat(hashFormat)), hashFormat); } From 7ff876b92b590fd9559472935f4adce1d3d5efb7 Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Tue, 28 Nov 2023 15:58:04 +0100 Subject: [PATCH 765/909] Add deprecation notice for old nix hash conversion subcommands. (But not yet nix-hash since `nix hash` is still hidden behind a feature flag.) https://github.com/NixOS/nix/issues/8876 --- src/nix/hash.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/nix/hash.cc b/src/nix/hash.cc index f9c7592a3..2c9deb0d5 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -127,6 +127,7 @@ struct CmdToBase : Command void run() override { + warn("The old format conversion sub commands of `nix hash` where deprecated in favor of `nix hash convert`."); for (auto s : args) logger->cout(Hash::parseAny(s, ht).to_string(hashFormat, hashFormat == HashFormat::SRI)); } @@ -208,6 +209,9 @@ static auto rCmdHash = registerCommand("hash"); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) { + // Wait until `nix hash convert` is not hidden behind experimental flags anymore. + // warn("`nix-hash` has been deprecated in favor of `nix hash convert`."); + std::optional ha; bool flat = false; HashFormat hashFormat = HashFormat::Base16; From 8afeaf05c4063d48e65d2d82c31c3323c3237f7c Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Tue, 28 Nov 2023 19:02:15 +0100 Subject: [PATCH 766/909] Add docs/rl-notes for `nix hash convert` / `builtins.convertHash` https://github.com/NixOS/nix/issues/8876 --- doc/manual/rl-next/hash-format-nix32.md | 22 ++++++++++++ doc/manual/rl-next/nix-hash-convert.md | 47 +++++++++++++++++++++++++ src/libexpr/primops.cc | 8 ++--- src/nix/hash.cc | 2 +- 4 files changed, 74 insertions(+), 5 deletions(-) create mode 100644 doc/manual/rl-next/hash-format-nix32.md create mode 100644 doc/manual/rl-next/nix-hash-convert.md diff --git a/doc/manual/rl-next/hash-format-nix32.md b/doc/manual/rl-next/hash-format-nix32.md new file mode 100644 index 000000000..20c557da9 --- /dev/null +++ b/doc/manual/rl-next/hash-format-nix32.md @@ -0,0 +1,22 @@ +synopsis: Rename hash format `base32` to `nix32` +prs: #9452 +description: { + +Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for +[Base32](https://en.wikipedia.org/wiki/Base32). + +## Deprecation: Use `nix32` instead of `base32` as `toHashFormat` + +For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` +parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value +remains as a deprecated alias for `"base32"`. Please convert your code from: + +```nix +builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} +``` + +to + +```nix +builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";} +``` \ No newline at end of file diff --git a/doc/manual/rl-next/nix-hash-convert.md b/doc/manual/rl-next/nix-hash-convert.md new file mode 100644 index 000000000..de4367c5b --- /dev/null +++ b/doc/manual/rl-next/nix-hash-convert.md @@ -0,0 +1,47 @@ +synopsis: Add `nix hash convert` +prs: #9452 +description: { + +New [`nix hash convert`](https://github.com/NixOS/nix/issues/8876) sub command with a fast track +to stabilization! Examples: + +- Convert the hash to `nix32`. + + ```bash + $ nix hash convert --algo "sha1" --to nix32 "800d59cfcd3c05e900cb4e214be48f6b886a08df" + vw46m23bizj4n8afrc0fj19wrp7mj3c0 + ``` + `nix32` is a base32 encoding with a nix-specific character set. + Explicitly specify the hashing algorithm (optional with SRI hashes) but detect hash format by the length of the input + hash. +- Convert the hash to the `sri` format that includes an algorithm specification: + ```bash + nix hash convert --algo "sha1" "800d59cfcd3c05e900cb4e214be48f6b886a08df" + sha1-gA1Zz808BekAy04hS+SPa4hqCN8= + ``` + or with an explicit `-to` format: + ```bash + nix hash convert --algo "sha1" --to sri "800d59cfcd3c05e900cb4e214be48f6b886a08df" + sha1-gA1Zz808BekAy04hS+SPa4hqCN8= + ``` +- Assert the input format of the hash: + ```bash + nix hash convert --algo "sha256" --from nix32 "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" + error: input hash 'ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=' does not have the expected format '--from nix32' + nix hash convert --algo "sha256" --from nix32 "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" + sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= + ``` + +The `--to`/`--from`/`--algo` parameters have context-sensitive auto-completion. + +## Related Deprecations + +The following commands are still available but will emit a deprecation warning. Please convert your code to +`nix hash convert`: + +- `nix hash to-base16 $hash1 $hash2`: Use `nix hash convert --to base16 $hash1 $hash2` instead. +- `nix hash to-base32 $hash1 $hash2`: Use `nix hash convert --to nix32 $hash1 $hash2` instead. +- `nix hash to-base64 $hash1 $hash2`: Use `nix hash convert --to base64 $hash1 $hash2` instead. +- `nix hash to-sri $hash1 $hash2`: : Use `nix hash convert --to sri $hash1 $hash2` + or even just `nix hash convert $hash1 $hash2` instead. +} diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7831f3803..4162a8da3 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1339,7 +1339,7 @@ drvName, Bindings * attrs, Value & v) .errPos = state.positions[noPos] }); - auto ht = parseHashAlgoOpt(outputHashAlgo).value_or(HashAlgorithm::SHA256); + auto ha = parseHashAlgoOpt(outputHashAlgo).value_or(HashAlgorithm::SHA256); auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive); for (auto & i : outputs) { @@ -1348,13 +1348,13 @@ drvName, Bindings * attrs, Value & v) drv.outputs.insert_or_assign(i, DerivationOutput::Impure { .method = method, - .hashAlgo = ht, + .hashAlgo = ha, }); else drv.outputs.insert_or_assign(i, DerivationOutput::CAFloating { .method = method, - .hashAlgo = ht, + .hashAlgo = ha, }); } } @@ -3837,7 +3837,7 @@ static RegisterPrimOp primop_convertHash({ The format of the resulting hash. Must be one of - `"base16"` - - `"base32"` + - `"nix32"` - `"base64"` - `"sri"` diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 2c9deb0d5..0bba3b7d2 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -118,7 +118,7 @@ struct CmdToBase : Command std::string description() override { - return fmt("convert a hash to %s representation", + return fmt("convert a hash to %s representation (deprecated, use `nix hash convert` instead)", hashFormat == HashFormat::Base16 ? "base-16" : hashFormat == HashFormat::Nix32 ? "base-32" : hashFormat == HashFormat::Base64 ? "base-64" : From d38ec1285573c98c987ec1421f7cec68754204f9 Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Sat, 2 Dec 2023 11:53:50 +0100 Subject: [PATCH 767/909] Update src/libexpr/primops.cc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- src/libexpr/primops.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4162a8da3..828d118eb 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3838,6 +3838,7 @@ static RegisterPrimOp primop_convertHash({ The format of the resulting hash. Must be one of - `"base16"` - `"nix32"` + - `"base32"` (deprecated alias for `"nix32"`) - `"base64"` - `"sri"` From bbba2055f0b77e9677ef318ceea3084906eccd7d Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Sat, 2 Dec 2023 16:43:52 +0100 Subject: [PATCH 768/909] Refactor concurrently added tests to use HashAlgorithm. https://github.com/NixOS/nix/issues/8876 --- tests/unit/libutil-support/tests/hash.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/libutil-support/tests/hash.cc b/tests/unit/libutil-support/tests/hash.cc index 577e9890e..50889cd33 100644 --- a/tests/unit/libutil-support/tests/hash.cc +++ b/tests/unit/libutil-support/tests/hash.cc @@ -11,7 +11,7 @@ using namespace nix; Gen Arbitrary::arbitrary() { - Hash hash(htSHA1); + Hash hash(HashAlgorithm::SHA1); for (size_t i = 0; i < hash.hashSize; ++i) hash.hash[i] = *gen::arbitrary(); return gen::just(hash); From e9a5365db66737d1438fd91eba6529d278e1efca Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Sat, 2 Dec 2023 18:19:51 +0100 Subject: [PATCH 769/909] hash.sh: Make failure tests more tolerant of additional output "warning: you don'\''t have Internet access; disabling some network-dependent features" ... https://github.com/NixOS/nix/issues/8876 --- tests/functional/hash.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/functional/hash.sh b/tests/functional/hash.sh index 278ed83b9..47eed5178 100644 --- a/tests/functional/hash.sh +++ b/tests/functional/hash.sh @@ -173,11 +173,11 @@ try3() { # fail=$(nix hash convert --algo "$1" --from nix32 "$2" 2>&1 || echo "exit: $?") - [[ "$fail" == "error: input hash"*"exit: 1" ]] + [[ "$fail" == *"error: input hash"*"exit: 1" ]] fail=$(nix hash convert --algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?") - [[ "$fail" == "error: input hash"*"exit: 1" ]] + [[ "$fail" == *"error: input hash"*"exit: 1" ]] fail=$(nix hash convert --algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?") - [[ "$fail" == "error: input hash"*"exit: 1" ]] + [[ "$fail" == *"error: input hash"*"exit: 1" ]] } From 9a1a3c43bf11912ad32c433219c4c21a1b6ca9dd Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Sun, 3 Dec 2023 09:50:44 +0100 Subject: [PATCH 770/909] Store.xs: fix references to HashFormat::Nix32 https://github.com/NixOS/nix/issues/8876 --- perl/lib/Nix/Store.xs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 50148141b..82c7db608 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -78,7 +78,7 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Base32, true); + auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Nix32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -104,7 +104,7 @@ SV * queryPathInfo(char * path, int base32) XPUSHs(&PL_sv_undef); else XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); - auto s = info->narHash.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, true); + auto s = info->narHash.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); @@ -206,7 +206,7 @@ SV * hashPath(char * algo, int base32, char * path) PPCODE: try { Hash h = hashPath(parseHashAlgo(algo), path).first; - auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); + auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -217,7 +217,7 @@ SV * hashFile(char * algo, int base32, char * path) PPCODE: try { Hash h = hashFile(parseHashAlgo(algo), path); - auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); + auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -228,7 +228,7 @@ SV * hashString(char * algo, int base32, char * s) PPCODE: try { Hash h = hashString(parseHashAlgo(algo), s); - auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); + auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -239,7 +239,7 @@ SV * convertHash(char * algo, char * s, int toBase32) PPCODE: try { auto h = Hash::parseAny(s, parseHashAlgo(algo)); - auto s = h.to_string(toBase32 ? HashFormat::Base32 : HashFormat::Base16, false); + auto s = h.to_string(toBase32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); From bf00d5ecef20c11eb7e49dff3482b9e536cf7abe Mon Sep 17 00:00:00 2001 From: Bryan Honof Date: Thu, 7 Dec 2023 11:04:48 +0100 Subject: [PATCH 771/909] fix(libutil/tarfile): add option to libarchive so it behaves correctly with AppleDouble files AppleDouble files were extracted differently on macOS machines than on other UNIX's. Setting `archive_read_set_format_option(this->archive, NULL ,"mac-ext",NULL)` fixes this problem, since it just ignores the AppleDouble file and treats it as a normal one. This was a problem since it caused source archives to be different between macOS and Linux. Ref: nixos/nix#9290 --- src/libutil/tarfile.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 1733c791c..187b3e948 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -53,6 +53,7 @@ TarArchive::TarArchive(Source & source, bool raw) : buffer(65536) archive_read_support_format_raw(archive); archive_read_support_format_empty(archive); } + archive_read_set_option(archive, NULL, "mac-ext", NULL); check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)"); } @@ -63,6 +64,7 @@ TarArchive::TarArchive(const Path & path) archive_read_support_filter_all(archive); archive_read_support_format_all(archive); + archive_read_set_option(archive, NULL, "mac-ext", NULL); check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s"); } From a5521b7d9445af63a159d4fe7b44a0902c3a2a24 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Dec 2023 10:49:29 -0500 Subject: [PATCH 772/909] Factor out `ServeProto::Serialiser` and test In the process, partially undo e89b5bd0bfeb4dfdd8fe7e6929544cb9ceb8a505 in that the ancient < 2.4 version is now supported again by the serializer again. `LegacySSHStore`, instead of also asserting that the version is at least 4, just checks that `narHash` is set. This allows us to better test the serializer in isolation for both versions (< 4 and >= 4). --- src/libstore/legacy-ssh-store.cc | 22 ++--- src/libstore/serve-protocol.cc | 44 ++++++++++ src/libstore/serve-protocol.hh | 3 + src/nix-store/nix-store.cc | 12 +-- .../unkeyed-valid-path-info-2.3.bin | Bin 0 -> 184 bytes .../unkeyed-valid-path-info-2.4.bin | Bin 0 -> 648 bytes tests/unit/libstore/serve-protocol.cc | 77 ++++++++++++++++++ 7 files changed, 131 insertions(+), 27 deletions(-) create mode 100644 tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.3.bin create mode 100644 tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.4.bin diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index fb1580dd6..277445ee6 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -172,24 +172,12 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor if (p.empty()) return callback(nullptr); auto path2 = parseStorePath(p); assert(path == path2); - /* Hash will be set below. FIXME construct ValidPathInfo at end. */ - auto info = std::make_shared(path, Hash::dummy); + auto info = std::make_shared( + path, + ServeProto::Serialise::read(*this, *conn)); - auto deriver = readString(conn->from); - if (deriver != "") - info->deriver = parseStorePath(deriver); - info->references = ServeProto::Serialise::read(*this, *conn); - readLongLong(conn->from); // download size - info->narSize = readLongLong(conn->from); - - { - auto s = readString(conn->from); - if (s == "") - throw Error("NAR hash is now mandatory"); - info->narHash = Hash::parseAnyPrefixed(s); - } - info->ca = ContentAddress::parseOpt(readString(conn->from)); - info->sigs = readStrings(conn->from); + if (info->narHash == Hash::dummy) + throw Error("NAR hash is now mandatory"); auto s = readString(conn->from); assert(s == ""); diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index fb33553c5..c37b3095c 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -5,6 +5,7 @@ #include "serve-protocol.hh" #include "serve-protocol-impl.hh" #include "archive.hh" +#include "path-info.hh" #include @@ -54,4 +55,47 @@ void ServeProto::Serialise::write(const StoreDirConfig & store, Ser } } + +UnkeyedValidPathInfo ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + /* Hash should be set below unless very old `nix-store --serve`. + Caller should assert that it did set it. */ + UnkeyedValidPathInfo info { Hash::dummy }; + + auto deriver = readString(conn.from); + if (deriver != "") + info.deriver = store.parseStorePath(deriver); + info.references = ServeProto::Serialise::read(store, conn); + + readLongLong(conn.from); // download size, unused + info.narSize = readLongLong(conn.from); + + if (GET_PROTOCOL_MINOR(conn.version) >= 4) { + auto s = readString(conn.from); + if (!s.empty()) + info.narHash = Hash::parseAnyPrefixed(s); + info.ca = ContentAddress::parseOpt(readString(conn.from)); + info.sigs = readStrings(conn.from); + } + + return info; +} + +void ServeProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const UnkeyedValidPathInfo & info) +{ + conn.to + << (info.deriver ? store.printStorePath(*info.deriver) : ""); + + ServeProto::write(store, conn, info.references); + // !!! Maybe we want compression? + conn.to + << info.narSize // downloadSize, lie a little + << info.narSize; + if (GET_PROTOCOL_MINOR(conn.version) >= 4) + conn.to + << info.narHash.to_string(HashFormat::Nix32, true) + << renderContentAddress(info.ca) + << info.sigs; +} + } diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 6e9d66e2d..ada67a149 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -18,6 +18,7 @@ struct Source; // items being serialised struct BuildResult; +struct UnkeyedValidPathInfo; /** @@ -141,6 +142,8 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op) template<> DECLARE_SERVE_SERIALISER(BuildResult); +template<> +DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo); template DECLARE_SERVE_SERIALISER(std::vector); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index db45be2a8..45af7879c 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -891,16 +891,8 @@ static void opServe(Strings opFlags, Strings opArgs) for (auto & i : paths) { try { auto info = store->queryPathInfo(i); - out << store->printStorePath(info->path) - << (info->deriver ? store->printStorePath(*info->deriver) : ""); - ServeProto::write(*store, wconn, info->references); - // !!! Maybe we want compression? - out << info->narSize // downloadSize - << info->narSize; - if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << info->narHash.to_string(HashFormat::Nix32, true) - << renderContentAddress(info->ca) - << info->sigs; + out << store->printStorePath(info->path); + ServeProto::write(*store, wconn, static_cast(*info)); } catch (InvalidPath &) { } } diff --git a/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.3.bin b/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.3.bin new file mode 100644 index 0000000000000000000000000000000000000000..8056ec055ed2039814ab2654aad984f07115518f GIT binary patch literal 184 zcmZQzKm~Rk5I&4HhDz(_Wmf1Hm*f|v>Zco)n`cxS7viFIlM;*cQi{sJIvJt*ahg+E XS&435ArJt~*Gcp=fmdD+P%du3+9=$K@X zTQa|lwn`cL3n8wg;O8!IE8WDiY-4aNzU{MP{{~7(rMCb8 literal 0 HcmV?d00001 diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc index 6d2054f7d..c2298c6db 100644 --- a/tests/unit/libstore/serve-protocol.cc +++ b/tests/unit/libstore/serve-protocol.cc @@ -225,6 +225,83 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + unkeyedValidPathInfo_2_3, + "unkeyed-valid-path-info-2.3", + 2 << 8 | 3, + (std::tuple { + ({ + UnkeyedValidPathInfo info { Hash::dummy }; + info.narSize = 34878; + info; + }), + ({ + UnkeyedValidPathInfo info { Hash::dummy }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv", + }, + }; + info.narSize = 34878; + info; + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + unkeyedValidPathInfo_2_4, + "unkeyed-valid-path-info-2.4", + 2 << 8 | 4, + (std::tuple { + ({ + UnkeyedValidPathInfo info { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv", + }, + }; + info.narSize = 34878; + info; + }), + ({ + ValidPathInfo info { + *LibStoreTest::store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashAlgorithm::SHA256, "(...)"), + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.narSize = 34878; + info.sigs = { + "fake-sig-1", + "fake-sig-2", + }, + static_cast(std::move(info)); + }), + })) + VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, vector, From 0b80935c22f367b1deecffeddb97c90d7ed985e9 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Thu, 7 Dec 2023 10:01:42 -0800 Subject: [PATCH 773/909] Pass positions when evaluating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This includes position information in more places, making debugging easier. Before: ``` $ nix-instantiate --show-trace --eval tests/functional/lang/eval-fail-using-set-as-attr-name.nix error: … while evaluating an attribute name at «none»:0: (source not available) error: value is a set while a string was expected ``` After: ``` error: … while evaluating an attribute name at /pwd/lang/eval-fail-using-set-as-attr-name.nix:5:10: 4| in 5| attr.${key} | ^ 6| error: value is a set while a string was expected ``` --- .../rl-next/source-positions-in-errors.md | 45 +++++++++++++++++++ src/libexpr/eval-inline.hh | 12 ++--- src/libexpr/eval.cc | 18 ++++---- src/libexpr/nixexpr.hh | 1 + .../lang/eval-fail-attr-name-type.err.exp | 20 +++++++++ .../lang/eval-fail-attr-name-type.nix | 7 +++ .../lang/eval-fail-call-primop.err.exp | 12 +++++ .../functional/lang/eval-fail-call-primop.nix | 1 + .../lang/eval-fail-not-throws.err.exp | 18 ++++++++ .../functional/lang/eval-fail-not-throws.nix | 1 + .../eval-fail-using-set-as-attr-name.err.exp | 11 +++++ .../lang/eval-fail-using-set-as-attr-name.nix | 5 +++ 12 files changed, 137 insertions(+), 14 deletions(-) create mode 100644 doc/manual/rl-next/source-positions-in-errors.md create mode 100644 tests/functional/lang/eval-fail-attr-name-type.err.exp create mode 100644 tests/functional/lang/eval-fail-attr-name-type.nix create mode 100644 tests/functional/lang/eval-fail-call-primop.err.exp create mode 100644 tests/functional/lang/eval-fail-call-primop.nix create mode 100644 tests/functional/lang/eval-fail-not-throws.err.exp create mode 100644 tests/functional/lang/eval-fail-not-throws.nix create mode 100644 tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp create mode 100644 tests/functional/lang/eval-fail-using-set-as-attr-name.nix diff --git a/doc/manual/rl-next/source-positions-in-errors.md b/doc/manual/rl-next/source-positions-in-errors.md new file mode 100644 index 000000000..00f0b27e8 --- /dev/null +++ b/doc/manual/rl-next/source-positions-in-errors.md @@ -0,0 +1,45 @@ +synopsis: Source locations are printed more consistently in errors +issues: #561 +prs: #9555 +description: { + +Source location information is now included in error messages more +consistently. Given this code: + +```nix +let + attr = {foo = "bar";}; + key = {}; +in + attr.${key} +``` + +Previously, Nix would show this unhelpful message when attempting to evaluate +it: + +``` +error: + … while evaluating an attribute name + + at «none»:0: (source not available) + + error: value is a set while a string was expected +``` + +Now, the error message displays where the problematic value was found: + +``` +error: + … while evaluating an attribute name + + at bad.nix:4:11: + + 3| key = {}; + 4| in attr.${key} + | ^ + 5| + + error: value is a set while a string was expected +``` + +} diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index a988fa40c..c37b1d62b 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -103,8 +103,10 @@ void EvalState::forceValue(Value & v, Callable getPos) throw; } } - else if (v.isApp()) - callFunction(*v.app.left, *v.app.right, v, noPos); + else if (v.isApp()) { + PosIdx pos = getPos(); + callFunction(*v.app.left, *v.app.right, v, pos); + } else if (v.isBlackhole()) error("infinite recursion encountered").atPos(getPos()).template debugThrow(); } @@ -121,9 +123,9 @@ template [[gnu::always_inline]] inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view errorCtx) { - forceValue(v, noPos); + PosIdx pos = getPos(); + forceValue(v, pos); if (v.type() != nAttrs) { - PosIdx pos = getPos(); error("value is %1% while a set was expected", showType(v)).withTrace(pos, errorCtx).debugThrow(); } } @@ -132,7 +134,7 @@ inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view e [[gnu::always_inline]] inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view errorCtx) { - forceValue(v, noPos); + forceValue(v, pos); if (!v.isList()) { error("value is %1% while a list was expected", showType(v)).withTrace(pos, errorCtx).debugThrow(); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7e68e6f9b..8a6e07fb0 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -344,7 +344,7 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) } else { Value nameValue; name.expr->eval(state, env, nameValue); - state.forceStringNoCtx(nameValue, noPos, "while evaluating an attribute name"); + state.forceStringNoCtx(nameValue, name.expr->getPos(), "while evaluating an attribute name"); return state.symbols.create(nameValue.string_view()); } } @@ -1514,7 +1514,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) e->eval(state, env, vTmp); for (auto & i : attrPath) { - state.forceValue(*vAttrs, noPos); + state.forceValue(*vAttrs, getPos()); Bindings::iterator j; auto name = getName(i, state, env); if (vAttrs->type() != nAttrs || @@ -1683,7 +1683,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & if (countCalls) primOpCalls[name]++; try { - vCur.primOp->fun(*this, noPos, args, vCur); + vCur.primOp->fun(*this, vCur.determinePos(noPos), args, vCur); } catch (Error & e) { addErrorTrace(e, pos, "while calling the '%1%' builtin", name); throw; @@ -1731,7 +1731,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & // 1. Unify this and above code. Heavily redundant. // 2. Create a fake env (arg1, arg2, etc.) and a fake expr (arg1: arg2: etc: builtins.name arg1 arg2 etc) // so the debugger allows to inspect the wrong parameters passed to the builtin. - primOp->primOp->fun(*this, noPos, vArgs, vCur); + primOp->primOp->fun(*this, vCur.determinePos(noPos), vArgs, vCur); } catch (Error & e) { addErrorTrace(e, pos, "while calling the '%1%' builtin", name); throw; @@ -1839,7 +1839,7 @@ https://nixos.org/manual/nix/stable/language/constructs.html#functions.)", symbo } } - callFunction(fun, allocValue()->mkAttrs(attrs), res, noPos); + callFunction(fun, allocValue()->mkAttrs(attrs), res, pos); } @@ -1875,7 +1875,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v) void ExprOpNot::eval(EvalState & state, Env & env, Value & v) { - v.mkBool(!state.evalBool(env, e, noPos, "in the argument of the not operator")); // XXX: FIXME: ! + v.mkBool(!state.evalBool(env, e, getPos(), "in the argument of the not operator")); // XXX: FIXME: ! } @@ -2316,7 +2316,7 @@ BackedStringView EvalState::coerceToString( std::string result; for (auto [n, v2] : enumerate(v.listItems())) { try { - result += *coerceToString(noPos, *v2, context, + result += *coerceToString(pos, *v2, context, "while evaluating one element of the list", coerceMore, copyToStore, canonicalizePath); } catch (Error & e) { @@ -2463,8 +2463,8 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx) { - forceValue(v1, noPos); - forceValue(v2, noPos); + forceValue(v1, pos); + forceValue(v2, pos); /* !!! Hack to support some old broken code that relies on pointer equality tests between sets. (Specifically, builderDefs calls diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 10099d49e..020286815 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -405,6 +405,7 @@ struct ExprOpNot : Expr { Expr * e; ExprOpNot(Expr * e) : e(e) { }; + PosIdx getPos() const override { return e->getPos(); } COMMON_METHODS }; diff --git a/tests/functional/lang/eval-fail-attr-name-type.err.exp b/tests/functional/lang/eval-fail-attr-name-type.err.exp new file mode 100644 index 000000000..5f9a073dd --- /dev/null +++ b/tests/functional/lang/eval-fail-attr-name-type.err.exp @@ -0,0 +1,20 @@ +error: + … while evaluating the attribute 'puppy."${key}"' + + at /pwd/lang/eval-fail-attr-name-type.nix:3:5: + + 2| attrs = { + 3| puppy.doggy = {}; + | ^ + 4| }; + + … while evaluating an attribute name + + at /pwd/lang/eval-fail-attr-name-type.nix:7:17: + + 6| in + 7| attrs.puppy.${key} + | ^ + 8| + + error: value is an integer while a string was expected diff --git a/tests/functional/lang/eval-fail-attr-name-type.nix b/tests/functional/lang/eval-fail-attr-name-type.nix new file mode 100644 index 000000000..a0e76004a --- /dev/null +++ b/tests/functional/lang/eval-fail-attr-name-type.nix @@ -0,0 +1,7 @@ +let + attrs = { + puppy.doggy = {}; + }; + key = 1; +in + attrs.puppy.${key} diff --git a/tests/functional/lang/eval-fail-call-primop.err.exp b/tests/functional/lang/eval-fail-call-primop.err.exp new file mode 100644 index 000000000..19b407c47 --- /dev/null +++ b/tests/functional/lang/eval-fail-call-primop.err.exp @@ -0,0 +1,12 @@ +error: + … while calling the 'length' builtin + + at /pwd/lang/eval-fail-call-primop.nix:1:1: + + 1| builtins.length 1 + | ^ + 2| + + … while evaluating the first argument passed to builtins.length + + error: value is an integer while a list was expected diff --git a/tests/functional/lang/eval-fail-call-primop.nix b/tests/functional/lang/eval-fail-call-primop.nix new file mode 100644 index 000000000..972eb72c7 --- /dev/null +++ b/tests/functional/lang/eval-fail-call-primop.nix @@ -0,0 +1 @@ +builtins.length 1 diff --git a/tests/functional/lang/eval-fail-not-throws.err.exp b/tests/functional/lang/eval-fail-not-throws.err.exp new file mode 100644 index 000000000..b290afb0a --- /dev/null +++ b/tests/functional/lang/eval-fail-not-throws.err.exp @@ -0,0 +1,18 @@ +error: + … in the argument of the not operator + + at /pwd/lang/eval-fail-not-throws.nix:1:4: + + 1| ! (throw "uh oh!") + | ^ + 2| + + … while calling the 'throw' builtin + + at /pwd/lang/eval-fail-not-throws.nix:1:4: + + 1| ! (throw "uh oh!") + | ^ + 2| + + error: uh oh! diff --git a/tests/functional/lang/eval-fail-not-throws.nix b/tests/functional/lang/eval-fail-not-throws.nix new file mode 100644 index 000000000..a74ce4ebe --- /dev/null +++ b/tests/functional/lang/eval-fail-not-throws.nix @@ -0,0 +1 @@ +! (throw "uh oh!") diff --git a/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp b/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp new file mode 100644 index 000000000..811d01b03 --- /dev/null +++ b/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp @@ -0,0 +1,11 @@ +error: + … while evaluating an attribute name + + at /pwd/lang/eval-fail-using-set-as-attr-name.nix:5:10: + + 4| in + 5| attr.${key} + | ^ + 6| + + error: value is a set while a string was expected diff --git a/tests/functional/lang/eval-fail-using-set-as-attr-name.nix b/tests/functional/lang/eval-fail-using-set-as-attr-name.nix new file mode 100644 index 000000000..48e071a41 --- /dev/null +++ b/tests/functional/lang/eval-fail-using-set-as-attr-name.nix @@ -0,0 +1,5 @@ +let + attr = {foo = "bar";}; + key = {}; +in + attr.${key} From 96dd757b0c0f3d6702f8e38467a8bf467b43154e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 8 Dec 2023 00:44:55 -0500 Subject: [PATCH 774/909] Give `Derivation::tryResolve` an `evalStore` argument This is needed for building CA deriations with a src store / dest store split. In particular it is needed for Hydra. https://github.com/NixOS/hydra/issues/838 currently puts realizations, and thus build outputs, in the local store, but it should not. --- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/derivations.cc | 4 ++-- src/libstore/derivations.hh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 81eef7c47..d4da374ba 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -558,7 +558,7 @@ void DerivationGoal::inputsRealised() inputDrvOutputs statefully, sometimes it gets out of sync with the real source of truth (store). So we query the store directly if there's a problem. */ - attempt = fullDrv.tryResolve(worker.store); + attempt = fullDrv.tryResolve(worker.store, &worker.evalStore); } assert(attempt); Derivation drvResolved { std::move(*attempt) }; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 664ab7556..c35150b57 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1002,13 +1002,13 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String } -std::optional Derivation::tryResolve(Store & store) const +std::optional Derivation::tryResolve(Store & store, Store * evalStore) const { std::map, StorePath> inputDrvOutputs; std::function::ChildNode &)> accum; accum = [&](auto & inputDrv, auto & node) { - for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv)) { + for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv, evalStore)) { if (outputPath) { inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath); if (auto p = get(node.childMap, outputName)) diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 290abedcf..2a326b578 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -342,7 +342,7 @@ struct Derivation : BasicDerivation * 2. Input placeholders are replaced with realized input store * paths. */ - std::optional tryResolve(Store & store) const; + std::optional tryResolve(Store & store, Store * evalStore = nullptr) const; /** * Like the above, but instead of querying the Nix database for From f0ac2a35d5e9dfb3a53e6cc810e871fe119cbf4b Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Fri, 8 Dec 2023 11:36:57 -0500 Subject: [PATCH 775/909] Print the value in `error: cannot coerce` messages (#9553) * Print the value in `error: cannot coerce` messages This extends the `error: cannot coerce a TYPE to a string` message to print the value that could not be coerced. This helps with debugging by making it easier to track down where the value is being produced from, especially in errors with deep or unhelpful stack traces. Co-authored-by: Valentin Gagarin --- .../rl-next/print-value-in-coercion-error.md | 50 +++++++++++++++++++ .../src/language/string-interpolation.md | 2 +- src/libexpr/eval.cc | 10 ++-- ...al-fail-bad-string-interpolation-1.err.exp | 2 +- ...al-fail-bad-string-interpolation-3.err.exp | 2 +- tests/unit/libexpr/error_traces.cc | 28 +++++------ 6 files changed, 73 insertions(+), 21 deletions(-) create mode 100644 doc/manual/rl-next/print-value-in-coercion-error.md diff --git a/doc/manual/rl-next/print-value-in-coercion-error.md b/doc/manual/rl-next/print-value-in-coercion-error.md new file mode 100644 index 000000000..504ea67b9 --- /dev/null +++ b/doc/manual/rl-next/print-value-in-coercion-error.md @@ -0,0 +1,50 @@ +synopsis: Coercion errors include the failing value +issues: #561 +prs: #9553 +description: { + +The `error: cannot coerce a to a string` message now includes the value which caused the error. + +Previously, a failed string coercion produced a confusing error message if the trace didn't show where the offending value was defined: + +```bash +$ nix-instantiate --eval --expr ' +let x = { a = 1; }; in + +"${x}" +' +error: + … while evaluating a path segment + + at «string»:4:2: + + 3| + 4| "${x}" + | ^ + 5| + + error: cannot coerce a set to a string +``` + +Now, the error message includes the value itself: + +```bash +$ nix-instantiate --eval --expr ' +let x = { a = 1; }; in + +"${x}" +' +error: + … while evaluating a path segment + + at «string»:4:2: + + 3| + 4| "${x}" + | ^ + 5| + + error: cannot coerce a set to a string: { a = 1; } +``` + +} diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index e999b287b..6e28d2664 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -189,7 +189,7 @@ If neither is present, an error is thrown. > "${a}" > ``` > -> error: cannot coerce a set to a string +> error: cannot coerce a set to a string: { } > > at «string»:4:2: > diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7e68e6f9b..b52274b64 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -26,9 +26,9 @@ #include #include #include -#include #include #include +#include #include #include @@ -2286,7 +2286,7 @@ BackedStringView EvalState::coerceToString( return std::move(*maybeString); auto i = v.attrs->find(sOutPath); if (i == v.attrs->end()) { - error("cannot coerce %1% to a string", showType(v)) + error("cannot coerce %1% to a string: %2%", showType(v), printValue(*this, v)) .withTrace(pos, errorCtx) .debugThrow(); } @@ -2332,7 +2332,7 @@ BackedStringView EvalState::coerceToString( } } - error("cannot coerce %1% to a string", showType(v)) + error("cannot coerce %1% to a string: %2%", showType(v), printValue(*this, v)) .withTrace(pos, errorCtx) .debugThrow(); } @@ -2691,8 +2691,10 @@ void EvalState::printStatistics() std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const { + std::strstream printed; + print(printed); throw TypeError({ - .msg = hintfmt("cannot coerce %1% to a string", showType()) + .msg = hintfmt("cannot coerce %1% to a string: %2%", showType(), printed.str()) }); } diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp index eb73e9a52..e54ecc6d1 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp @@ -7,4 +7,4 @@ error: | ^ 2| - error: cannot coerce a function to a string + error: cannot coerce a function to a string: diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp index ac14f329b..6f0a96f78 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp @@ -7,4 +7,4 @@ error: | ^ 2| - error: cannot coerce a function to a string + error: cannot coerce a function to a string: diff --git a/tests/unit/libexpr/error_traces.cc b/tests/unit/libexpr/error_traces.cc index 81498f65a..c2403bee9 100644 --- a/tests/unit/libexpr/error_traces.cc +++ b/tests/unit/libexpr/error_traces.cc @@ -295,7 +295,7 @@ namespace nix { TEST_F(ErrorTraceTest, toPath) { ASSERT_TRACE2("toPath []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"), hintfmt("while evaluating the first argument passed to builtins.toPath")); ASSERT_TRACE2("toPath \"foo\"", @@ -309,7 +309,7 @@ namespace nix { TEST_F(ErrorTraceTest, storePath) { ASSERT_TRACE2("storePath true", TypeError, - hintfmt("cannot coerce %s to a string", "a Boolean"), + hintfmt("cannot coerce %s to a string: %s", "a Boolean", "true"), hintfmt("while evaluating the first argument passed to 'builtins.storePath'")); } @@ -318,7 +318,7 @@ namespace nix { TEST_F(ErrorTraceTest, pathExists) { ASSERT_TRACE2("pathExists []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"), hintfmt("while realising the context of a path")); ASSERT_TRACE2("pathExists \"zorglub\"", @@ -332,7 +332,7 @@ namespace nix { TEST_F(ErrorTraceTest, baseNameOf) { ASSERT_TRACE2("baseNameOf []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"), hintfmt("while evaluating the first argument passed to builtins.baseNameOf")); } @@ -377,7 +377,7 @@ namespace nix { TEST_F(ErrorTraceTest, filterSource) { ASSERT_TRACE2("filterSource [] []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"), hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] \"foo\"", @@ -1038,7 +1038,7 @@ namespace nix { TEST_F(ErrorTraceTest, toString) { ASSERT_TRACE2("toString { a = 1; }", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("cannot coerce %s to a string: %s", "a set", "{ a = 1; }"), hintfmt("while evaluating the first argument passed to builtins.toString")); } @@ -1057,7 +1057,7 @@ namespace nix { ASSERT_TRACE2("substring 0 3 {}", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), hintfmt("while evaluating the third argument (the string) passed to builtins.substring")); ASSERT_TRACE1("substring (-3) 3 \"sometext\"", @@ -1070,7 +1070,7 @@ namespace nix { TEST_F(ErrorTraceTest, stringLength) { ASSERT_TRACE2("stringLength {} # TODO: context is missing ???", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), hintfmt("while evaluating the argument passed to builtins.stringLength")); } @@ -1143,7 +1143,7 @@ namespace nix { ASSERT_TRACE2("concatStringsSep \"foo\" [ 1 2 {} ] # TODO: coerce to string is buggy", TypeError, - hintfmt("cannot coerce %s to a string", "an integer"), + hintfmt("cannot coerce %s to a string: %s", "an integer", "1"), hintfmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep")); } @@ -1229,12 +1229,12 @@ namespace nix { ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = {}; }", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), hintfmt("while evaluating the attribute 'system' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = {}; }", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"drv\"; }", @@ -1279,17 +1279,17 @@ namespace nix { ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ {} ]; }", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), hintfmt("while evaluating an element of the argument list")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ \"a\" {} ]; }", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), hintfmt("while evaluating an element of the argument list")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; FOO = {}; }", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), hintfmt("while evaluating the attribute 'FOO' of derivation 'foo'")); } From f9ee1bedcf98334d8bc015c2e04e30fbba958a3e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 8 Dec 2023 13:18:52 -0500 Subject: [PATCH 776/909] Avoid `std::strstream`, fix the clang build According https://en.cppreference.com/w/cpp/io/strstream, it has been deprecated since C++98! The Clang + Linux build systems to not have it at all, or at least be hiding it. We can just use `std::stringstream` instead, I think. --- src/libexpr/eval.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index b52274b64..5d627224f 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -28,7 +28,7 @@ #include #include #include -#include +#include #include #include @@ -2691,7 +2691,7 @@ void EvalState::printStatistics() std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const { - std::strstream printed; + std::stringstream printed; print(printed); throw TypeError({ .msg = hintfmt("cannot coerce %1% to a string: %2%", showType(), printed.str()) From ce4ca574d24abe233b717babc679e4c9228ba94b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 6 Nov 2023 09:04:50 -0500 Subject: [PATCH 777/909] Clarify `SourceAccessor` methods should never implicitly follow symlinks The code has already been fixed (yay!) so what is left of this commit is just updating the API docs. Co-authored-by: Cole Helbling --- src/libutil/source-accessor.hh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index 3ca12d624..4f4ff09c1 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -26,6 +26,13 @@ struct SourceAccessor /** * Return the contents of a file as a string. + * + * @note Unlike Unix, this method should *not* follow symlinks. Nix + * by default wants to manipulate symlinks explicitly, and not + * implictly follow them, as they are frequently untrusted user data + * and thus may point to arbitrary locations. Acting on the targets + * targets of symlinks should only occasionally be done, and only + * with care. */ virtual std::string readFile(const CanonPath & path); @@ -34,7 +41,10 @@ struct SourceAccessor * called with the size of the file before any data is written to * the sink. * - * Note: subclasses of `SourceAccessor` need to implement at least + * @note Like the other `readFile`, this method should *not* follow + * symlinks. + * + * @note subclasses of `SourceAccessor` need to implement at least * one of the `readFile()` variants. */ virtual void readFile( @@ -87,6 +97,9 @@ struct SourceAccessor typedef std::map DirEntries; + /** + * @note Like `readFile`, this method should *not* follow symlinks. + */ virtual DirEntries readDirectory(const CanonPath & path) = 0; virtual std::string readLink(const CanonPath & path) = 0; From 9b7b7a7561b24d48452627709e6872d9c610428b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 9 Dec 2023 02:13:32 +0100 Subject: [PATCH 778/909] Revert "Print the value in `error: cannot coerce` messages (#9553)" This reverts commit f0ac2a35d5e9dfb3a53e6cc810e871fe119cbf4b. The request from the sibling PR, which also applies here, was not addressed. https://github.com/NixOS/nix/pull/9554#issuecomment-1845095735 --- .../rl-next/print-value-in-coercion-error.md | 50 ------------------- .../src/language/string-interpolation.md | 2 +- src/libexpr/eval.cc | 10 ++-- ...al-fail-bad-string-interpolation-1.err.exp | 2 +- ...al-fail-bad-string-interpolation-3.err.exp | 2 +- tests/unit/libexpr/error_traces.cc | 28 +++++------ 6 files changed, 21 insertions(+), 73 deletions(-) delete mode 100644 doc/manual/rl-next/print-value-in-coercion-error.md diff --git a/doc/manual/rl-next/print-value-in-coercion-error.md b/doc/manual/rl-next/print-value-in-coercion-error.md deleted file mode 100644 index 504ea67b9..000000000 --- a/doc/manual/rl-next/print-value-in-coercion-error.md +++ /dev/null @@ -1,50 +0,0 @@ -synopsis: Coercion errors include the failing value -issues: #561 -prs: #9553 -description: { - -The `error: cannot coerce a to a string` message now includes the value which caused the error. - -Previously, a failed string coercion produced a confusing error message if the trace didn't show where the offending value was defined: - -```bash -$ nix-instantiate --eval --expr ' -let x = { a = 1; }; in - -"${x}" -' -error: - … while evaluating a path segment - - at «string»:4:2: - - 3| - 4| "${x}" - | ^ - 5| - - error: cannot coerce a set to a string -``` - -Now, the error message includes the value itself: - -```bash -$ nix-instantiate --eval --expr ' -let x = { a = 1; }; in - -"${x}" -' -error: - … while evaluating a path segment - - at «string»:4:2: - - 3| - 4| "${x}" - | ^ - 5| - - error: cannot coerce a set to a string: { a = 1; } -``` - -} diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 6e28d2664..e999b287b 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -189,7 +189,7 @@ If neither is present, an error is thrown. > "${a}" > ``` > -> error: cannot coerce a set to a string: { } +> error: cannot coerce a set to a string > > at «string»:4:2: > diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index c04e2d53d..841c223cd 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -27,9 +27,9 @@ #include #include #include +#include #include #include -#include #include #include @@ -2230,7 +2230,7 @@ BackedStringView EvalState::coerceToString( return std::move(*maybeString); auto i = v.attrs->find(sOutPath); if (i == v.attrs->end()) { - error("cannot coerce %1% to a string: %2%", showType(v), printValue(*this, v)) + error("cannot coerce %1% to a string", showType(v)) .withTrace(pos, errorCtx) .debugThrow(); } @@ -2276,7 +2276,7 @@ BackedStringView EvalState::coerceToString( } } - error("cannot coerce %1% to a string: %2%", showType(v), printValue(*this, v)) + error("cannot coerce %1% to a string", showType(v)) .withTrace(pos, errorCtx) .debugThrow(); } @@ -2635,10 +2635,8 @@ void EvalState::printStatistics() std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const { - std::stringstream printed; - print(printed); throw TypeError({ - .msg = hintfmt("cannot coerce %1% to a string: %2%", showType(), printed.str()) + .msg = hintfmt("cannot coerce %1% to a string", showType()) }); } diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp index e54ecc6d1..eb73e9a52 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp @@ -7,4 +7,4 @@ error: | ^ 2| - error: cannot coerce a function to a string: + error: cannot coerce a function to a string diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp index 6f0a96f78..ac14f329b 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp @@ -7,4 +7,4 @@ error: | ^ 2| - error: cannot coerce a function to a string: + error: cannot coerce a function to a string diff --git a/tests/unit/libexpr/error_traces.cc b/tests/unit/libexpr/error_traces.cc index c2403bee9..81498f65a 100644 --- a/tests/unit/libexpr/error_traces.cc +++ b/tests/unit/libexpr/error_traces.cc @@ -295,7 +295,7 @@ namespace nix { TEST_F(ErrorTraceTest, toPath) { ASSERT_TRACE2("toPath []", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"), + hintfmt("cannot coerce %s to a string", "a list"), hintfmt("while evaluating the first argument passed to builtins.toPath")); ASSERT_TRACE2("toPath \"foo\"", @@ -309,7 +309,7 @@ namespace nix { TEST_F(ErrorTraceTest, storePath) { ASSERT_TRACE2("storePath true", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a Boolean", "true"), + hintfmt("cannot coerce %s to a string", "a Boolean"), hintfmt("while evaluating the first argument passed to 'builtins.storePath'")); } @@ -318,7 +318,7 @@ namespace nix { TEST_F(ErrorTraceTest, pathExists) { ASSERT_TRACE2("pathExists []", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"), + hintfmt("cannot coerce %s to a string", "a list"), hintfmt("while realising the context of a path")); ASSERT_TRACE2("pathExists \"zorglub\"", @@ -332,7 +332,7 @@ namespace nix { TEST_F(ErrorTraceTest, baseNameOf) { ASSERT_TRACE2("baseNameOf []", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"), + hintfmt("cannot coerce %s to a string", "a list"), hintfmt("while evaluating the first argument passed to builtins.baseNameOf")); } @@ -377,7 +377,7 @@ namespace nix { TEST_F(ErrorTraceTest, filterSource) { ASSERT_TRACE2("filterSource [] []", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"), + hintfmt("cannot coerce %s to a string", "a list"), hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] \"foo\"", @@ -1038,7 +1038,7 @@ namespace nix { TEST_F(ErrorTraceTest, toString) { ASSERT_TRACE2("toString { a = 1; }", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ a = 1; }"), + hintfmt("cannot coerce %s to a string", "a set"), hintfmt("while evaluating the first argument passed to builtins.toString")); } @@ -1057,7 +1057,7 @@ namespace nix { ASSERT_TRACE2("substring 0 3 {}", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), + hintfmt("cannot coerce %s to a string", "a set"), hintfmt("while evaluating the third argument (the string) passed to builtins.substring")); ASSERT_TRACE1("substring (-3) 3 \"sometext\"", @@ -1070,7 +1070,7 @@ namespace nix { TEST_F(ErrorTraceTest, stringLength) { ASSERT_TRACE2("stringLength {} # TODO: context is missing ???", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), + hintfmt("cannot coerce %s to a string", "a set"), hintfmt("while evaluating the argument passed to builtins.stringLength")); } @@ -1143,7 +1143,7 @@ namespace nix { ASSERT_TRACE2("concatStringsSep \"foo\" [ 1 2 {} ] # TODO: coerce to string is buggy", TypeError, - hintfmt("cannot coerce %s to a string: %s", "an integer", "1"), + hintfmt("cannot coerce %s to a string", "an integer"), hintfmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep")); } @@ -1229,12 +1229,12 @@ namespace nix { ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = {}; }", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), + hintfmt("cannot coerce %s to a string", "a set"), hintfmt("while evaluating the attribute 'system' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = {}; }", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), + hintfmt("cannot coerce %s to a string", "a set"), hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"drv\"; }", @@ -1279,17 +1279,17 @@ namespace nix { ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ {} ]; }", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), + hintfmt("cannot coerce %s to a string", "a set"), hintfmt("while evaluating an element of the argument list")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ \"a\" {} ]; }", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), + hintfmt("cannot coerce %s to a string", "a set"), hintfmt("while evaluating an element of the argument list")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; FOO = {}; }", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), + hintfmt("cannot coerce %s to a string", "a set"), hintfmt("while evaluating the attribute 'FOO' of derivation 'foo'")); } From b9980b377ede0aca542b2baeeef9e4538dec20db Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 9 Dec 2023 02:36:33 +0100 Subject: [PATCH 779/909] Update rl-next/source-positions-in-errors for Nix 2.19+ --- doc/manual/rl-next/source-positions-in-errors.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/manual/rl-next/source-positions-in-errors.md b/doc/manual/rl-next/source-positions-in-errors.md index 00f0b27e8..15df884ea 100644 --- a/doc/manual/rl-next/source-positions-in-errors.md +++ b/doc/manual/rl-next/source-positions-in-errors.md @@ -21,8 +21,6 @@ it: error: … while evaluating an attribute name - at «none»:0: (source not available) - error: value is a set while a string was expected ``` From 6e8d5983143ae576e3f4b1d2954a5267f2943a49 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 9 Dec 2023 02:17:36 +0100 Subject: [PATCH 780/909] tests/lang/eval-fail-bad-string-interpolation-4: init --- .../lang/eval-fail-bad-string-interpolation-4.err.exp | 11 +++++++++++ .../lang/eval-fail-bad-string-interpolation-4.nix | 9 +++++++++ 2 files changed, 20 insertions(+) create mode 100644 tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp create mode 100644 tests/functional/lang/eval-fail-bad-string-interpolation-4.nix diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp new file mode 100644 index 000000000..07843a480 --- /dev/null +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp @@ -0,0 +1,11 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-string-interpolation-4.nix:9:3: + + 8| # The error message should not be too long. + 9| ''${pkgs}'' + | ^ + 10| + + error: cannot coerce a set to a string diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-4.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-4.nix new file mode 100644 index 000000000..457b5f06a --- /dev/null +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-4.nix @@ -0,0 +1,9 @@ +let + # Basically a "billion laughs" attack, but toned down to simulated `pkgs`. + ha = x: y: { a = x y; b = x y; c = x y; d = x y; e = x y; f = x y; g = x y; h = x y; j = x y; }; + has = ha (ha (ha (ha (x: x)))) "ha"; + # A large structure that has already been evaluated. + pkgs = builtins.deepSeq has has; +in +# The error message should not be too long. +''${pkgs}'' From 5417990e313272a5f1129ac39228b111e8dac857 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 8 Dec 2023 14:32:22 -0500 Subject: [PATCH 781/909] Create `ServeProto::BuildOptions` and a serializer for it More tests, and more serializers for Hydra reuse. --- src/libstore/legacy-ssh-store.cc | 22 +++----- src/libstore/serve-protocol.cc | 36 +++++++++++++ src/libstore/serve-protocol.hh | 25 +++++++++ src/nix-store/nix-store.cc | 34 ++++++++----- .../data/serve-protocol/build-options-2.1.bin | Bin 0 -> 16 bytes .../data/serve-protocol/build-options-2.2.bin | Bin 0 -> 24 bytes .../data/serve-protocol/build-options-2.3.bin | Bin 0 -> 40 bytes .../data/serve-protocol/build-options-2.7.bin | Bin 0 -> 48 bytes tests/unit/libstore/serve-protocol.cc | 48 ++++++++++++++++++ 9 files changed, 137 insertions(+), 28 deletions(-) create mode 100644 tests/unit/libstore/data/serve-protocol/build-options-2.1.bin create mode 100644 tests/unit/libstore/data/serve-protocol/build-options-2.2.bin create mode 100644 tests/unit/libstore/data/serve-protocol/build-options-2.3.bin create mode 100644 tests/unit/libstore/data/serve-protocol/build-options-2.7.bin diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 277445ee6..8ef2daa7b 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -275,20 +275,14 @@ private: void putBuildSettings(Connection & conn) { - conn.to - << settings.maxSilentTime - << settings.buildTimeout; - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2) - conn.to - << settings.maxLogSize; - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) - conn.to - << 0 // buildRepeat hasn't worked for ages anyway - << 0; - - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 7) { - conn.to << ((int) settings.keepFailed); - } + ServeProto::write(*this, conn, ServeProto::BuildOptions { + .maxSilentTime = settings.maxSilentTime, + .buildTimeout = settings.buildTimeout, + .maxLogSize = settings.maxLogSize, + .nrRepeats = 0, // buildRepeat hasn't worked for ages anyway + .enforceDeterminism = 0, + .keepFailed = settings.keepFailed, + }); } public: diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index c37b3095c..08bfad9e4 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -98,4 +98,40 @@ void ServeProto::Serialise::write(const StoreDirConfig & s << info.sigs; } + +ServeProto::BuildOptions ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + BuildOptions options; + options.maxSilentTime = readInt(conn.from); + options.buildTimeout = readInt(conn.from); + if (GET_PROTOCOL_MINOR(conn.version) >= 2) + options.maxLogSize = readNum(conn.from); + if (GET_PROTOCOL_MINOR(conn.version) >= 3) { + options.nrRepeats = readInt(conn.from); + options.enforceDeterminism = readInt(conn.from); + } + if (GET_PROTOCOL_MINOR(conn.version) >= 7) { + options.keepFailed = (bool) readInt(conn.from); + } + return options; +} + +void ServeProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const ServeProto::BuildOptions & options) +{ + conn.to + << options.maxSilentTime + << options.buildTimeout; + if (GET_PROTOCOL_MINOR(conn.version) >= 2) + conn.to + << options.maxLogSize; + if (GET_PROTOCOL_MINOR(conn.version) >= 3) + conn.to + << options.nrRepeats + << options.enforceDeterminism; + + if (GET_PROTOCOL_MINOR(conn.version) >= 7) { + conn.to << ((int) options.keepFailed); + } +} + } diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index ada67a149..1665b935f 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -87,6 +87,13 @@ struct ServeProto { ServeProto::Serialise::write(store, conn, t); } + + /** + * Options for building shared between + * `ServeProto::Command::BuildPaths` and + * `ServeProto::Command::BuildDerivation`. + */ + struct BuildOptions; }; enum struct ServeProto::Command : uint64_t @@ -102,6 +109,22 @@ enum struct ServeProto::Command : uint64_t AddToStoreNar = 9, }; + +struct ServeProto::BuildOptions { + /** + * Default value in this and every other field is so tests pass when + * testing older deserialisers which do not set all the fields. + */ + time_t maxSilentTime = -1; + time_t buildTimeout = -1; + size_t maxLogSize = -1; + size_t nrRepeats = -1; + bool enforceDeterminism = -1; + bool keepFailed = -1; + + bool operator == (const ServeProto::BuildOptions &) const = default; +}; + /** * Convenience for sending operation codes. * @@ -144,6 +167,8 @@ template<> DECLARE_SERVE_SERIALISER(BuildResult); template<> DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo); +template<> +DECLARE_SERVE_SERIALISER(ServeProto::BuildOptions); template DECLARE_SERVE_SERIALISER(std::vector); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 45af7879c..d361dc0ac 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -835,27 +835,33 @@ static void opServe(Strings opFlags, Strings opArgs) verbosity = lvlError; settings.keepLog = false; settings.useSubstitutes = false; - settings.maxSilentTime = readInt(in); - settings.buildTimeout = readInt(in); + + auto options = ServeProto::Serialise::read(*store, rconn); + + // Only certain feilds get initialized based on the protocol + // version. This is why not all the code below is unconditional. + // See how the serialization logic in + // `ServeProto::Serialise` matches + // these conditions. + settings.maxSilentTime = options.maxSilentTime; + settings.buildTimeout = options.buildTimeout; if (GET_PROTOCOL_MINOR(clientVersion) >= 2) - settings.maxLogSize = readNum(in); + settings.maxLogSize = options.maxLogSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { - auto nrRepeats = readInt(in); - if (nrRepeats != 0) { + if (options.nrRepeats != 0) { throw Error("client requested repeating builds, but this is not currently implemented"); } - // Ignore 'enforceDeterminism'. It used to be true by - // default, but also only never had any effect when - // `nrRepeats == 0`. We have already asserted that - // `nrRepeats` in fact is 0, so we can safely ignore this - // without doing something other than what the client - // asked for. - readInt(in); - + // Ignore 'options.enforceDeterminism'. + // + // It used to be true by default, but also only never had + // any effect when `nrRepeats == 0`. We have already + // checked that `nrRepeats` in fact is 0, so we can safely + // ignore this without doing something other than what the + // client asked for. settings.runDiffHook = true; } if (GET_PROTOCOL_MINOR(clientVersion) >= 7) { - settings.keepFailed = (bool) readInt(in); + settings.keepFailed = options.keepFailed; } }; diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.1.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.1.bin new file mode 100644 index 0000000000000000000000000000000000000000..61e1d97286139e43918505b1b953128360d27853 GIT binary patch literal 16 NcmZQ&fB-fq4FCX;01N;C literal 0 HcmV?d00001 diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.2.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.2.bin new file mode 100644 index 0000000000000000000000000000000000000000..045c2ff2b54ba708bc1d411f0e8786207c4e660a GIT binary patch literal 24 PcmZQ&fB-fq%?_mj0Vn_y literal 0 HcmV?d00001 diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.3.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.3.bin new file mode 100644 index 0000000000000000000000000000000000000000..5c53458831dca70d5303363919f46f20f88993a2 GIT binary patch literal 40 VcmZQ&fB-fq%?_nGpfn?t1^@!!02}}S literal 0 HcmV?d00001 diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.7.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.7.bin new file mode 100644 index 0000000000000000000000000000000000000000..1bc7b02db38f5f751c2610de84ff937e630567c9 GIT binary patch literal 48 WcmZQ&fB-fq%?_nGpfrqPgfajFxBwgg literal 0 HcmV?d00001 diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc index c2298c6db..8f256d1e6 100644 --- a/tests/unit/libstore/serve-protocol.cc +++ b/tests/unit/libstore/serve-protocol.cc @@ -302,6 +302,54 @@ VERSIONED_CHARACTERIZATION_TEST( }), })) +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + build_options_2_1, + "build-options-2.1", + 2 << 8 | 1, + (ServeProto::BuildOptions { + .maxSilentTime = 5, + .buildTimeout = 6, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + build_options_2_2, + "build-options-2.2", + 2 << 8 | 2, + (ServeProto::BuildOptions { + .maxSilentTime = 5, + .buildTimeout = 6, + .maxLogSize = 7, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + build_options_2_3, + "build-options-2.3", + 2 << 8 | 3, + (ServeProto::BuildOptions { + .maxSilentTime = 5, + .buildTimeout = 6, + .maxLogSize = 7, + .nrRepeats = 8, + .enforceDeterminism = true, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + build_options_2_7, + "build-options-2.7", + 2 << 8 | 7, + (ServeProto::BuildOptions { + .maxSilentTime = 5, + .buildTimeout = 6, + .maxLogSize = 7, + .nrRepeats = 8, + .enforceDeterminism = false, + .keepFailed = true, + })) + VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, vector, From 360f3b3a9e0a74eb8b7d5a1744ad58f4cd487ca0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 9 Dec 2023 19:50:33 +0100 Subject: [PATCH 782/909] changelog-d: Use roberth fork with markdown frontmatter support --- flake.nix | 2 +- misc/changelog-d.cabal.nix | 31 +++++++++++++++++++++++++++++++ misc/changelog-d.nix | 31 +++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 misc/changelog-d.cabal.nix create mode 100644 misc/changelog-d.nix diff --git a/flake.nix b/flake.nix index dbd45f053..90203e7d5 100644 --- a/flake.nix +++ b/flake.nix @@ -182,7 +182,7 @@ "--enable-internal-api-docs" ]; - changelog-d = pkgs.buildPackages.changelog-d; + changelog-d = pkgs.buildPackages.callPackage ./misc/changelog-d.nix { }; nativeBuildDeps = [ diff --git a/misc/changelog-d.cabal.nix b/misc/changelog-d.cabal.nix new file mode 100644 index 000000000..76f9353cd --- /dev/null +++ b/misc/changelog-d.cabal.nix @@ -0,0 +1,31 @@ +{ mkDerivation, aeson, base, bytestring, cabal-install-parsers +, Cabal-syntax, containers, directory, filepath, frontmatter +, generic-lens-lite, lib, mtl, optparse-applicative, parsec, pretty +, regex-applicative, text, pkgs +}: +let rev = "f30f6969e9cd8b56242309639d58acea21c99d06"; +in +mkDerivation { + pname = "changelog-d"; + version = "0.1"; + src = pkgs.fetchurl { + name = "changelog-d-${rev}.tar.gz"; + url = "https://codeberg.org/roberth/changelog-d/archive/${rev}.tar.gz"; + hash = "sha256-8a2+i5u7YoszAgd5OIEW0eYUcP8yfhtoOIhLJkylYJ4="; + } // { inherit rev; }; + isLibrary = false; + isExecutable = true; + libraryHaskellDepends = [ + aeson base bytestring cabal-install-parsers Cabal-syntax containers + directory filepath frontmatter generic-lens-lite mtl parsec pretty + regex-applicative text + ]; + executableHaskellDepends = [ + base bytestring Cabal-syntax directory filepath + optparse-applicative + ]; + doHaddock = false; + description = "Concatenate changelog entries into a single one"; + license = lib.licenses.gpl3Plus; + mainProgram = "changelog-d"; +} diff --git a/misc/changelog-d.nix b/misc/changelog-d.nix new file mode 100644 index 000000000..1b20f4596 --- /dev/null +++ b/misc/changelog-d.nix @@ -0,0 +1,31 @@ +# Taken temporarily from +{ + callPackage, + lib, + haskell, + haskellPackages, +}: + +let + hsPkg = haskellPackages.callPackage ./changelog-d.cabal.nix { }; + + addCompletions = haskellPackages.generateOptparseApplicativeCompletions ["changelog-d"]; + + haskellModifications = + lib.flip lib.pipe [ + addCompletions + haskell.lib.justStaticExecutables + ]; + + mkDerivationOverrides = finalAttrs: oldAttrs: { + + version = oldAttrs.version + "-git-${lib.strings.substring 0 7 oldAttrs.src.rev}"; + + meta = oldAttrs.meta // { + homepage = "https://codeberg.org/roberth/changelog-d"; + maintainers = [ lib.maintainers.roberth ]; + }; + + }; +in + (haskellModifications hsPkg).overrideAttrs mkDerivationOverrides From 3811b334c646bc3b4bf8caef6d13c9f5027246f1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 9 Dec 2023 19:51:20 +0100 Subject: [PATCH 783/909] rl-next: Use markdown frontmatter syntax The old syntax is still supported, as long as you don't use a { in the description - the reason to migrate. --- doc/manual/rl-next/hash-format-nix32.md | 5 +++-- doc/manual/rl-next/mounted-ssh-store.md | 9 ++++----- doc/manual/rl-next/nix-config-show.md | 11 +++++------ doc/manual/rl-next/nix-env-json-drv-path.md | 9 +++------ doc/manual/rl-next/nix-hash-convert.md | 6 +++--- doc/manual/rl-next/source-positions-in-errors.md | 9 ++++----- doc/manual/src/contributing/hacking.md | 9 ++++----- 7 files changed, 26 insertions(+), 32 deletions(-) diff --git a/doc/manual/rl-next/hash-format-nix32.md b/doc/manual/rl-next/hash-format-nix32.md index 20c557da9..73e6fbb24 100644 --- a/doc/manual/rl-next/hash-format-nix32.md +++ b/doc/manual/rl-next/hash-format-nix32.md @@ -1,6 +1,7 @@ +--- synopsis: Rename hash format `base32` to `nix32` -prs: #9452 -description: { +prs: 9452 +--- Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for [Base32](https://en.wikipedia.org/wiki/Base32). diff --git a/doc/manual/rl-next/mounted-ssh-store.md b/doc/manual/rl-next/mounted-ssh-store.md index 39fac5283..6df44dbb6 100644 --- a/doc/manual/rl-next/mounted-ssh-store.md +++ b/doc/manual/rl-next/mounted-ssh-store.md @@ -1,9 +1,8 @@ +--- synopsis: Mounted SSH Store -issues: #7890 -prs: #7912 -description: { +issues: 7890 +prs: 7912 +--- Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md). This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem. - -} diff --git a/doc/manual/rl-next/nix-config-show.md b/doc/manual/rl-next/nix-config-show.md index b2ad3c666..26b961b76 100644 --- a/doc/manual/rl-next/nix-config-show.md +++ b/doc/manual/rl-next/nix-config-show.md @@ -1,8 +1,7 @@ -synopsis: `nix config show` -issues: #7672 -prs: #9477 -description: { +--- +synopsis: Rename to `nix config show` +issues: 7672 +prs: 9477 +--- `nix show-config` was renamed to `nix config show`, and `nix doctor` was renamed to `nix config check`, to be more consistent with the rest of the command-line interface. - -} diff --git a/doc/manual/rl-next/nix-env-json-drv-path.md b/doc/manual/rl-next/nix-env-json-drv-path.md index fbe2b67d8..734cefd1b 100644 --- a/doc/manual/rl-next/nix-env-json-drv-path.md +++ b/doc/manual/rl-next/nix-env-json-drv-path.md @@ -1,9 +1,6 @@ +--- synopsis: Fix `nix-env --query --drv-path --json` -prs: #9257 -description: { +prs: 9257 +--- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set. - -} - - diff --git a/doc/manual/rl-next/nix-hash-convert.md b/doc/manual/rl-next/nix-hash-convert.md index de4367c5b..2b718a66b 100644 --- a/doc/manual/rl-next/nix-hash-convert.md +++ b/doc/manual/rl-next/nix-hash-convert.md @@ -1,6 +1,7 @@ +--- synopsis: Add `nix hash convert` -prs: #9452 -description: { +prs: 9452 +--- New [`nix hash convert`](https://github.com/NixOS/nix/issues/8876) sub command with a fast track to stabilization! Examples: @@ -44,4 +45,3 @@ The following commands are still available but will emit a deprecation warning. - `nix hash to-base64 $hash1 $hash2`: Use `nix hash convert --to base64 $hash1 $hash2` instead. - `nix hash to-sri $hash1 $hash2`: : Use `nix hash convert --to sri $hash1 $hash2` or even just `nix hash convert $hash1 $hash2` instead. -} diff --git a/doc/manual/rl-next/source-positions-in-errors.md b/doc/manual/rl-next/source-positions-in-errors.md index 15df884ea..5b210289d 100644 --- a/doc/manual/rl-next/source-positions-in-errors.md +++ b/doc/manual/rl-next/source-positions-in-errors.md @@ -1,7 +1,8 @@ +--- synopsis: Source locations are printed more consistently in errors -issues: #561 -prs: #9555 -description: { +issues: 561 +prs: 9555 +--- Source location information is now included in error messages more consistently. Given this code: @@ -39,5 +40,3 @@ error: error: value is a set while a string was expected ``` - -} diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 9de5ad39b..237eff925 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -257,17 +257,16 @@ User-visible changes should come with a release note. Here's what a complete entry looks like. The file name is not incorporated in the document. ``` +--- synopsis: Basically a title -issues: #1234 -prs: #1238 -description: { +issues: 1234 +prs: 1238 +--- Here's one or more paragraphs that describe the change. - It's markdown - Add references to the manual using @docroot@ - -} ``` Significant changes should add the following header, which moves them to the top. From a856f603ed5a124f7eb818dadab6c88da73570fb Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 9 Dec 2023 19:55:47 +0100 Subject: [PATCH 784/909] Add checks.rl-next --- flake.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/flake.nix b/flake.nix index 90203e7d5..f499b0a9b 100644 --- a/flake.nix +++ b/flake.nix @@ -691,6 +691,11 @@ perlBindings = self.hydraJobs.perlBindings.${system}; installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; + rl-next = + let pkgs = nixpkgsFor.${system}.native; + in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' + LANG=C.UTF-8 ${(commonDeps { inherit pkgs; }).changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out + ''; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; }); From a63be6578f7e17182fdec8e3d3fdbab19a814152 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 9 Dec 2023 21:22:20 +0100 Subject: [PATCH 785/909] flake.nix: Cache shell inputs through hydra --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index f499b0a9b..99480183a 100644 --- a/flake.nix +++ b/flake.nix @@ -540,6 +540,8 @@ # Binary package for various platforms. build = forAllSystems (system: self.packages.${system}.nix); + shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); + buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); buildCross = forAllCrossSystems (crossSystem: From 3c200da242d8f0ccda447866028bb757e0b0bbd9 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sun, 10 Dec 2023 06:16:32 +0100 Subject: [PATCH 786/909] document `fetchTree` (#9258) * document `fetchTree` * display experimental feature note at the top we have to enable the new `fetchTree` experimental feature to render it at all. this was a bug introduced when adding that new feature flag. Co-authored-by: tomberek Co-authored-by: Robert Hensing Co-authored-by: Silvan Mosberger --- doc/manual/generate-builtins.nix | 13 +- doc/manual/generate-settings.nix | 4 +- doc/manual/generate-store-info.nix | 4 +- src/libexpr/primops/fetchTree.cc | 242 ++++++++++++++++++++++++----- src/nix/main.cc | 1 + 5 files changed, 221 insertions(+), 43 deletions(-) diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 05cae1c46..007b698f1 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -8,7 +8,15 @@ let showBuiltin = name: { doc, args, arity, experimental-feature }: let experimentalNotice = optionalString (experimental-feature != null) '' - This function is only available if the [${experimental-feature}](@docroot@/contributing/experimental-features.md#xp-feature-${experimental-feature}) experimental feature is enabled. + > **Note** + > + > This function is only available if the [`${experimental-feature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimental-feature}) is enabled. + > + > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): + > + > ``` + > extra-experimental-features = ${experimental-feature} + > ``` ''; in squash '' @@ -17,10 +25,9 @@ let
- ${doc} - ${experimentalNotice} + ${doc}
''; listArgs = args: concatStringsSep " " (map (s: "${s}") args); diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 74446b70b..504cda362 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -20,10 +20,10 @@ let else "`${setting}`"; # separate body to cleanly handle indentation body = '' - ${description} - ${experimentalFeatureNote} + ${description} + **Default:** ${showDefault documentDefault defaultValue} ${showAliases aliases} diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index 57247a181..c311c3c39 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -19,10 +19,10 @@ let result = squash '' # ${name} - ${doc} - ${experimentalFeatureNote} + ${doc} + ## Settings ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings} diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 15f870a95..eb2df8626 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -187,45 +187,215 @@ static RegisterPrimOp primop_fetchTree({ .name = "fetchTree", .args = {"input"}, .doc = R"( - Fetch a source tree or a plain file using one of the supported backends. - *input* must be a [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references), either in attribute set representation or in the URL-like syntax. - The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is enabled. + Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with: - > **Note** + - the resulting fixed-output [store path](@docroot@/glossary.md#gloss-store-path) + - the corresponding [NAR](@docroot@/glossary.md#gloss-nar) hash + - backend-specific metadata (currently not documented). + + *input* must be an attribute set with the following attributes: + + - `type` (String, required) + + One of the [supported source types](#source-types). + This determines other required and allowed input attributes. + + - `narHash` (String, optional) + + The `narHash` parameter can be used to substitute the source of the tree. + It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism. + If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available. + + A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again. + That is, `fetchTree` is idempotent. + + Downloads are cached in `$XDG_CACHE_HOME/nix`. + The remote source will be fetched from the network if both are true: + - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store + + > **Note** + > + > [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching. + + - There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) + + ## Source types + + The following source types and associated input attributes are supported. + + + + - `"file"` + + Place a plain file into the Nix store. + This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl) + + - `url` (String, required) + + Supported protocols: + + - `https` + + > **Example** + > + > ```nix + > fetchTree { + > type = "file"; + > url = "https://example.com/index.html"; + > } + > ``` + + - `http` + + Insecure HTTP transfer for legacy sources. + + > **Warning** + > + > HTTP performs no encryption or authentication. + > Use a `narHash` known in advance to ensure the output has expected contents. + + - `file` + + A file on the local file system. + + > **Example** + > + > ```nix + > fetchTree { + > type = "file"; + > url = "file:///home/eelco/nix/README.md"; + > } + > ``` + + - `"tarball"` + + Download a tar archive and extract it into the Nix store. + This has the same underyling implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball) + + - `url` (String, required) + + > **Example** + > + > ```nix + > fetchTree { + > type = "tarball"; + > url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11"; + > } + > ``` + + - `"git"` + + Fetch a Git tree and copy it to the Nix store. + This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit). + + - `url` (String, required) + + The URL formats supported are the same as for Git itself. + + > **Example** + > + > ```nix + > fetchTree { + > type = "git"; + > url = "git@github.com:NixOS/nixpkgs.git"; + > } + > ``` + + > **Note** + > + > If the URL points to a local directory, and no `ref` or `rev` is given, Nix will only consider files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory. + + - `ref` (String, optional) + + A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name. + + Default: `"HEAD"` + + - `rev` (String, optional) + + A Git revision; a commit hash. + + Default: the tip of `ref` + + - `shallow` (Bool, optional) + + Make a shallow clone when fetching the Git tree. + + Default: `false` + + - `submodules` (Bool, optional) + + Also fetch submodules if available. + + Default: `false` + + - `allRefs` (Bool, optional) + + If set to `true`, always fetch the entire repository, even if the latest commit is still in the cache. + Otherwise, only the latest commit is fetched if it is not already cached. + + Default: `false` + + - `lastModified` (Integer, optional) + + Unix timestamp of the fetched commit. + + If set, pass through the value to the output attribute set. + Otherwise, generated from the fetched Git tree. + + - `revCount` (Integer, optional) + + Number of revisions in the history of the Git repository before the fetched commit. + + If set, pass through the value to the output attribute set. + Otherwise, generated from the fetched Git tree. + + The following input types are still subject to change: + + - `"path"` + - `"github"` + - `"gitlab"` + - `"sourcehut"` + - `"mercurial"` + + *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references). + The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. + + > **Example** > - > The URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. + > Fetch a GitHub repository using the attribute set representation: + > + > ```nix + > builtins.fetchTree { + > type = "github"; + > owner = "NixOS"; + > repo = "nixpkgs"; + > rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; + > } + > ``` + > + > This evaluates to the following attribute set: + > + > ```nix + > { + > lastModified = 1686503798; + > lastModifiedDate = "20230611171638"; + > narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc="; + > outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source"; + > rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; + > shortRev = "ae2e6b3"; + > } + > ``` - Here are some examples of how to use `fetchTree`: - - - Fetch a GitHub repository using the attribute set representation: - - ```nix - builtins.fetchTree { - type = "github"; - owner = "NixOS"; - repo = "nixpkgs"; - rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; - } - ``` - - This evaluates to the following attribute set: - - ``` - { - lastModified = 1686503798; - lastModifiedDate = "20230611171638"; - narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc="; - outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source"; - rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; - shortRev = "ae2e6b3"; - } - ``` - - - Fetch the same GitHub repository using the URL-like syntax: - - ``` - builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd" - ``` + > **Example** + > + > Fetch the same GitHub repository using the URL-like syntax: + > + > ```nix + > builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd" + > ``` )", .fun = prim_fetchTree, .experimentalFeature = Xp::FetchTree, diff --git a/src/nix/main.cc b/src/nix/main.cc index 109d2cc04..39c04069b 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -373,6 +373,7 @@ void mainWrapped(int argc, char * * argv) Xp::Flakes, Xp::FetchClosure, Xp::DynamicDerivations, + Xp::FetchTree, }; evalSettings.pureEval = false; EvalState state({}, openStore("dummy://")); From deadb3bfe9cde3e78e8e89340e4c92499069461a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 10 Dec 2023 14:28:14 -0500 Subject: [PATCH 787/909] Create header for `LegacySSHStore` In https://github.com/NixOS/nix/pull/6134#issuecomment-1079199888, @thuffschmitt proposed exposing `LegacySSHStore` in Nix for deduplication with Hydra, at least temporarily. I think that is a good idea. Note that the diff will look bad unless one ignores whitespace! Also try this locally: ```shell-session git diff --ignore-all-space HEAD^:src/libstore/legacy-ssh-store.cc HEAD:src/libstore/legacy-ssh-store.cc git diff --ignore-all-space HEAD^:src/libstore/legacy-ssh-store.cc HEAD:src/libstore/legacy-ssh-store.hh ``` --- src/libstore/legacy-ssh-store.cc | 726 ++++++++++++++----------------- src/libstore/legacy-ssh-store.hh | 132 ++++++ 2 files changed, 466 insertions(+), 392 deletions(-) create mode 100644 src/libstore/legacy-ssh-store.hh diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 8ef2daa7b..06bef9d08 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -1,3 +1,4 @@ +#include "legacy-ssh-store.hh" #include "ssh-store-config.hh" #include "archive.hh" #include "pool.hh" @@ -13,414 +14,355 @@ namespace nix { -struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig +std::string LegacySSHStoreConfig::doc() { - using CommonSSHStoreConfig::CommonSSHStoreConfig; + return + #include "legacy-ssh-store.md" + ; +} - const Setting remoteProgram{this, "nix-store", "remote-program", - "Path to the `nix-store` executable on the remote machine."}; - const Setting maxConnections{this, 1, "max-connections", - "Maximum number of concurrent SSH connections."}; - - const std::string name() override { return "SSH Store"; } - - std::string doc() override - { - return - #include "legacy-ssh-store.md" - ; - } -}; - -struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store +struct LegacySSHStore::Connection { - // Hack for getting remote build log output. - // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in - // the documentation - const Setting logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; - - struct Connection - { - std::unique_ptr sshConn; - FdSink to; - FdSource from; - ServeProto::Version remoteVersion; - bool good = true; - - /** - * Coercion to `ServeProto::ReadConn`. This makes it easy to use the - * factored out serve protocol searlizers with a - * `LegacySSHStore::Connection`. - * - * The serve protocol connection types are unidirectional, unlike - * this type. - */ - operator ServeProto::ReadConn () - { - return ServeProto::ReadConn { - .from = from, - .version = remoteVersion, - }; - } - - /* - * Coercion to `ServeProto::WriteConn`. This makes it easy to use the - * factored out serve protocol searlizers with a - * `LegacySSHStore::Connection`. - * - * The serve protocol connection types are unidirectional, unlike - * this type. - */ - operator ServeProto::WriteConn () - { - return ServeProto::WriteConn { - .to = to, - .version = remoteVersion, - }; - } - }; - - std::string host; - - ref> connections; - - SSHMaster master; - - static std::set uriSchemes() { return {"ssh"}; } - - LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params) - : StoreConfig(params) - , CommonSSHStoreConfig(params) - , LegacySSHStoreConfig(params) - , Store(params) - , host(host) - , connections(make_ref>( - std::max(1, (int) maxConnections), - [this]() { return openConnection(); }, - [](const ref & r) { return r->good; } - )) - , master( - host, - sshKey, - sshPublicHostKey, - // Use SSH master only if using more than 1 connection. - connections->capacity() > 1, - compress, - logFD) - { - } - - ref openConnection() - { - auto conn = make_ref(); - conn->sshConn = master.startCommand( - fmt("%s --serve --write", remoteProgram) - + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); - conn->to = FdSink(conn->sshConn->in.get()); - conn->from = FdSource(conn->sshConn->out.get()); - - try { - conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION; - conn->to.flush(); - - StringSink saved; - try { - TeeSource tee(conn->from, saved); - unsigned int magic = readInt(tee); - if (magic != SERVE_MAGIC_2) - throw Error("'nix-store --serve' protocol mismatch from '%s'", host); - } catch (SerialisationError & e) { - /* In case the other side is waiting for our input, - close it. */ - conn->sshConn->in.close(); - auto msg = conn->from.drain(); - throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'", - host, chomp(saved.s + msg)); - } - conn->remoteVersion = readInt(conn->from); - if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) - throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host); - - } catch (EndOfFile & e) { - throw Error("cannot connect to '%1%'", host); - } - - return conn; - }; - - std::string getUri() override - { - return *uriSchemes().begin() + "://" + host; - } - - void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override - { - try { - auto conn(connections->get()); - - /* No longer support missing NAR hash */ - assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4); - - debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); - - conn->to << ServeProto::Command::QueryPathInfos << PathSet{printStorePath(path)}; - conn->to.flush(); - - auto p = readString(conn->from); - if (p.empty()) return callback(nullptr); - auto path2 = parseStorePath(p); - assert(path == path2); - auto info = std::make_shared( - path, - ServeProto::Serialise::read(*this, *conn)); - - if (info->narHash == Hash::dummy) - throw Error("NAR hash is now mandatory"); - - auto s = readString(conn->from); - assert(s == ""); - - callback(std::move(info)); - } catch (...) { callback.rethrow(); } - } - - void addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs) override - { - debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); - - auto conn(connections->get()); - - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) { - - conn->to - << ServeProto::Command::AddToStoreNar - << printStorePath(info.path) - << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(HashFormat::Base16, false); - ServeProto::write(*this, *conn, info.references); - conn->to - << info.registrationTime - << info.narSize - << info.ultimate - << info.sigs - << renderContentAddress(info.ca); - try { - copyNAR(source, conn->to); - } catch (...) { - conn->good = false; - throw; - } - conn->to.flush(); - - } else { - - conn->to - << ServeProto::Command::ImportPaths - << 1; - try { - copyNAR(source, conn->to); - } catch (...) { - conn->good = false; - throw; - } - conn->to - << exportMagic - << printStorePath(info.path); - ServeProto::write(*this, *conn, info.references); - conn->to - << (info.deriver ? printStorePath(*info.deriver) : "") - << 0 - << 0; - conn->to.flush(); - - } - - if (readInt(conn->from) != 1) - throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); - } - - void narFromPath(const StorePath & path, Sink & sink) override - { - auto conn(connections->get()); - - conn->to << ServeProto::Command::DumpStorePath << printStorePath(path); - conn->to.flush(); - copyNAR(conn->from, sink); - } - - std::optional queryPathFromHashPart(const std::string & hashPart) override - { unsupported("queryPathFromHashPart"); } - - StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashAlgorithm hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override - { unsupported("addToStore"); } - - StorePath addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) override - { unsupported("addTextToStore"); } - -private: - - void putBuildSettings(Connection & conn) - { - ServeProto::write(*this, conn, ServeProto::BuildOptions { - .maxSilentTime = settings.maxSilentTime, - .buildTimeout = settings.buildTimeout, - .maxLogSize = settings.maxLogSize, - .nrRepeats = 0, // buildRepeat hasn't worked for ages anyway - .enforceDeterminism = 0, - .keepFailed = settings.keepFailed, - }); - } - -public: - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override - { - auto conn(connections->get()); - - conn->to - << ServeProto::Command::BuildDerivation - << printStorePath(drvPath); - writeDerivation(conn->to, *this, drv); - - putBuildSettings(*conn); - - conn->to.flush(); - - return ServeProto::Serialise::read(*this, *conn); - } - - void buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) override - { - if (evalStore && evalStore.get() != this) - throw Error("building on an SSH store is incompatible with '--eval-store'"); - - auto conn(connections->get()); - - conn->to << ServeProto::Command::BuildPaths; - Strings ss; - for (auto & p : drvPaths) { - auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); - std::visit(overloaded { - [&](const StorePathWithOutputs & s) { - ss.push_back(s.to_string(*this)); - }, - [&](const StorePath & drvPath) { - throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath)); - }, - [&](std::monostate) { - throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://"); - }, - }, sOrDrvPath); - } - conn->to << ss; - - putBuildSettings(*conn); - - conn->to.flush(); - - BuildResult result; - result.status = (BuildResult::Status) readInt(conn->from); - - if (!result.success()) { - conn->from >> result.errorMsg; - throw Error(result.status, result.errorMsg); - } - } - - void ensurePath(const StorePath & path) override - { unsupported("ensurePath"); } - - virtual ref getFSAccessor(bool requireValidPath) override - { unsupported("getFSAccessor"); } + std::unique_ptr sshConn; + FdSink to; + FdSource from; + ServeProto::Version remoteVersion; + bool good = true; /** - * The default instance would schedule the work on the client side, but - * for consistency with `buildPaths` and `buildDerivation` it should happen - * on the remote side. + * Coercion to `ServeProto::ReadConn`. This makes it easy to use the + * factored out serve protocol searlizers with a + * `LegacySSHStore::Connection`. * - * We make this fail for now so we can add implement this properly later - * without it being a breaking change. + * The serve protocol connection types are unidirectional, unlike + * this type. */ - void repairPath(const StorePath & path) override - { unsupported("repairPath"); } - - void computeFSClosure(const StorePathSet & paths, - StorePathSet & out, bool flipDirection = false, - bool includeOutputs = false, bool includeDerivers = false) override + operator ServeProto::ReadConn () { - if (flipDirection || includeDerivers) { - Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers); - return; - } - - auto conn(connections->get()); - - conn->to - << ServeProto::Command::QueryClosure - << includeOutputs; - ServeProto::write(*this, *conn, paths); - conn->to.flush(); - - for (auto & i : ServeProto::Serialise::read(*this, *conn)) - out.insert(i); + return ServeProto::ReadConn { + .from = from, + .version = remoteVersion, + }; } - StorePathSet queryValidPaths(const StorePathSet & paths, - SubstituteFlag maybeSubstitute = NoSubstitute) override - { - auto conn(connections->get()); - - conn->to - << ServeProto::Command::QueryValidPaths - << false // lock - << maybeSubstitute; - ServeProto::write(*this, *conn, paths); - conn->to.flush(); - - return ServeProto::Serialise::read(*this, *conn); - } - - void connect() override - { - auto conn(connections->get()); - } - - unsigned int getProtocol() override - { - auto conn(connections->get()); - return conn->remoteVersion; - } - - /** - * The legacy ssh protocol doesn't support checking for trusted-user. - * Try using ssh-ng:// instead if you want to know. + /* + * Coercion to `ServeProto::WriteConn`. This makes it easy to use the + * factored out serve protocol searlizers with a + * `LegacySSHStore::Connection`. + * + * The serve protocol connection types are unidirectional, unlike + * this type. */ - std::optional isTrustedClient() override + operator ServeProto::WriteConn () { - return std::nullopt; + return ServeProto::WriteConn { + .to = to, + .version = remoteVersion, + }; } - - void queryRealisationUncached(const DrvOutput &, - Callback> callback) noexcept override - // TODO: Implement - { unsupported("queryRealisation"); } }; + +LegacySSHStore::LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params) + : StoreConfig(params) + , CommonSSHStoreConfig(params) + , LegacySSHStoreConfig(params) + , Store(params) + , host(host) + , connections(make_ref>( + std::max(1, (int) maxConnections), + [this]() { return openConnection(); }, + [](const ref & r) { return r->good; } + )) + , master( + host, + sshKey, + sshPublicHostKey, + // Use SSH master only if using more than 1 connection. + connections->capacity() > 1, + compress, + logFD) +{ +} + + +ref LegacySSHStore::openConnection() +{ + auto conn = make_ref(); + conn->sshConn = master.startCommand( + fmt("%s --serve --write", remoteProgram) + + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); + conn->to = FdSink(conn->sshConn->in.get()); + conn->from = FdSource(conn->sshConn->out.get()); + + try { + conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION; + conn->to.flush(); + + StringSink saved; + try { + TeeSource tee(conn->from, saved); + unsigned int magic = readInt(tee); + if (magic != SERVE_MAGIC_2) + throw Error("'nix-store --serve' protocol mismatch from '%s'", host); + } catch (SerialisationError & e) { + /* In case the other side is waiting for our input, + close it. */ + conn->sshConn->in.close(); + auto msg = conn->from.drain(); + throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'", + host, chomp(saved.s + msg)); + } + conn->remoteVersion = readInt(conn->from); + if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) + throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host); + + } catch (EndOfFile & e) { + throw Error("cannot connect to '%1%'", host); + } + + return conn; +}; + + +std::string LegacySSHStore::getUri() +{ + return *uriSchemes().begin() + "://" + host; +} + + +void LegacySSHStore::queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept +{ + try { + auto conn(connections->get()); + + /* No longer support missing NAR hash */ + assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4); + + debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); + + conn->to << ServeProto::Command::QueryPathInfos << PathSet{printStorePath(path)}; + conn->to.flush(); + + auto p = readString(conn->from); + if (p.empty()) return callback(nullptr); + auto path2 = parseStorePath(p); + assert(path == path2); + auto info = std::make_shared( + path, + ServeProto::Serialise::read(*this, *conn)); + + if (info->narHash == Hash::dummy) + throw Error("NAR hash is now mandatory"); + + auto s = readString(conn->from); + assert(s == ""); + + callback(std::move(info)); + } catch (...) { callback.rethrow(); } +} + + +void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs) +{ + debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); + + auto conn(connections->get()); + + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) { + + conn->to + << ServeProto::Command::AddToStoreNar + << printStorePath(info.path) + << (info.deriver ? printStorePath(*info.deriver) : "") + << info.narHash.to_string(HashFormat::Base16, false); + ServeProto::write(*this, *conn, info.references); + conn->to + << info.registrationTime + << info.narSize + << info.ultimate + << info.sigs + << renderContentAddress(info.ca); + try { + copyNAR(source, conn->to); + } catch (...) { + conn->good = false; + throw; + } + conn->to.flush(); + + } else { + + conn->to + << ServeProto::Command::ImportPaths + << 1; + try { + copyNAR(source, conn->to); + } catch (...) { + conn->good = false; + throw; + } + conn->to + << exportMagic + << printStorePath(info.path); + ServeProto::write(*this, *conn, info.references); + conn->to + << (info.deriver ? printStorePath(*info.deriver) : "") + << 0 + << 0; + conn->to.flush(); + + } + + if (readInt(conn->from) != 1) + throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); +} + + +void LegacySSHStore::narFromPath(const StorePath & path, Sink & sink) +{ + auto conn(connections->get()); + + conn->to << ServeProto::Command::DumpStorePath << printStorePath(path); + conn->to.flush(); + copyNAR(conn->from, sink); +} + + +void LegacySSHStore::putBuildSettings(Connection & conn) +{ + ServeProto::write(*this, conn, ServeProto::BuildOptions { + .maxSilentTime = settings.maxSilentTime, + .buildTimeout = settings.buildTimeout, + .maxLogSize = settings.maxLogSize, + .nrRepeats = 0, // buildRepeat hasn't worked for ages anyway + .enforceDeterminism = 0, + .keepFailed = settings.keepFailed, + }); +} + + +BuildResult LegacySSHStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode) +{ + auto conn(connections->get()); + + conn->to + << ServeProto::Command::BuildDerivation + << printStorePath(drvPath); + writeDerivation(conn->to, *this, drv); + + putBuildSettings(*conn); + + conn->to.flush(); + + return ServeProto::Serialise::read(*this, *conn); +} + + +void LegacySSHStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) +{ + if (evalStore && evalStore.get() != this) + throw Error("building on an SSH store is incompatible with '--eval-store'"); + + auto conn(connections->get()); + + conn->to << ServeProto::Command::BuildPaths; + Strings ss; + for (auto & p : drvPaths) { + auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); + std::visit(overloaded { + [&](const StorePathWithOutputs & s) { + ss.push_back(s.to_string(*this)); + }, + [&](const StorePath & drvPath) { + throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath)); + }, + [&](std::monostate) { + throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://"); + }, + }, sOrDrvPath); + } + conn->to << ss; + + putBuildSettings(*conn); + + conn->to.flush(); + + BuildResult result; + result.status = (BuildResult::Status) readInt(conn->from); + + if (!result.success()) { + conn->from >> result.errorMsg; + throw Error(result.status, result.errorMsg); + } +} + + +void LegacySSHStore::computeFSClosure(const StorePathSet & paths, + StorePathSet & out, bool flipDirection, + bool includeOutputs, bool includeDerivers) +{ + if (flipDirection || includeDerivers) { + Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers); + return; + } + + auto conn(connections->get()); + + conn->to + << ServeProto::Command::QueryClosure + << includeOutputs; + ServeProto::write(*this, *conn, paths); + conn->to.flush(); + + for (auto & i : ServeProto::Serialise::read(*this, *conn)) + out.insert(i); +} + + +StorePathSet LegacySSHStore::queryValidPaths(const StorePathSet & paths, + SubstituteFlag maybeSubstitute) +{ + auto conn(connections->get()); + + conn->to + << ServeProto::Command::QueryValidPaths + << false // lock + << maybeSubstitute; + ServeProto::write(*this, *conn, paths); + conn->to.flush(); + + return ServeProto::Serialise::read(*this, *conn); +} + + +void LegacySSHStore::connect() +{ + auto conn(connections->get()); +} + + +unsigned int LegacySSHStore::getProtocol() +{ + auto conn(connections->get()); + return conn->remoteVersion; +} + + +/** + * The legacy ssh protocol doesn't support checking for trusted-user. + * Try using ssh-ng:// instead if you want to know. + */ +std::optional isTrustedClient() +{ + return std::nullopt; +} + + static RegisterStoreImplementation regLegacySSHStore; } diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh new file mode 100644 index 000000000..c40c256bb --- /dev/null +++ b/src/libstore/legacy-ssh-store.hh @@ -0,0 +1,132 @@ +#pragma once +///@file + +#include "ssh-store-config.hh" +#include "store-api.hh" +#include "ssh.hh" +#include "callback.hh" +#include "pool.hh" + +namespace nix { + +struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig +{ + using CommonSSHStoreConfig::CommonSSHStoreConfig; + + const Setting remoteProgram{this, "nix-store", "remote-program", + "Path to the `nix-store` executable on the remote machine."}; + + const Setting maxConnections{this, 1, "max-connections", + "Maximum number of concurrent SSH connections."}; + + const std::string name() override { return "SSH Store"; } + + std::string doc() override; +}; + +struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store +{ + // Hack for getting remote build log output. + // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in + // the documentation + const Setting logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; + + struct Connection; + + std::string host; + + ref> connections; + + SSHMaster master; + + static std::set uriSchemes() { return {"ssh"}; } + + LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params); + + ref openConnection(); + + std::string getUri() override; + + void queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept override; + + void addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs) override; + + void narFromPath(const StorePath & path, Sink & sink) override; + + std::optional queryPathFromHashPart(const std::string & hashPart) override + { unsupported("queryPathFromHashPart"); } + + StorePath addToStore( + std::string_view name, + const Path & srcPath, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) override + { unsupported("addToStore"); } + + StorePath addTextToStore( + std::string_view name, + std::string_view s, + const StorePathSet & references, + RepairFlag repair) override + { unsupported("addTextToStore"); } + +private: + + void putBuildSettings(Connection & conn); + +public: + + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode) override; + + void buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) override; + + void ensurePath(const StorePath & path) override + { unsupported("ensurePath"); } + + virtual ref getFSAccessor(bool requireValidPath) override + { unsupported("getFSAccessor"); } + + /** + * The default instance would schedule the work on the client side, but + * for consistency with `buildPaths` and `buildDerivation` it should happen + * on the remote side. + * + * We make this fail for now so we can add implement this properly later + * without it being a breaking change. + */ + void repairPath(const StorePath & path) override + { unsupported("repairPath"); } + + void computeFSClosure(const StorePathSet & paths, + StorePathSet & out, bool flipDirection = false, + bool includeOutputs = false, bool includeDerivers = false) override; + + StorePathSet queryValidPaths(const StorePathSet & paths, + SubstituteFlag maybeSubstitute = NoSubstitute) override; + + void connect() override; + + unsigned int getProtocol() override; + + /** + * The legacy ssh protocol doesn't support checking for trusted-user. + * Try using ssh-ng:// instead if you want to know. + */ + std::optional isTrustedClient() override + { + return std::nullopt; + } + + void queryRealisationUncached(const DrvOutput &, + Callback> callback) noexcept override + // TODO: Implement + { unsupported("queryRealisation"); } +}; + +} From e43bb655feaa23977322d68278c88ac075eb2c41 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sun, 10 Dec 2023 13:58:35 -0800 Subject: [PATCH 788/909] libstore/daemon.cc: note trust model difference in readDerivation()s Below the comment added by this commit is a much longer comment followed by a trust check, both of which have confused me on at least two occasions. I figured it out once, forgot it, then had to ask @Ericson2314 to explain it, at which point I understood it again. I think this might confuse other people too, or maybe I will just forget it a third time. So let's add a comment. Farther down in the function is the following check: ``` if (!(drvType.isCA() || trusted)) throw Error("you are not privileged to build input-addressed derivations"); ``` This seems really strange at first. A key property of Nix is that you can compute the outpath of a derivation using the derivation (and its references-closure) without trusting anybody! The missing insight is that at this point in the code the builder doesn't necessarily have the references-closure of the derivation being built, and therefore needs to trust that the derivation's outPath is honest. It's incredibly easy to overlook this, because the only difference between these two cases is which of these identically-named functions we used: - `readDerivation(Source,Store)` - `Store::readDerivation()` These functions have different trust models (except in the special case where the first function is used on the local store). We should call the reader's attention to this fact. Co-authored-by: Cole Helbling --- src/libstore/daemon.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 530b1a178..a112d6d31 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -574,6 +574,15 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::BuildDerivation: { auto drvPath = store->parseStorePath(readString(from)); BasicDerivation drv; + /* + * Note: unlike wopEnsurePath, this operation reads a + * derivation-to-be-realized from the client with + * readDerivation(Source,Store) rather than reading it from + * the local store with Store::readDerivation(). Since the + * derivation-to-be-realized is not registered in the store + * it cannot be trusted that its outPath was calculated + * correctly. + */ readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath)); BuildMode buildMode = (BuildMode) readInt(from); logger->startWork(); From 91ba7b230777e3fb023bda48c269d533702e50e8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Dec 2023 12:41:47 +0100 Subject: [PATCH 789/909] isAllowedURI: Extract function and test --- src/libexpr/eval.cc | 18 +++++-- src/libexpr/eval.hh | 5 ++ tests/unit/libexpr/eval.cc | 106 +++++++++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 5 deletions(-) create mode 100644 tests/unit/libexpr/eval.cc diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 9e494148e..0eb6f406e 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -599,21 +599,29 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & mkStorePathString(storePath, v); } -void EvalState::checkURI(const std::string & uri) +bool isAllowedURI(std::string_view uri, const Strings & allowedUris) { - if (!evalSettings.restrictEval) return; - /* 'uri' should be equal to a prefix, or in a subdirectory of a prefix. Thus, the prefix https://github.co does not permit access to https://github.com. Note: this allows 'http://' and 'https://' as prefixes for any http/https URI. */ - for (auto & prefix : evalSettings.allowedUris.get()) + for (auto & prefix : allowedUris) { if (uri == prefix || (uri.size() > prefix.size() && prefix.size() > 0 && hasPrefix(uri, prefix) && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/'))) - return; + return true; + } + + return false; +} + +void EvalState::checkURI(const std::string & uri) +{ + if (!evalSettings.restrictEval) return; + + if (isAllowedURI(uri, evalSettings.allowedUris.get())) return; /* If the URI is a path, then check it against allowedPaths as well. */ diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index f3f6d35b9..6008c3f60 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -837,6 +837,11 @@ std::string showType(const Value & v); */ SourcePath resolveExprPath(SourcePath path); +/** + * Whether a URI is allowed, assuming restrictEval is enabled + */ +bool isAllowedURI(std::string_view uri, const Strings & allowedPaths); + struct InvalidPathError : EvalError { Path path; diff --git a/tests/unit/libexpr/eval.cc b/tests/unit/libexpr/eval.cc new file mode 100644 index 000000000..cc5d6bbfa --- /dev/null +++ b/tests/unit/libexpr/eval.cc @@ -0,0 +1,106 @@ +#include +#include + +#include "eval.hh" +#include "tests/libexpr.hh" + +namespace nix { + +TEST(nix_isAllowedURI, http_example_com) { + Strings allowed; + allowed.push_back("http://example.com"); + + ASSERT_TRUE(isAllowedURI("http://example.com", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.co", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.como", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.org", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.org/foo", allowed)); +} + +TEST(nix_isAllowedURI, http_example_com_foo) { + Strings allowed; + allowed.push_back("http://example.com/foo"); + + ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.como", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.org/foo", allowed)); + // Broken? + // ASSERT_TRUE(isAllowedURI("http://example.com/foo?ok=1", allowed)); +} + +TEST(nix_isAllowedURI, http) { + Strings allowed; + allowed.push_back("http://"); + + ASSERT_TRUE(isAllowedURI("http://", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("https://", allowed)); + ASSERT_FALSE(isAllowedURI("http:foo", allowed)); +} + +TEST(nix_isAllowedURI, https) { + Strings allowed; + allowed.push_back("https://"); + + ASSERT_TRUE(isAllowedURI("https://example.com", allowed)); + ASSERT_TRUE(isAllowedURI("https://example.com/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com/https:", allowed)); +} + +TEST(nix_isAllowedURI, absolute_path) { + Strings allowed; + allowed.push_back("/var/evil"); // bad idea + + ASSERT_TRUE(isAllowedURI("/var/evil", allowed)); + ASSERT_TRUE(isAllowedURI("/var/evil/", allowed)); + ASSERT_TRUE(isAllowedURI("/var/evil/foo", allowed)); + ASSERT_TRUE(isAllowedURI("/var/evil/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evi", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com/var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil/foo", allowed)); +} + +TEST(nix_isAllowedURI, file_url) { + Strings allowed; + allowed.push_back("file:///var/evil"); // bad idea + + ASSERT_TRUE(isAllowedURI("file:///var/evil", allowed)); + ASSERT_TRUE(isAllowedURI("file:///var/evil/", allowed)); + ASSERT_TRUE(isAllowedURI("file:///var/evil/foo", allowed)); + ASSERT_TRUE(isAllowedURI("file:///var/evil/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evi", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com/var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http:///var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://var/evil/", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evi", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evilo", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evilo/", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evilo/foo", allowed)); + ASSERT_FALSE(isAllowedURI("file:///", allowed)); + ASSERT_FALSE(isAllowedURI("file://", allowed)); +} + +} // namespace nix \ No newline at end of file From 6cbba914a70eb5da6447fee5528a63723ed13245 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Dec 2023 12:43:20 +0100 Subject: [PATCH 790/909] isAllowedURI: Remove incorrect note --- src/libexpr/eval.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 0eb6f406e..d8a36fa02 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -603,8 +603,7 @@ bool isAllowedURI(std::string_view uri, const Strings & allowedUris) { /* 'uri' should be equal to a prefix, or in a subdirectory of a prefix. Thus, the prefix https://github.co does not permit - access to https://github.com. Note: this allows 'http://' and - 'https://' as prefixes for any http/https URI. */ + access to https://github.com. */ for (auto & prefix : allowedUris) { if (uri == prefix || (uri.size() > prefix.size() From 1fa958dda1ef0cb37441ef8d1a84faf6d501ac12 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Dec 2023 14:08:22 +0100 Subject: [PATCH 791/909] isAllowedURI: Format --- src/libexpr/eval.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d8a36fa02..9e541f293 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -605,11 +605,14 @@ bool isAllowedURI(std::string_view uri, const Strings & allowedUris) prefix. Thus, the prefix https://github.co does not permit access to https://github.com. */ for (auto & prefix : allowedUris) { - if (uri == prefix || - (uri.size() > prefix.size() - && prefix.size() > 0 - && hasPrefix(uri, prefix) - && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/'))) + if (uri == prefix + // Allow access to subdirectories of the prefix. + || (uri.size() > prefix.size() + && prefix.size() > 0 + && hasPrefix(uri, prefix) + && ( + prefix[prefix.size() - 1] == '/' + || uri[prefix.size()] == '/'))) return true; } From 79eb2920bb51c7ec9528a403986e79f04738e2be Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Dec 2023 14:39:49 +0100 Subject: [PATCH 792/909] Add nix::isASCII*, locale-independent --- src/libutil/string.hh | 17 +++++++++++ tests/unit/libutil/string.cc | 59 ++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 src/libutil/string.hh create mode 100644 tests/unit/libutil/string.cc diff --git a/src/libutil/string.hh b/src/libutil/string.hh new file mode 100644 index 000000000..16ef75643 --- /dev/null +++ b/src/libutil/string.hh @@ -0,0 +1,17 @@ +#pragma once + +namespace nix { + + /** Locale-independent version of std::islower(). */ + inline bool isASCIILower(char c) { return c >= 'a' && c <= 'z'; }; + + /** Locale-independent version of std::isupper(). */ + inline bool isASCIIUpper(char c) { return c >= 'A' && c <= 'Z'; }; + + /** Locale-independent version of std::isalpha(). */ + inline bool isASCIIAlpha(char c) { return isASCIILower(c) || isASCIIUpper(c); }; + + /** Locale-independent version of std::isdigit(). */ + inline bool isASCIIDigit(char c) { return c >= '0' && c <= '9'; }; + +} diff --git a/tests/unit/libutil/string.cc b/tests/unit/libutil/string.cc new file mode 100644 index 000000000..381f2cc15 --- /dev/null +++ b/tests/unit/libutil/string.cc @@ -0,0 +1,59 @@ +#include +#include "string.hh" + +namespace nix { + +TEST(string, isASCIILower) { + ASSERT_TRUE(isASCIILower('a')); + ASSERT_TRUE(isASCIILower('z')); + ASSERT_FALSE(isASCIILower('A')); + ASSERT_FALSE(isASCIILower('Z')); + ASSERT_FALSE(isASCIILower('0')); + ASSERT_FALSE(isASCIILower('9')); + ASSERT_FALSE(isASCIILower(' ')); + ASSERT_FALSE(isASCIILower('\n')); + ASSERT_FALSE(isASCIILower('\t')); + ASSERT_FALSE(isASCIILower(':')); +} + +TEST(string, isASCIIUpper) { + ASSERT_FALSE(isASCIIUpper('a')); + ASSERT_FALSE(isASCIIUpper('z')); + ASSERT_TRUE(isASCIIUpper('A')); + ASSERT_TRUE(isASCIIUpper('Z')); + ASSERT_FALSE(isASCIIUpper('0')); + ASSERT_FALSE(isASCIIUpper('9')); + ASSERT_FALSE(isASCIIUpper(' ')); + ASSERT_FALSE(isASCIIUpper('\n')); + ASSERT_FALSE(isASCIIUpper('\t')); + ASSERT_FALSE(isASCIIUpper(':')); +} + +TEST(string, isASCIIAlpha) { + ASSERT_TRUE(isASCIIAlpha('a')); + ASSERT_TRUE(isASCIIAlpha('z')); + ASSERT_TRUE(isASCIIAlpha('A')); + ASSERT_TRUE(isASCIIAlpha('Z')); + ASSERT_FALSE(isASCIIAlpha('0')); + ASSERT_FALSE(isASCIIAlpha('9')); + ASSERT_FALSE(isASCIIAlpha(' ')); + ASSERT_FALSE(isASCIIAlpha('\n')); + ASSERT_FALSE(isASCIIAlpha('\t')); + ASSERT_FALSE(isASCIIAlpha(':')); +} + +TEST(string, isASCIIDigit) { + ASSERT_FALSE(isASCIIDigit('a')); + ASSERT_FALSE(isASCIIDigit('z')); + ASSERT_FALSE(isASCIIDigit('A')); + ASSERT_FALSE(isASCIIDigit('Z')); + ASSERT_TRUE(isASCIIDigit('0')); + ASSERT_TRUE(isASCIIDigit('1')); + ASSERT_TRUE(isASCIIDigit('9')); + ASSERT_FALSE(isASCIIDigit(' ')); + ASSERT_FALSE(isASCIIDigit('\n')); + ASSERT_FALSE(isASCIIDigit('\t')); + ASSERT_FALSE(isASCIIDigit(':')); +} + +} \ No newline at end of file From d3a85b68347071d8d93ec796a38c707483d7b272 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Dec 2023 15:14:41 +0100 Subject: [PATCH 793/909] isValidSchemeName: Add function --- src/libutil/url.cc | 17 +++++++++++++++++ src/libutil/url.hh | 9 +++++++++ tests/unit/libutil/url.cc | 18 ++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 57b64d607..f2d5f1782 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -3,6 +3,7 @@ #include "util.hh" #include "split.hh" #include "canon-path.hh" +#include "string.hh" namespace nix { @@ -183,4 +184,20 @@ std::string fixGitURL(const std::string & url) } } +// https://www.rfc-editor.org/rfc/rfc3986#section-3.1 +bool isValidSchemeName(std::string_view s) +{ + if (s.empty()) return false; + if (!isASCIIAlpha(s[0])) return false; + for (auto c : s.substr(1)) { + if (isASCIIAlpha(c)) continue; + if (isASCIIDigit(c)) continue; + if (c == '+') continue; + if (c == '-') continue; + if (c == '.') continue; + return false; + } + return true; +} + } diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 833f54678..24806bbff 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -55,4 +55,13 @@ ParsedUrlScheme parseUrlScheme(std::string_view scheme); changes absolute paths into file:// URLs. */ std::string fixGitURL(const std::string & url); +/** + * Whether a string is valid as RFC 3986 scheme name. + * Colon `:` is part of the URI; not the scheme name, and therefore rejected. + * See https://www.rfc-editor.org/rfc/rfc3986#section-3.1 + * + * Does not check whether the scheme is understood, as that's context-dependent. + */ +bool isValidSchemeName(std::string_view scheme); + } diff --git a/tests/unit/libutil/url.cc b/tests/unit/libutil/url.cc index a678dad20..09fa4e218 100644 --- a/tests/unit/libutil/url.cc +++ b/tests/unit/libutil/url.cc @@ -344,4 +344,22 @@ namespace nix { ASSERT_EQ(percentDecode(e), s); } +TEST(nix, isValidSchemeName) { + ASSERT_TRUE(isValidSchemeName("http")); + ASSERT_TRUE(isValidSchemeName("https")); + ASSERT_TRUE(isValidSchemeName("file")); + ASSERT_TRUE(isValidSchemeName("file+https")); + ASSERT_TRUE(isValidSchemeName("fi.le")); + ASSERT_TRUE(isValidSchemeName("file-ssh")); + ASSERT_TRUE(isValidSchemeName("file+")); + ASSERT_TRUE(isValidSchemeName("file.")); + ASSERT_TRUE(isValidSchemeName("file1")); + ASSERT_FALSE(isValidSchemeName("file:")); + ASSERT_FALSE(isValidSchemeName("file/")); + ASSERT_FALSE(isValidSchemeName("+file")); + ASSERT_FALSE(isValidSchemeName(".file")); + ASSERT_FALSE(isValidSchemeName("-file")); + ASSERT_FALSE(isValidSchemeName("1file")); +} + } From a05bc9eb92371af631fc9fb83c3595957fb56943 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Dec 2023 15:27:29 +0100 Subject: [PATCH 794/909] allowed-uris: Match whole schemes also when scheme is not followed by slashes --- ...llowed-uris-can-now-match-whole-schemes.md | 7 ++++ src/libexpr/eval-settings.hh | 5 +++ src/libexpr/eval.cc | 17 ++++++++- tests/unit/libexpr/eval.cc | 35 +++++++++++++++++++ 4 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 doc/manual/rl-next/allowed-uris-can-now-match-whole-schemes.md diff --git a/doc/manual/rl-next/allowed-uris-can-now-match-whole-schemes.md b/doc/manual/rl-next/allowed-uris-can-now-match-whole-schemes.md new file mode 100644 index 000000000..3cf75a612 --- /dev/null +++ b/doc/manual/rl-next/allowed-uris-can-now-match-whole-schemes.md @@ -0,0 +1,7 @@ +--- +synopsis: Option `allowed-uris` can now match whole schemes in URIs without slashes +prs: 9547 +--- + +If a scheme, such as `github:` is specified in the `allowed-uris` option, all URIs starting with `github:` are allowed. +Previously this only worked for schemes whose URIs used the `://` syntax. diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index db2971acb..3009a462c 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -68,6 +68,11 @@ struct EvalSettings : Config evaluation mode. For example, when set to `https://github.com/NixOS`, builtin functions such as `fetchGit` are allowed to access `https://github.com/NixOS/patchelf.git`. + + Access is granted when + - the URI is equal to the prefix, + - or the URI is a subpath of the prefix, + - or the prefix is a URI scheme ended by a colon `:` and the URI has the same scheme. )"}; Setting traceFunctionCalls{this, false, "trace-function-calls", diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 9e541f293..1552e3e92 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -18,6 +18,7 @@ #include "memory-input-accessor.hh" #include "signals.hh" #include "gc-small-vector.hh" +#include "url.hh" #include #include @@ -599,6 +600,14 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & mkStorePathString(storePath, v); } +inline static bool isJustSchemePrefix(std::string_view prefix) +{ + return + !prefix.empty() + && prefix[prefix.size() - 1] == ':' + && isValidSchemeName(prefix.substr(0, prefix.size() - 1)); +} + bool isAllowedURI(std::string_view uri, const Strings & allowedUris) { /* 'uri' should be equal to a prefix, or in a subdirectory of a @@ -611,8 +620,14 @@ bool isAllowedURI(std::string_view uri, const Strings & allowedUris) && prefix.size() > 0 && hasPrefix(uri, prefix) && ( + // Allow access to subdirectories of the prefix. prefix[prefix.size() - 1] == '/' - || uri[prefix.size()] == '/'))) + || uri[prefix.size()] == '/' + + // Allow access to whole schemes + || isJustSchemePrefix(prefix) + ) + )) return true; } diff --git a/tests/unit/libexpr/eval.cc b/tests/unit/libexpr/eval.cc index cc5d6bbfa..93d3f658f 100644 --- a/tests/unit/libexpr/eval.cc +++ b/tests/unit/libexpr/eval.cc @@ -103,4 +103,39 @@ TEST(nix_isAllowedURI, file_url) { ASSERT_FALSE(isAllowedURI("file://", allowed)); } +TEST(nix_isAllowedURI, github_all) { + Strings allowed; + allowed.push_back("github:"); + ASSERT_TRUE(isAllowedURI("github:", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar/feat-multi-bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar?ref=refs/heads/feat-multi-bar", allowed)); + ASSERT_TRUE(isAllowedURI("github://foo/bar", allowed)); + ASSERT_FALSE(isAllowedURI("https://github:443/foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file://github:foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file:///github:foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("github", allowed)); +} + +TEST(nix_isAllowedURI, github_org) { + Strings allowed; + allowed.push_back("github:foo"); + ASSERT_FALSE(isAllowedURI("github:", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar/feat-multi-bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar?ref=refs/heads/feat-multi-bar", allowed)); + ASSERT_FALSE(isAllowedURI("github://foo/bar", allowed)); + ASSERT_FALSE(isAllowedURI("https://github:443/foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file://github:foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file:///github:foo/bar/archive/master.tar.gz", allowed)); +} + +TEST(nix_isAllowedURI, non_scheme_colon) { + Strings allowed; + allowed.push_back("https://foo/bar:"); + ASSERT_TRUE(isAllowedURI("https://foo/bar:", allowed)); + ASSERT_TRUE(isAllowedURI("https://foo/bar:/baz", allowed)); + ASSERT_FALSE(isAllowedURI("https://foo/bar:baz", allowed)); +} + } // namespace nix \ No newline at end of file From 89cf53648ca98434a40b0c0cef51fa64f6e0fa37 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 11 Dec 2023 12:26:31 +0100 Subject: [PATCH 795/909] Contributing branches and reverting (#9577) Co-authored-by: Valentin Gagarin --- doc/manual/src/contributing/hacking.md | 42 ++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 237eff925..4d3d66397 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -282,3 +282,45 @@ See also the [format documentation](https://github.com/haskell/cabal/blob/master Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. Set `buildUnreleasedNotes = true;` in `flake.nix` to build the release notes on the fly. + +## Branches + +- [`master`](https://github.com/NixOS/nix/commits/master) + + The main development branch. All changes are approved and merged here. + When developing a change, create a branch based on the latest `master`. + + Maintainers try to [keep it in a release-worthy state](#reverting). + +- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) + + These branches are the subject of backports only, and are + also [kept](#reverting) in a release-worthy state. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) + + The latest patch release of the latest minor version. + + See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) + +- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) + + Generally branches created by the backport action. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [_other_](https://github.com/NixOS/nix/branches/all) + + Branches that do not conform to the above patterns should be feature branches. + +## Reverting + +If a change turns out to be merged by mistake, or contain a regression, it may be reverted. +A revert is not a rejection of the contribution, but merely part of an effective development process. +It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. +If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. +By embracing reverts as a good part of the development process, everyone wins. + +However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. From f45d2ee2b7090560fc30a227d638684268af700d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=B6ller?= Date: Mon, 11 Dec 2023 16:02:09 +0100 Subject: [PATCH 796/909] Fix query parsing for path-like flakes --- src/libexpr/flake/flakeref.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 16f45ace7..8b0eb7460 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -90,7 +90,7 @@ std::pair parsePathFlakeRefWithFragment( fragment = percentDecode(url.substr(fragmentStart+1)); } if (pathEnd != std::string::npos && fragmentStart != std::string::npos) { - query = decodeQuery(url.substr(pathEnd+1, fragmentStart)); + query = decodeQuery(url.substr(pathEnd+1, fragmentStart-pathEnd-1)); } if (baseDir) { From 994f1b5c0de44319992ef6b1b106cee3fa400dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=B6ller?= Date: Mon, 11 Dec 2023 16:05:34 +0100 Subject: [PATCH 797/909] Add test cases for flake urls with fragments --- tests/functional/flakes/flakes.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index ccf1699f9..7506b6b3b 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -193,6 +193,14 @@ nix build -o "$TEST_ROOT/result" flake1 nix build -o "$TEST_ROOT/result" "$flake1Dir" nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir" +# Test explicit packages.default. +nix build -o "$TEST_ROOT/result" "$flake1Dir#default" +nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir#default" + +# Test explicit packages.default with query. +nix build -o "$TEST_ROOT/result" "$flake1Dir?ref=HEAD#default" +nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" + # Check that store symlinks inside a flake are not interpreted as flakes. nix build -o "$flake1Dir/result" "git+file://$flake1Dir" nix path-info "$flake1Dir/result" From 5f30c8acc7e0cad08924cc53e350e811d097fae7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 10 Dec 2023 18:51:23 -0500 Subject: [PATCH 798/909] Give `Store::queryDerivationOutputMap` and `evalStore` argument Picking up where https://github.com/NixOS/nix/pull/9563 left off. --- src/libstore/store-api.cc | 4 ++-- src/libstore/store-api.hh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 800df7fa0..7f35e74af 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -547,8 +547,8 @@ std::map> Store::queryPartialDerivationOut return outputs; } -OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { - auto resp = queryPartialDerivationOutputMap(path); +OutputPathMap Store::queryDerivationOutputMap(const StorePath & path, Store * evalStore) { + auto resp = queryPartialDerivationOutputMap(path, evalStore); OutputPathMap result; for (auto & [outName, optOutPath] : resp) { if (!optOutPath) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index ada6699d5..13e5a1446 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -369,7 +369,7 @@ public: * Query the mapping outputName=>outputPath for the given derivation. * Assume every output has a mapping and throw an exception otherwise. */ - OutputPathMap queryDerivationOutputMap(const StorePath & path); + OutputPathMap queryDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr); /** * Query the full store path given the hash part of a valid store From 9f39dda66ce0f92707d4be05d0a90961c78f8bd4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 10 Dec 2023 21:21:21 -0500 Subject: [PATCH 799/909] Fix building CA derivations with and eval store I don't love the way this code looks. There are two larger problems: - eval, build/scratch, destination stores (#5025) should have different types to reflect the fact that they are used for different purposes and those purposes correspond to different operations. It should be impossible to "use the wrong store" in my cases. - Since drvs can end up in both the eval and build/scratch store, we should have some sort of union/layered store (not on the file sytem level, just conceptual level) that allows accessing both. This would get rid of the ugly "check both" boilerplate in this PR. Still, it might be better to land this now / soon after minimal cleanup, so we have a concrete idea of what problem better abstractions are supposed to solve. --- src/libstore/build/derivation-goal.cc | 51 +++++++++++++++++++++------ src/libstore/misc.cc | 9 +++-- src/libstore/store-api.hh | 3 +- src/nix-build/nix-build.cc | 6 ++-- tests/functional/ca/eval-store.sh | 10 ++++++ tests/functional/ca/local.mk | 1 + tests/functional/eval-store.sh | 16 +++++++-- 7 files changed, 76 insertions(+), 20 deletions(-) create mode 100644 tests/functional/ca/eval-store.sh diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index d4da374ba..f8728ed4a 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -196,10 +196,19 @@ void DerivationGoal::loadDerivation() things being garbage collected while we're busy. */ worker.evalStore.addTempRoot(drvPath); - assert(worker.evalStore.isValidPath(drvPath)); + /* Get the derivation. It is probably in the eval store, but it might be inthe main store: - /* Get the derivation. */ - drv = std::make_unique(worker.evalStore.readDerivation(drvPath)); + - Resolved derivation are resolved against main store realisations, and so must be stored there. + + - Dynamic derivations are built, and so are found in the main store. + */ + for (auto * drvStore : { &worker.evalStore, &worker.store }) { + if (drvStore->isValidPath(drvPath)) { + drv = std::make_unique(drvStore->readDerivation(drvPath)); + break; + } + } + assert(drv); haveDerivation(); } @@ -401,11 +410,15 @@ void DerivationGoal::gaveUpOnSubstitution() } /* Copy the input sources from the eval store to the build - store. */ + store. + + Note that some inputs might not be in the eval store because they + are (resolved) derivation outputs in a resolved derivation. */ if (&worker.evalStore != &worker.store) { RealisedPath::Set inputSrcs; for (auto & i : drv->inputSrcs) - inputSrcs.insert(i); + if (worker.evalStore.isValidPath(i)) + inputSrcs.insert(i); copyClosure(worker.evalStore, worker.store, inputSrcs); } @@ -453,7 +466,7 @@ void DerivationGoal::repairClosure() std::map outputsToDrv; for (auto & i : inputClosure) if (i.isDerivation()) { - auto depOutputs = worker.store.queryPartialDerivationOutputMap(i); + auto depOutputs = worker.store.queryPartialDerivationOutputMap(i, &worker.evalStore); for (auto & j : depOutputs) if (j.second) outputsToDrv.insert_or_assign(*j.second, i); @@ -604,7 +617,13 @@ void DerivationGoal::inputsRealised() return *outPath; } else { - auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath); + auto outMap = [&]{ + for (auto * drvStore : { &worker.evalStore, &worker.store }) + if (drvStore->isValidPath(depDrvPath)) + return worker.store.queryDerivationOutputMap(depDrvPath, drvStore); + assert(false); + }(); + auto outMapPath = outMap.find(outputName); if (outMapPath == outMap.end()) { throw Error( @@ -1085,8 +1104,12 @@ void DerivationGoal::resolvedFinished() auto newRealisation = realisation; newRealisation.id = DrvOutput { initialOutput->outputHash, outputName }; newRealisation.signatures.clear(); - if (!drv->type().isFixed()) - newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath); + if (!drv->type().isFixed()) { + auto & drvStore = worker.evalStore.isValidPath(drvPath) + ? worker.evalStore + : worker.store; + newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore); + } signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); } @@ -1379,7 +1402,10 @@ std::map> DerivationGoal::queryPartialDeri res.insert_or_assign(name, output.path(worker.store, drv->name, name)); return res; } else { - return worker.store.queryPartialDerivationOutputMap(drvPath); + for (auto * drvStore : { &worker.evalStore, &worker.store }) + if (drvStore->isValidPath(drvPath)) + return worker.store.queryPartialDerivationOutputMap(drvPath, drvStore); + assert(false); } } @@ -1392,7 +1418,10 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap() res.insert_or_assign(name, *output.second); return res; } else { - return worker.store.queryDerivationOutputMap(drvPath); + for (auto * drvStore : { &worker.evalStore, &worker.store }) + if (drvStore->isValidPath(drvPath)) + return worker.store.queryDerivationOutputMap(drvPath, drvStore); + assert(false); } } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 9f63fbbb5..cc8ad3d02 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -331,8 +331,11 @@ std::map drvOutputReferences( std::map drvOutputReferences( Store & store, const Derivation & drv, - const StorePath & outputPath) + const StorePath & outputPath, + Store * evalStore_) { + auto & evalStore = evalStore_ ? *evalStore_ : store; + std::set inputRealisations; std::function::ChildNode &)> accumRealisations; @@ -340,7 +343,7 @@ std::map drvOutputReferences( accumRealisations = [&](const StorePath & inputDrv, const DerivedPathMap::ChildNode & inputNode) { if (!inputNode.value.empty()) { auto outputHashes = - staticOutputHashes(store, store.readDerivation(inputDrv)); + staticOutputHashes(evalStore, evalStore.readDerivation(inputDrv)); for (const auto & outputName : inputNode.value) { auto outputHash = get(outputHashes, outputName); if (!outputHash) @@ -362,7 +365,7 @@ std::map drvOutputReferences( SingleDerivedPath next = SingleDerivedPath::Built { d, outputName }; accumRealisations( // TODO deep resolutions for dynamic derivations, issue #8947, would go here. - resolveDerivedPath(store, next), + resolveDerivedPath(store, next, evalStore_), childNode); } } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 13e5a1446..2c883ce97 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -943,6 +943,7 @@ const ContentAddress * getDerivationCA(const BasicDerivation & drv); std::map drvOutputReferences( Store & store, const Derivation & drv, - const StorePath & outputPath); + const StorePath & outputPath, + Store * evalStore = nullptr); } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 01da028d8..8e9be14c1 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -462,7 +462,7 @@ static void main_nix_build(int argc, char * * argv) if (dryRun) return; if (shellDrv) { - auto shellDrvOutputs = store->queryPartialDerivationOutputMap(shellDrv.value()); + auto shellDrvOutputs = store->queryPartialDerivationOutputMap(shellDrv.value(), &*evalStore); shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash"; } @@ -515,7 +515,7 @@ static void main_nix_build(int argc, char * * argv) std::function::ChildNode &)> accumInputClosure; accumInputClosure = [&](const StorePath & inputDrv, const DerivedPathMap::ChildNode & inputNode) { - auto outputs = evalStore->queryPartialDerivationOutputMap(inputDrv); + auto outputs = store->queryPartialDerivationOutputMap(inputDrv, &*evalStore); for (auto & i : inputNode.value) { auto o = outputs.at(i); store->computeFSClosure(*o, inputs); @@ -653,7 +653,7 @@ static void main_nix_build(int argc, char * * argv) if (counter) drvPrefix += fmt("-%d", counter + 1); - auto builtOutputs = evalStore->queryPartialDerivationOutputMap(drvPath); + auto builtOutputs = store->queryPartialDerivationOutputMap(drvPath, &*evalStore); auto maybeOutputPath = builtOutputs.at(outputName); assert(maybeOutputPath); diff --git a/tests/functional/ca/eval-store.sh b/tests/functional/ca/eval-store.sh new file mode 100644 index 000000000..9cc499606 --- /dev/null +++ b/tests/functional/ca/eval-store.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +# Ensure that garbage collection works properly with ca derivations + +source common.sh + +export NIX_TESTS_CA_BY_DEFAULT=1 + +cd .. +source eval-store.sh diff --git a/tests/functional/ca/local.mk b/tests/functional/ca/local.mk index fd87b8d1f..4f86b268f 100644 --- a/tests/functional/ca/local.mk +++ b/tests/functional/ca/local.mk @@ -5,6 +5,7 @@ ca-tests := \ $(d)/concurrent-builds.sh \ $(d)/derivation-json.sh \ $(d)/duplicate-realisation-in-closure.sh \ + $(d)/eval-store.sh \ $(d)/gc.sh \ $(d)/import-derivation.sh \ $(d)/new-build-cmd.sh \ diff --git a/tests/functional/eval-store.sh b/tests/functional/eval-store.sh index 8fc859730..ec99fd953 100644 --- a/tests/functional/eval-store.sh +++ b/tests/functional/eval-store.sh @@ -11,7 +11,16 @@ rm -rf "$eval_store" nix build -f dependencies.nix --eval-store "$eval_store" -o "$TEST_ROOT/result" [[ -e $TEST_ROOT/result/foobar ]] -(! ls $NIX_STORE_DIR/*.drv) +if [[ ! -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then + # Resolved CA derivations are written to store for building + # + # TODO when we something more systematic + # (https://github.com/NixOS/nix/issues/5025) that distinguishes + # between scratch storage for building and the final destination + # store, we'll be able to make this unconditional again -- resolved + # derivations should only appear in the scratch store. + (! ls $NIX_STORE_DIR/*.drv) +fi ls $eval_store/nix/store/*.drv clearStore @@ -26,5 +35,8 @@ rm -rf "$eval_store" nix-build dependencies.nix --eval-store "$eval_store" -o "$TEST_ROOT/result" [[ -e $TEST_ROOT/result/foobar ]] -(! ls $NIX_STORE_DIR/*.drv) +if [[ ! -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then + # See above + (! ls $NIX_STORE_DIR/*.drv) +fi ls $eval_store/nix/store/*.drv From 0b81557e2cf30cebb916f82f192f04df38c810d7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 6 Dec 2023 20:14:14 -0500 Subject: [PATCH 800/909] flake.nix: Put some list items on their own line These things are about to become longer --- flake.nix | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index 99480183a..973bb55af 100644 --- a/flake.nix +++ b/flake.nix @@ -36,8 +36,10 @@ systems = linuxSystems ++ darwinSystems; crossSystems = [ - "armv6l-linux" "armv7l-linux" - "x86_64-freebsd13" "x86_64-netbsd" + "armv6l-linux" + "armv7l-linux" + "x86_64-freebsd13" + "x86_64-netbsd" ]; stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ]; @@ -575,8 +577,25 @@ # to https://nixos.org/nix/install. It downloads the binary # tarball for the user's system and calls the second half of the # installation script. - installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ]; - installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"]; + installerScript = installScriptFor [ + # Native + "x86_64-linux" + "i686-linux" + "aarch64-linux" + "x86_64-darwin" + "aarch64-darwin" + # Cross + "armv6l-linux" + "armv7l-linux" + ]; + installerScriptForGHA = installScriptFor [ + # Native + "x86_64-linux" + "x86_64-darwin" + # Cross + "armv6l-linux" + "armv7l-linux" + ]; # docker image with Nix inside dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); From f60c2e8a5acabf5fb554f77014904bb0d0c91604 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 6 Dec 2023 20:28:11 -0500 Subject: [PATCH 801/909] flake.nix: `installScriptFor` take tarballs not strings Trying to look up keys in multiple places is not nice, better for the caller to be explicit. --- flake.nix | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/flake.nix b/flake.nix index 973bb55af..185140092 100644 --- a/flake.nix +++ b/flake.nix @@ -256,7 +256,7 @@ ]; }; - installScriptFor = systems: + installScriptFor = tarballs: with nixpkgsFor.x86_64-linux.native; runCommand "installer-script" { buildInputs = [ nix ]; @@ -277,14 +277,14 @@ substitute ${./scripts/install.in} $out/install \ ${pkgs.lib.concatMapStrings - (system: let - tarball = if builtins.elem system crossSystems then self.hydraJobs.binaryTarballCross.x86_64-linux.${system} else self.hydraJobs.binaryTarball.${system}; + (tarball: let + inherit (tarball.stdenv.hostPlatform) system; in '' \ --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \ --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \ '' ) - systems + tarballs } --replace '@nixVersion@' ${version} echo "file installer $out/install" >> $out/nix-support/hydra-build-products @@ -341,7 +341,7 @@ installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; }; in - buildPackages.runCommand "nix-binary-tarball-${version}" + pkgs.runCommand "nix-binary-tarball-${version}" { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; meta.description = "Distribution-independent Nix bootstrap binaries for ${pkgs.system}"; } @@ -579,22 +579,22 @@ # installation script. installerScript = installScriptFor [ # Native - "x86_64-linux" - "i686-linux" - "aarch64-linux" - "x86_64-darwin" - "aarch64-darwin" + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."i686-linux" + self.hydraJobs.binaryTarball."aarch64-linux" + self.hydraJobs.binaryTarball."x86_64-darwin" + self.hydraJobs.binaryTarball."aarch64-darwin" # Cross - "armv6l-linux" - "armv7l-linux" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-linux" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-linux" ]; installerScriptForGHA = installScriptFor [ # Native - "x86_64-linux" - "x86_64-darwin" + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."x86_64-darwin" # Cross - "armv6l-linux" - "armv7l-linux" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-linux" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-linux" ]; # docker image with Nix inside From 78492cfde73d57ca01c73d77a23440754c9e7ee4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 2 Sep 2023 14:36:25 -0400 Subject: [PATCH 802/909] flake.nix: Use `config` not `system` for cross so we can be a bit more precise --- flake.nix | 20 ++++++++++---------- maintainers/upload-release.pl | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/flake.nix b/flake.nix index 185140092..2f6668b81 100644 --- a/flake.nix +++ b/flake.nix @@ -36,10 +36,10 @@ systems = linuxSystems ++ darwinSystems; crossSystems = [ - "armv6l-linux" - "armv7l-linux" - "x86_64-freebsd13" - "x86_64-netbsd" + "armv6l-unknown-linux-gnueabihf" + "armv7l-unknown-linux-gnueabihf" + "x86_64-unknown-freebsd13" + "x86_64-unknown-netbsd" ]; stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ]; @@ -116,8 +116,8 @@ inherit system; }; crossSystem = if crossSystem == null then null else { - system = crossSystem; - } // lib.optionalAttrs (crossSystem == "x86_64-freebsd13") { + config = crossSystem; + } // lib.optionalAttrs (crossSystem == "x86_64-unknown-freebsd13") { useLLVM = true; }; overlays = [ @@ -585,16 +585,16 @@ self.hydraJobs.binaryTarball."x86_64-darwin" self.hydraJobs.binaryTarball."aarch64-darwin" # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-linux" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-linux" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" ]; installerScriptForGHA = installScriptFor [ # Native self.hydraJobs.binaryTarball."x86_64-linux" self.hydraJobs.binaryTarball."x86_64-darwin" # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-linux" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-linux" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" ]; # docker image with Nix inside diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index ebc536f12..4e2c379f0 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -154,8 +154,8 @@ downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.x86_64-darwin", "1"); downloadFile("binaryTarball.aarch64-darwin", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv6l-linux", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv7l-linux", "1"); +downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); +downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); downloadFile("installerScript", "1"); # Upload docker images to dockerhub. From 46b98a40a7c5488a99525bc780b7f7bba0131545 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 2 Sep 2023 14:38:33 -0400 Subject: [PATCH 803/909] flake.nix: Make changes so a MinGW dev shell would work --- flake.nix | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index 2f6668b81..f0dd9a3e3 100644 --- a/flake.nix +++ b/flake.nix @@ -210,7 +210,7 @@ buildDeps = [ curl - bzip2 xz brotli editline + bzip2 xz brotli openssl sqlite libarchive (pkgs.libgit2.overrideAttrs (attrs: { @@ -219,10 +219,13 @@ cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"]; })) boost - lowdown-nix libsodium ] - ++ lib.optionals stdenv.isLinux [libseccomp] + ++ lib.optionals (!stdenv.hostPlatform.isWindows) [ + editline + lowdown-nix + ] + ++ lib.optional stdenv.isLinux libseccomp ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; checkDeps = [ @@ -510,7 +513,7 @@ stdenv = currentStdenv; }; - meta.platforms = lib.platforms.unix; + meta.platforms = lib.platforms.unix ++ lib.platforms.windows; meta.mainProgram = "nix"; }); From b892161e314d976e7692ffcf487e1aa042165745 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 12:26:42 -0500 Subject: [PATCH 804/909] flake.nix: Make a MinGW dev shell This requires a `shellCrossSystems` for now, since Nix doesn't actually build on Windows. This can be dropped once it does. --- flake.nix | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index f0dd9a3e3..bbdfc38e9 100644 --- a/flake.nix +++ b/flake.nix @@ -42,6 +42,13 @@ "x86_64-unknown-netbsd" ]; + # Nix doesn't yet build on this platform, so we put it in a + # separate list. We just use this for `devShells` and + # `nixpkgsFor`, which this depends on. + shellCrossSystems = crossSystems ++ [ + "x86_64-w64-mingw32" + ]; + stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ]; forAllSystems = lib.genAttrs systems; @@ -129,7 +136,7 @@ in { inherit stdenvs native; static = native.pkgsStatic; - cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); + cross = lib.genAttrs shellCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); commonDeps = @@ -808,7 +815,7 @@ in (makeShells "native" nixpkgsFor.${system}.native) // (makeShells "static" nixpkgsFor.${system}.static) // - (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // + (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // { default = self.devShells.${system}.native-stdenvPackages; } From 589fb105f311af65230d374cbbddf7173c7ad103 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 12 Dec 2023 16:05:32 +0100 Subject: [PATCH 805/909] Fix the VM tests Work around https://github.com/NixOS/nixpkgs/issues/271146 until we can depend on a Nixpkgs version containing https://github.com/NixOS/nixpkgs/pull/271423 --- flake.nix | 4 ++++ tests/nixos/default.nix | 1 + 2 files changed, 5 insertions(+) diff --git a/flake.nix b/flake.nix index bbdfc38e9..ada52c05d 100644 --- a/flake.nix +++ b/flake.nix @@ -7,6 +7,10 @@ # Also, do not grab arbitrary further staging commits. This PR was # carefully made to be based on release-23.05 and just contain # rebuild-causing changes to packages that Nix actually uses. + # + # Once this is updated to something containing + # https://github.com/NixOS/nixpkgs/pull/271423, don't forget + # to remove the `nix.checkAllErrors = false;` line in the tests. inputs.nixpkgs.url = "github:NixOS/nixpkgs/staging-23.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 4459aa664..2645cac8e 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -10,6 +10,7 @@ let hostPkgs = nixpkgsFor.${system}.native; defaults = { nixpkgs.pkgs = nixpkgsFor.${system}.native; + nix.checkAllErrors = false; }; _module.args.nixpkgs = nixpkgs; }; From 2e451a663eff96b89360cfd3c0d5eaa60ca46181 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 12 Dec 2023 17:22:54 +0100 Subject: [PATCH 806/909] schemeRegex -> schemeNameRegex Scheme could be understood to include the typical `:` separator. --- src/libutil/url-parts.hh | 2 +- src/libutil/url.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 5c5a30dc2..07bc8d0cd 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -8,7 +8,7 @@ namespace nix { // URI stuff. const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; -const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)"; +const static std::string schemeNameRegex = "(?:[a-z][a-z0-9+.-]*)"; const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+(?:%\\w+)?"; const static std::string ipv6AddressRegex = "(?:\\[" + ipv6AddressSegmentRegex + "\\]|" + ipv6AddressSegmentRegex + ")"; const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; diff --git a/src/libutil/url.cc b/src/libutil/url.cc index f2d5f1782..e9acd67d0 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -14,7 +14,7 @@ std::regex revRegex(revRegexS, std::regex::ECMAScript); ParsedURL parseURL(const std::string & url) { static std::regex uriRegex( - "((" + schemeRegex + "):" + "((" + schemeNameRegex + "):" + "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))" + "(?:\\?(" + queryRegex + "))?" + "(?:#(" + queryRegex + "))?", From 4eaeda6604e2f8977728f14415fe92350d047970 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 12 Dec 2023 17:43:54 +0100 Subject: [PATCH 807/909] isValidSchemeName: Use regex As requested by Eelco Dolstra. I think it used to be simpler. --- src/libutil/url.cc | 15 +++------------ tests/unit/libutil/url.cc | 5 +++++ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/libutil/url.cc b/src/libutil/url.cc index e9acd67d0..152c06d8e 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -3,7 +3,6 @@ #include "util.hh" #include "split.hh" #include "canon-path.hh" -#include "string.hh" namespace nix { @@ -187,17 +186,9 @@ std::string fixGitURL(const std::string & url) // https://www.rfc-editor.org/rfc/rfc3986#section-3.1 bool isValidSchemeName(std::string_view s) { - if (s.empty()) return false; - if (!isASCIIAlpha(s[0])) return false; - for (auto c : s.substr(1)) { - if (isASCIIAlpha(c)) continue; - if (isASCIIDigit(c)) continue; - if (c == '+') continue; - if (c == '-') continue; - if (c == '.') continue; - return false; - } - return true; + static std::regex regex(schemeNameRegex, std::regex::ECMAScript); + + return std::regex_match(s.begin(), s.end(), regex, std::regex_constants::match_default); } } diff --git a/tests/unit/libutil/url.cc b/tests/unit/libutil/url.cc index 09fa4e218..7d08f467e 100644 --- a/tests/unit/libutil/url.cc +++ b/tests/unit/libutil/url.cc @@ -360,6 +360,11 @@ TEST(nix, isValidSchemeName) { ASSERT_FALSE(isValidSchemeName(".file")); ASSERT_FALSE(isValidSchemeName("-file")); ASSERT_FALSE(isValidSchemeName("1file")); + // regex ok? + ASSERT_FALSE(isValidSchemeName("\nhttp")); + ASSERT_FALSE(isValidSchemeName("\nhttp\n")); + ASSERT_FALSE(isValidSchemeName("http\n")); + ASSERT_FALSE(isValidSchemeName("http ")); } } From 0b87ba50c08d83384e11a8e6db1e2f97fba4b61c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 12 Dec 2023 17:44:53 +0100 Subject: [PATCH 808/909] Revert "Add nix::isASCII*, locale-independent" This reverts commit 79eb2920bb51c7ec9528a403986e79f04738e2be. Not used at this time. --- src/libutil/string.hh | 17 ----------- tests/unit/libutil/string.cc | 59 ------------------------------------ 2 files changed, 76 deletions(-) delete mode 100644 src/libutil/string.hh delete mode 100644 tests/unit/libutil/string.cc diff --git a/src/libutil/string.hh b/src/libutil/string.hh deleted file mode 100644 index 16ef75643..000000000 --- a/src/libutil/string.hh +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -namespace nix { - - /** Locale-independent version of std::islower(). */ - inline bool isASCIILower(char c) { return c >= 'a' && c <= 'z'; }; - - /** Locale-independent version of std::isupper(). */ - inline bool isASCIIUpper(char c) { return c >= 'A' && c <= 'Z'; }; - - /** Locale-independent version of std::isalpha(). */ - inline bool isASCIIAlpha(char c) { return isASCIILower(c) || isASCIIUpper(c); }; - - /** Locale-independent version of std::isdigit(). */ - inline bool isASCIIDigit(char c) { return c >= '0' && c <= '9'; }; - -} diff --git a/tests/unit/libutil/string.cc b/tests/unit/libutil/string.cc deleted file mode 100644 index 381f2cc15..000000000 --- a/tests/unit/libutil/string.cc +++ /dev/null @@ -1,59 +0,0 @@ -#include -#include "string.hh" - -namespace nix { - -TEST(string, isASCIILower) { - ASSERT_TRUE(isASCIILower('a')); - ASSERT_TRUE(isASCIILower('z')); - ASSERT_FALSE(isASCIILower('A')); - ASSERT_FALSE(isASCIILower('Z')); - ASSERT_FALSE(isASCIILower('0')); - ASSERT_FALSE(isASCIILower('9')); - ASSERT_FALSE(isASCIILower(' ')); - ASSERT_FALSE(isASCIILower('\n')); - ASSERT_FALSE(isASCIILower('\t')); - ASSERT_FALSE(isASCIILower(':')); -} - -TEST(string, isASCIIUpper) { - ASSERT_FALSE(isASCIIUpper('a')); - ASSERT_FALSE(isASCIIUpper('z')); - ASSERT_TRUE(isASCIIUpper('A')); - ASSERT_TRUE(isASCIIUpper('Z')); - ASSERT_FALSE(isASCIIUpper('0')); - ASSERT_FALSE(isASCIIUpper('9')); - ASSERT_FALSE(isASCIIUpper(' ')); - ASSERT_FALSE(isASCIIUpper('\n')); - ASSERT_FALSE(isASCIIUpper('\t')); - ASSERT_FALSE(isASCIIUpper(':')); -} - -TEST(string, isASCIIAlpha) { - ASSERT_TRUE(isASCIIAlpha('a')); - ASSERT_TRUE(isASCIIAlpha('z')); - ASSERT_TRUE(isASCIIAlpha('A')); - ASSERT_TRUE(isASCIIAlpha('Z')); - ASSERT_FALSE(isASCIIAlpha('0')); - ASSERT_FALSE(isASCIIAlpha('9')); - ASSERT_FALSE(isASCIIAlpha(' ')); - ASSERT_FALSE(isASCIIAlpha('\n')); - ASSERT_FALSE(isASCIIAlpha('\t')); - ASSERT_FALSE(isASCIIAlpha(':')); -} - -TEST(string, isASCIIDigit) { - ASSERT_FALSE(isASCIIDigit('a')); - ASSERT_FALSE(isASCIIDigit('z')); - ASSERT_FALSE(isASCIIDigit('A')); - ASSERT_FALSE(isASCIIDigit('Z')); - ASSERT_TRUE(isASCIIDigit('0')); - ASSERT_TRUE(isASCIIDigit('1')); - ASSERT_TRUE(isASCIIDigit('9')); - ASSERT_FALSE(isASCIIDigit(' ')); - ASSERT_FALSE(isASCIIDigit('\n')); - ASSERT_FALSE(isASCIIDigit('\t')); - ASSERT_FALSE(isASCIIDigit(':')); -} - -} \ No newline at end of file From 04f454f2a0e1bfb4fc0368872f215cb690df11bc Mon Sep 17 00:00:00 2001 From: SharzyL Date: Wed, 13 Dec 2023 10:30:28 +0800 Subject: [PATCH 809/909] fix: nix copy ssh-ng:// not respecting --substitute-on-destination --- src/libstore/remote-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index cc26c2a94..dd6347468 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -225,7 +225,7 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute conn->to << WorkerProto::Op::QueryValidPaths; WorkerProto::write(*this, *conn, paths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) { - conn->to << (settings.buildersUseSubstitutes ? 1 : 0); + conn->to << maybeSubstitute; } conn.processStderr(); return WorkerProto::Serialise::read(*this, *conn); From cc3913e4584beb19e7af00572db119d2638333d5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Dec 2023 13:27:23 +0100 Subject: [PATCH 810/909] Remove unused variable --- src/libexpr/eval.hh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index f3f6d35b9..f452dcb9f 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -335,11 +335,6 @@ private: std::map> searchPathResolved; - /** - * Cache used by checkSourcePath(). - */ - std::unordered_map resolvedPaths; - /** * Cache used by prim_match(). */ From 103ca0bde5d4f32745d4c3aee534cf4aa0a69a9d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Dec 2023 13:27:29 +0100 Subject: [PATCH 811/909] Improve SourcePath display --- src/libfetchers/input-accessor.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index d5ac238b1..f385e6231 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -130,7 +130,7 @@ struct SourcePath { return accessor->getPhysicalPath(path); } std::string to_string() const - { return path.abs(); } + { return accessor->showPath(path); } /** * Append a `CanonPath` to this path. From faa4cae9aed4e9f8c40ed8c6fe00bd0216c3b0ea Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Dec 2023 13:27:39 +0100 Subject: [PATCH 812/909] LibExprTest: Ignore $NIX_PATH Otherwise a broken $NIX_PATH can cause the test suite to fail. --- tests/unit/libexpr-support/tests/libexpr.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/libexpr-support/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh index 968431446..d720cedde 100644 --- a/tests/unit/libexpr-support/tests/libexpr.hh +++ b/tests/unit/libexpr-support/tests/libexpr.hh @@ -8,6 +8,7 @@ #include "nixexpr.hh" #include "eval.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "store-api.hh" #include "tests/libstore.hh" @@ -18,6 +19,7 @@ namespace nix { static void SetUpTestSuite() { LibStoreTest::SetUpTestSuite(); initGC(); + evalSettings.nixPath = {}; } protected: From 19ec1c9fd4d4bf6e941b046b8549ba2a1a690937 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Dec 2023 15:15:30 +0100 Subject: [PATCH 813/909] Improve the unsafeGetAttrPos test We can use corepkgsFS->addFile() now to create a "real" position. --- tests/unit/libexpr/primops.cc | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/tests/unit/libexpr/primops.cc b/tests/unit/libexpr/primops.cc index 7485fa0d0..384d9924b 100644 --- a/tests/unit/libexpr/primops.cc +++ b/tests/unit/libexpr/primops.cc @@ -1,6 +1,8 @@ #include #include +#include "memory-input-accessor.hh" + #include "tests/libexpr.hh" namespace nix { @@ -148,10 +150,25 @@ namespace nix { } TEST_F(PrimOpTest, unsafeGetAttrPos) { - // The `y` attribute is at position - const char* expr = "builtins.unsafeGetAttrPos \"y\" { y = \"x\"; }"; + state.corepkgsFS->addFile(CanonPath("foo.nix"), "{ y = \"x\"; }"); + + auto expr = "builtins.unsafeGetAttrPos \"y\" (import )"; auto v = eval(expr); - ASSERT_THAT(v, IsNull()); + ASSERT_THAT(v, IsAttrsOfSize(3)); + + auto file = v.attrs->find(createSymbol("file")); + ASSERT_NE(file, nullptr); + ASSERT_THAT(*file->value, IsString()); + auto s = baseNameOf(file->value->string_view()); + ASSERT_EQ(s, "foo.nix"); + + auto line = v.attrs->find(createSymbol("line")); + ASSERT_NE(line, nullptr); + ASSERT_THAT(*line->value, IsIntEq(1)); + + auto column = v.attrs->find(createSymbol("column")); + ASSERT_NE(column, nullptr); + ASSERT_THAT(*column->value, IsIntEq(3)); } TEST_F(PrimOpTest, hasAttr) { From e76df8781417dad9ab4f0a6c3b28917e35f204bf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 13 Dec 2023 11:26:14 -0500 Subject: [PATCH 814/909] Test `nix copy --substitute-on-destination` It works with both `ssh://` and `ssh-ng://` now since #9600 (and `ssh-ng:// didn't work before that). Also, by making the two tests share code, we nudge ourselves towards making sure there is feature parity. --- tests/functional/nix-copy-ssh-common.sh | 70 +++++++++++++++++++++++++ tests/functional/nix-copy-ssh-ng.sh | 16 +++--- tests/functional/nix-copy-ssh.sh | 19 +------ 3 files changed, 77 insertions(+), 28 deletions(-) create mode 100644 tests/functional/nix-copy-ssh-common.sh diff --git a/tests/functional/nix-copy-ssh-common.sh b/tests/functional/nix-copy-ssh-common.sh new file mode 100644 index 000000000..cc8314ff7 --- /dev/null +++ b/tests/functional/nix-copy-ssh-common.sh @@ -0,0 +1,70 @@ +proto=$1 +shift +(( $# == 0 )) + +clearStore +clearCache + +mkdir -p $TEST_ROOT/stores + +# Create path to copy back and forth +outPath=$(nix-build --no-out-link dependencies.nix) + +storeQueryParam="store=${NIX_STORE_DIR}" + +realQueryParam () { + echo "real=$1$NIX_STORE_DIR" +} + +remoteRoot="$TEST_ROOT/stores/$proto" + +clearRemoteStore () { + chmod -R u+w "$remoteRoot" || true + rm -rf "$remoteRoot" +} + +clearRemoteStore + +remoteStore="${proto}://localhost?${storeQueryParam}&remote-store=${remoteRoot}%3f${storeQueryParam}%26$(realQueryParam "$remoteRoot")" + +# Copy to store + +args=() +if [[ "$proto" == "ssh-ng" ]]; then + # TODO investigate discrepancy + args+=(--no-check-sigs) +fi + +[ ! -f ${remoteRoot}${outPath}/foobar ] +nix copy "${args[@]}" --to "$remoteStore" $outPath +[ -f ${remoteRoot}${outPath}/foobar ] + +# Copy back from store + +clearStore + +[ ! -f $outPath/foobar ] +nix copy --no-check-sigs --from "$remoteStore" $outPath +[ -f $outPath/foobar ] + +# Check --substitute-on-destination, avoid corrupted store + +clearRemoteStore + +corruptedRoot=$TEST_ROOT/stores/corrupted +corruptedStore="${corruptedRoot}?${storeQueryParam}&$(realQueryParam "$corruptedRoot")" + +# Copy it to the corrupted store +nix copy --no-check-sigs "$outPath" --to "$corruptedStore" + +# Corrupt it in there +corruptPath="${corruptedRoot}${outPath}" +chmod +w "$corruptPath" +echo "not supposed to be here" > "$corruptPath/foobarbaz" +chmod -w "$corruptPath" + +# Copy from the corrupted store with the regular store as a +# substituter. It must use the substituter not the source store in +# order to avoid errors. +NIX_CONFIG=$(echo -e "substituters = local\nrequire-sigs = false") \ + nix copy --no-check-sigs --from "$corruptedStore" --to "$remoteStore" --substitute-on-destination "$outPath" diff --git a/tests/functional/nix-copy-ssh-ng.sh b/tests/functional/nix-copy-ssh-ng.sh index 463b5e0c4..62e99cd24 100644 --- a/tests/functional/nix-copy-ssh-ng.sh +++ b/tests/functional/nix-copy-ssh-ng.sh @@ -1,18 +1,14 @@ source common.sh -clearStore -clearCache +source nix-copy-ssh-common.sh "ssh-ng" -remoteRoot=$TEST_ROOT/store2 -chmod -R u+w "$remoteRoot" || true -rm -rf "$remoteRoot" +clearStore +clearRemoteStore outPath=$(nix-build --no-out-link dependencies.nix) -nix store info --store "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" +nix store info --store "$remoteStore" # Regression test for https://github.com/NixOS/nix/issues/6253 -nix copy --to "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath --no-check-sigs & -nix copy --to "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath --no-check-sigs - -[ -f $remoteRoot$outPath/foobar ] +nix copy --to "$remoteStore" $outPath --no-check-sigs & +nix copy --to "$remoteStore" $outPath --no-check-sigs diff --git a/tests/functional/nix-copy-ssh.sh b/tests/functional/nix-copy-ssh.sh index eb801548d..12e8346bc 100644 --- a/tests/functional/nix-copy-ssh.sh +++ b/tests/functional/nix-copy-ssh.sh @@ -1,20 +1,3 @@ source common.sh -clearStore -clearCache - -remoteRoot=$TEST_ROOT/store2 -chmod -R u+w "$remoteRoot" || true -rm -rf "$remoteRoot" - -outPath=$(nix-build --no-out-link dependencies.nix) - -nix copy --to "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath - -[ -f $remoteRoot$outPath/foobar ] - -clearStore - -nix copy --no-check-sigs --from "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath - -[ -f $outPath/foobar ] +source nix-copy-ssh-common.sh "ssh" From 19573f1b05b7d3ccfd07c9c351396494d488ab2d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 13 Dec 2023 15:33:15 -0500 Subject: [PATCH 815/909] Restore comment --- scripts/binary-tarball.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/binary-tarball.nix b/scripts/binary-tarball.nix index 32e811c94..104189b0c 100644 --- a/scripts/binary-tarball.nix +++ b/scripts/binary-tarball.nix @@ -14,6 +14,7 @@ let inherit (nix) version; env = { + #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; }; From f10f0f1b50228e09ad587a7c550df586061e4514 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 13 Dec 2023 20:41:20 +0000 Subject: [PATCH 816/909] Move `lowdown.nix` to `misc/` --- flake.nix | 2 +- lowdown.nix => misc/lowdown.nix | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename lowdown.nix => misc/lowdown.nix (100%) diff --git a/flake.nix b/flake.nix index 0cdd2b41f..c7ff7eb64 100644 --- a/flake.nix +++ b/flake.nix @@ -167,7 +167,7 @@ ''; }; - lowdown-nix = final.callPackage ./lowdown.nix { + lowdown-nix = final.callPackage ./misc/lowdown.nix { inherit lowdown-src stdenv; }; diff --git a/lowdown.nix b/misc/lowdown.nix similarity index 100% rename from lowdown.nix rename to misc/lowdown.nix From bf5804d46a0d0aa5eb40107b6eaeec4e95bbd4a2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 13 Dec 2023 20:41:41 +0000 Subject: [PATCH 817/909] flake.nix: Delete uneeded `attrs0` binding --- package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.nix b/package.nix index 7c1ba3130..8fc4be328 100644 --- a/package.nix +++ b/package.nix @@ -92,7 +92,7 @@ , __forDefaults ? { canRunInstalled = doBuild && stdenv.buildPlatform.canExecute stdenv.hostPlatform; } -} @ attrs0: +}: let version = lib.fileContents ./.version + versionSuffix; From 28f2f3136d19ef7de4c6acd9678aef72e80d4fb8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 13 Dec 2023 20:47:36 +0000 Subject: [PATCH 818/909] Delete stray `install_name_tool` call --- package.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/package.nix b/package.nix index 8fc4be328..0b5b512c7 100644 --- a/package.nix +++ b/package.nix @@ -320,7 +320,6 @@ in { -change ${boost}/lib/libboost_context.dylib \ $out/lib/libboost_context.dylib \ $out/lib/libnixutil.dylib - install_name_tool '' ) + lib.optionalString enableInternalAPIDocs '' mkdir -p ''${!outputDoc}/nix-support From 2d24875fe4aa7f31d15acfc29b9aa5c45109f99d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 13 Dec 2023 20:49:31 +0000 Subject: [PATCH 819/909] package.nix: Avoid `${..}` for conditional strings Using `+` is Nixpkgs standard ideom for this, and helps avoid needless rebuilds somewhat. --- package.nix | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/package.nix b/package.nix index 0b5b512c7..0b2ff43b0 100644 --- a/package.nix +++ b/package.nix @@ -254,25 +254,25 @@ in { disallowedReferences = [ boost ]; - preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib - rm -f $out/lib/*.a - ${lib.optionalString stdenv.hostPlatform.isLinux '' + preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) ( + '' + # Copy libboost_context so we don't get all of Boost in our closure. + # https://github.com/NixOS/nixpkgs/issues/45462 + mkdir -p $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib + rm -f $out/lib/*.a + '' + lib.optionalString stdenv.hostPlatform.isLinux '' chmod u+w $out/lib/*.so.* patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - ''} - ${lib.optionalString stdenv.hostPlatform.isDarwin '' + '' + lib.optionalString stdenv.hostPlatform.isDarwin '' for LIB in $out/lib/*.dylib; do chmod u+w $LIB install_name_tool -id $LIB $LIB install_name_tool -delete_rpath ${boost}/lib/ $LIB || true done install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - ''} - ''; + '' + ); configureFlags = [ "--sysconfdir=/etc" From 7b29b44d8e62f686aa9fbfafe53be959cdba03cb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 13 Dec 2023 16:22:35 -0500 Subject: [PATCH 820/909] Remove custom lowdown This was last upgraded in 788008385ef5bf7edb799977525d6f73f02c76bc, but the version in Nixpkgs is a now a lot newer. I think the custom was added to get ahead of Nixpkgs before, and so now that we are in fact behind, it is no longer needed. --- flake.lock | 17 ----------------- flake.nix | 11 +---------- misc/lowdown.nix | 22 ---------------------- 3 files changed, 1 insertion(+), 49 deletions(-) delete mode 100644 misc/lowdown.nix diff --git a/flake.lock b/flake.lock index 3cb9e72c9..db1a72c14 100644 --- a/flake.lock +++ b/flake.lock @@ -32,22 +32,6 @@ "type": "github" } }, - "lowdown-src": { - "flake": false, - "locked": { - "lastModified": 1633514407, - "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=", - "owner": "kristapsdz", - "repo": "lowdown", - "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8", - "type": "github" - }, - "original": { - "owner": "kristapsdz", - "repo": "lowdown", - "type": "github" - } - }, "nixpkgs": { "locked": { "lastModified": 1701355166, @@ -84,7 +68,6 @@ "inputs": { "flake-compat": "flake-compat", "libgit2": "libgit2", - "lowdown-src": "lowdown-src", "nixpkgs": "nixpkgs", "nixpkgs-regression": "nixpkgs-regression" } diff --git a/flake.nix b/flake.nix index c7ff7eb64..eb3846564 100644 --- a/flake.nix +++ b/flake.nix @@ -13,11 +13,10 @@ # to remove the `nix.checkAllErrors = false;` line in the tests. inputs.nixpkgs.url = "github:NixOS/nixpkgs/staging-23.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; - inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; - outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, libgit2, ... }: + outputs = { self, nixpkgs, nixpkgs-regression, libgit2, ... }: let inherit (nixpkgs) lib; @@ -140,9 +139,6 @@ { nixStable = prev.nix; - # Forward from the previous stage as we don’t want it to pick the lowdown override - inherit (prev) nixUnstable; - default-busybox-sandbox-shell = final.busybox.override { useMusl = true; enableStatic = true; @@ -167,10 +163,6 @@ ''; }; - lowdown-nix = final.callPackage ./misc/lowdown.nix { - inherit lowdown-src stdenv; - }; - libgit2-nix = final.libgit2.overrideAttrs (attrs: { src = libgit2; version = libgit2.lastModifiedDate; @@ -208,7 +200,6 @@ officialRelease = false; boehmgc = final.boehmgc-nix; libgit2 = final.libgit2-nix; - lowdown = final.lowdown-nix; busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; changelog-d = final.changelog-d-nix; } // { diff --git a/misc/lowdown.nix b/misc/lowdown.nix deleted file mode 100644 index 5f469fad5..000000000 --- a/misc/lowdown.nix +++ /dev/null @@ -1,22 +0,0 @@ -{ lib -, stdenv -, which -, lowdown-src -}: - -stdenv.mkDerivation rec { - name = "lowdown-0.9.0"; - - src = lowdown-src; - - outputs = [ "out" "bin" "dev" ]; - - nativeBuildInputs = [ which ]; - - configurePhase = '' - ${lib.optionalString (stdenv.isDarwin && stdenv.isAarch64) "echo \"HAVE_SANDBOX_INIT=false\" > configure.local"} - ./configure \ - PREFIX=${placeholder "dev"} \ - BINDIR=${placeholder "bin"}/bin - ''; -} From 1e3d8118401d80da54fb64641e606042c3499e4d Mon Sep 17 00:00:00 2001 From: Ramses Date: Wed, 13 Dec 2023 22:37:17 +0100 Subject: [PATCH 821/909] worker protocol: serialise cgroup stats in `BuildResult` (#9598) By doing so, they get reported when building through the daemon via either `unix://` or `ssh-ng://`. --- doc/manual/rl-next/cgroup-stats.md | 8 +++ src/libstore/worker-protocol.cc | 34 ++++++++++ src/libstore/worker-protocol.hh | 6 +- .../worker-protocol/build-result-1.37.bin | Bin 0 -> 808 bytes tests/unit/libstore/worker-protocol.cc | 61 ++++++++++++++++-- 5 files changed, 101 insertions(+), 8 deletions(-) create mode 100644 doc/manual/rl-next/cgroup-stats.md create mode 100644 tests/unit/libstore/data/worker-protocol/build-result-1.37.bin diff --git a/doc/manual/rl-next/cgroup-stats.md b/doc/manual/rl-next/cgroup-stats.md new file mode 100644 index 000000000..00853a0f8 --- /dev/null +++ b/doc/manual/rl-next/cgroup-stats.md @@ -0,0 +1,8 @@ +--- +synopsis: Include cgroup stats when building through the daemon +prs: 9598 +--- + +Nix now also reports cgroup statistics when building through the nix daemon and when doing remote builds using ssh-ng, +if both sides of the connection are this version of Nix or newer. + diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 2a379e75e..a50259d24 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -7,6 +7,7 @@ #include "archive.hh" #include "path-info.hh" +#include #include namespace nix { @@ -47,6 +48,31 @@ void WorkerProto::Serialise>::write(const StoreDirCon } +std::optional WorkerProto::Serialise>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) +{ + auto tag = readNum(conn.from); + switch (tag) { + case 0: + return std::nullopt; + case 1: + return std::optional{std::chrono::microseconds(readNum(conn.from))}; + default: + throw Error("Invalid optional tag from remote"); + } +} + +void WorkerProto::Serialise>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const std::optional & optDuration) +{ + if (!optDuration.has_value()) { + conn.to << uint8_t{0}; + } else { + conn.to + << uint8_t{1} + << optDuration.value().count(); + } +} + + DerivedPath WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { auto s = readString(conn.from); @@ -110,6 +136,10 @@ BuildResult WorkerProto::Serialise::read(const StoreDirConfig & sto >> res.startTime >> res.stopTime; } + if (GET_PROTOCOL_MINOR(conn.version) >= 37) { + res.cpuUser = WorkerProto::Serialise>::read(store, conn); + res.cpuSystem = WorkerProto::Serialise>::read(store, conn); + } if (GET_PROTOCOL_MINOR(conn.version) >= 28) { auto builtOutputs = WorkerProto::Serialise::read(store, conn); for (auto && [output, realisation] : builtOutputs) @@ -132,6 +162,10 @@ void WorkerProto::Serialise::write(const StoreDirConfig & store, Wo << res.startTime << res.stopTime; } + if (GET_PROTOCOL_MINOR(conn.version) >= 37) { + WorkerProto::write(store, conn, res.cpuUser); + WorkerProto::write(store, conn, res.cpuSystem); + } if (GET_PROTOCOL_MINOR(conn.version) >= 28) { DrvOutputs builtOutputs; for (auto & [output, realisation] : res.builtOutputs) diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index c26914289..91d277b77 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include + #include "common-protocol.hh" namespace nix { @@ -9,7 +11,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 36) +#define PROTOCOL_VERSION (1 << 8 | 37) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -214,6 +216,8 @@ template<> DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo); template<> DECLARE_WORKER_SERIALISER(std::optional); +template<> +DECLARE_WORKER_SERIALISER(std::optional); template DECLARE_WORKER_SERIALISER(std::vector); diff --git a/tests/unit/libstore/data/worker-protocol/build-result-1.37.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.37.bin new file mode 100644 index 0000000000000000000000000000000000000000..7d6e43fff2f593c9669897f1594ef2526293b866 GIT binary patch literal 808 zcmc(b!A=4(5QbNYCw&D7HXbqAZP|9&SMZ?mYCOy`Q??0dfh{3~@J_yn?_}3497ssi zkk~`NPG{zy$$yh{=Qh&1p+SP-rryS%zu_*nozv~b{8i*2l1Kg&hyFwTsm?J^pZ&Jx z7(XWuZG7Ec;XHLnni_a6OQ{Pv(Gvn*a8V&-GN)9gN@`4z#zM+q67p$E1#+ya@Ky@P zI?Ja*37y|pu=-Z~h`Kw5v>=OQ{VT!TG~kW14J&v15i`h2cEQPP#N67yfUkq@EZeAh nE0*W@7*-7pjhR{S>lKBa-ro0@_Cq`OPkw~Szw@JOIPv2Pj)JK^ literal 0 HcmV?d00001 diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc index 91f804f0c..2b2e559a9 100644 --- a/tests/unit/libstore/worker-protocol.cc +++ b/tests/unit/libstore/worker-protocol.cc @@ -280,13 +280,60 @@ VERSIONED_CHARACTERIZATION_TEST( }, .startTime = 30, .stopTime = 50, -#if 0 - // These fields are not yet serialized. - // FIXME Include in next version of protocol or document - // why they are skipped. - .cpuUser = std::chrono::milliseconds(500s), - .cpuSystem = std::chrono::milliseconds(604s), -#endif + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_37, + "build-result-1.37", + 1 << 8 | 37, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + BuildResult { + .status = BuildResult::Built, + .timesBuilt = 1, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + .startTime = 30, + .stopTime = 50, + .cpuUser = std::chrono::microseconds(500s), + .cpuSystem = std::chrono::microseconds(604s), }, }; t; From 6ed803737c587a0cc9026093c941c1d1172fa5dc Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Wed, 13 Dec 2023 14:02:52 -0800 Subject: [PATCH 822/909] Use `--with-boost` on macOS `configureFlags` only included `--with-boost` on Linux, which makes local builds as outlined in `doc/manual/src/contributing/hacking.md` fail when performed on macOS. --- package.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.nix b/package.nix index 0b2ff43b0..24395b484 100644 --- a/package.nix +++ b/package.nix @@ -284,8 +284,9 @@ in { ] ++ lib.optionals installUnitTests [ "--with-check-bin-dir=${builtins.placeholder "check"}/bin" "--with-check-lib-dir=${builtins.placeholder "check"}/lib" - ] ++ lib.optionals (doBuild && stdenv.isLinux) [ + ] ++ lib.optionals (doBuild) [ "--with-boost=${boost}/lib" + ] ++ lib.optionals (doBuild && stdenv.isLinux) [ "--with-sandbox-shell=${busybox-sandbox-shell}/bin/busybox" ] ++ lib.optional (doBuild && stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) "LDFLAGS=-fuse-ld=gold" From e13fc0bbdb1e1eefeb33ff4d18310958041b1ad5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 8 Dec 2023 19:52:08 -0500 Subject: [PATCH 823/909] Fix `sys/xattr.h` check I wrote the `configure.ac` wrong, and so we just got no builds supporting ACLs. Also, it needs to be more precise because Darwin puts other stuff in that same header, evidently. --- configure.ac | 3 ++- src/libstore/posix-fs-canonicalise.cc | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index f9ad3c840..a949f9df2 100644 --- a/configure.ac +++ b/configure.ac @@ -289,7 +289,8 @@ esac AC_SUBST(HAVE_SECCOMP, [$have_seccomp]) # Optional dependencies for better normalizing file system data -AC_CHECK_HEADERS[sys/xattr.h] +AC_CHECK_HEADERS([sys/xattr.h]) +AC_CHECK_FUNCS([llistxattr lremovexattr]) # Look for aws-cpp-sdk-s3. AC_LANG_PUSH(C++) diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc index f38fa8369..5edda0157 100644 --- a/src/libstore/posix-fs-canonicalise.cc +++ b/src/libstore/posix-fs-canonicalise.cc @@ -78,7 +78,7 @@ static void canonicalisePathMetaData_( if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) throw Error("file '%1%' has an unsupported type", path); -#ifdef HAVE_SYS_XATTR_H +#if HAVE_SYS_XATTR_H && HAVE_LLISTXATTR && HAVE_LREMOVEXATTR /* Remove extended attributes / ACLs. */ ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); From 8d39c0c19638eb0cd07c1d0af89320e33f9c02d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Luis=20Lafuente?= Date: Thu, 14 Dec 2023 23:14:59 +0100 Subject: [PATCH 824/909] Fix clang devshell Issue introduced in https://github.com/NixOS/nix/pull/9535 --- package.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/package.nix b/package.nix index 24395b484..6ea5bf9c9 100644 --- a/package.nix +++ b/package.nix @@ -5,6 +5,7 @@ , autoreconfHook , aws-sdk-cpp , boehmgc +, buildPackages , nlohmann_json , bison , boost @@ -207,6 +208,9 @@ in { # changelog ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d ++ lib.optional enableInternalAPIDocs doxygen + + ++ lib.optional stdenv.cc.isClang buildPackages.bear + ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) buildPackages.clang-tools ; buildInputs = lib.optionals doBuild [ From bcbdb09ccf7ca007d3c2046177356fbfe7b72304 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 29 Sep 2020 15:33:47 -0400 Subject: [PATCH 825/909] Add eval-system option `eval-system` option overrides just the value of `builtins.currentSystem`. This is more useful than overriding `system` since you can build these derivations on remote builders which can work on the given system. Co-authored-by: John Ericson Co-authored-by: Valentin Gagarin --- src/libexpr/eval-settings.cc | 6 ++++++ src/libexpr/eval-settings.hh | 20 ++++++++++++++++++++ src/libexpr/primops.cc | 11 +++++++---- src/libstore/globals.hh | 6 +++++- tests/unit/libexpr/primops.cc | 3 ++- 5 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 444a7d7d6..2ccbe327f 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -89,6 +89,12 @@ std::string EvalSettings::resolvePseudoUrl(std::string_view url) return std::string(url); } +const std::string & EvalSettings::getCurrentSystem() +{ + const auto & evalSystem = currentSystem.get(); + return evalSystem != "" ? evalSystem : settings.thisSystem.get(); +} + EvalSettings evalSettings; static GlobalConfig::Register rEvalSettings(&evalSettings); diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 3009a462c..ad187ca01 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -27,6 +27,26 @@ struct EvalSettings : Config [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). )"}; + Setting currentSystem{ + this, "", "eval-system", + R"( + This option defines + [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) + in the Nix language if it is set as a non-empty string. + Otherwise, if it is defined as the empty string (the default), the value of the + [`system` ](#conf-system) + configuration setting is used instead. + + Unlike `system`, this setting does not change what kind of derivations can be built locally. + This is useful for evaluating Nix code on one system to produce derivations to be built on another type of system. + )"}; + + /** + * Implements the `eval-system` vs `system` defaulting logic + * described for `eval-system`. + */ + const std::string & getCurrentSystem(); + Setting restrictEval{ this, false, "restrict-eval", R"( diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 89d5492da..d78a28c73 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4383,13 +4383,16 @@ void EvalState::createBaseEnv() .impureOnly = true, }); - if (!evalSettings.pureEval) { - v.mkString(settings.thisSystem.get()); - } + if (!evalSettings.pureEval) + v.mkString(evalSettings.getCurrentSystem()); addConstant("__currentSystem", v, { .type = nString, .doc = R"( - The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-system). + The value of the + [`eval-system`](@docroot@/command-ref/conf-file.md#conf-eval-system) + or else + [`system`](@docroot@/command-ref/conf-file.md#conf-system) + configuration option. It can be used to set the `system` attribute for [`builtins.derivation`](@docroot@/language/derivations.md) such that the resulting derivation can be built on the same system that evaluates the Nix expression: diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index df977e294..e28615cdc 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -214,7 +214,11 @@ public: In general, you do not have to modify this setting. While you can force Nix to run a Darwin-specific `builder` executable on a Linux machine, the result would obviously be wrong. - This value is available in the Nix language as [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem). + This value is available in the Nix language as + [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) + if the + [`eval-system`](#conf-eval-system) + configuration option is set as the empty string. )"}; Setting maxSilentTime{ diff --git a/tests/unit/libexpr/primops.cc b/tests/unit/libexpr/primops.cc index 384d9924b..31b1b49ae 100644 --- a/tests/unit/libexpr/primops.cc +++ b/tests/unit/libexpr/primops.cc @@ -1,6 +1,7 @@ #include #include +#include "eval-settings.hh" #include "memory-input-accessor.hh" #include "tests/libexpr.hh" @@ -631,7 +632,7 @@ namespace nix { TEST_F(PrimOpTest, currentSystem) { auto v = eval("builtins.currentSystem"); - ASSERT_THAT(v, IsStringEq(settings.thisSystem.get())); + ASSERT_THAT(v, IsStringEq(evalSettings.getCurrentSystem())); } TEST_F(PrimOpTest, derivation) { From 70f50cbb2aa35f1ad1e38c9c73a5f8267baac17d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 14 Dec 2023 19:47:10 -0500 Subject: [PATCH 826/909] Functional Test for `builtins.storeDir` and `builtins.currentSystem` These were under-tested. This tests the status quo and especially previous commit of this PR better. --- tests/functional/impure-eval.sh | 35 +++++++++++++++++++++++++++++++++ tests/functional/local.mk | 1 + 2 files changed, 36 insertions(+) create mode 100644 tests/functional/impure-eval.sh diff --git a/tests/functional/impure-eval.sh b/tests/functional/impure-eval.sh new file mode 100644 index 000000000..6c72f01d7 --- /dev/null +++ b/tests/functional/impure-eval.sh @@ -0,0 +1,35 @@ +source common.sh + +export REMOTE_STORE="dummy://" + +simpleTest () { + local expr=$1; shift + local result=$1; shift + # rest, extra args + + [[ "$(nix eval --impure --raw "$@" --expr "$expr")" == "$result" ]] +} + +# `builtins.storeDir` + +## Store dir follows `store` store setting +simpleTest 'builtins.storeDir' '/foo' --store "$REMOTE_STORE?store=/foo" +simpleTest 'builtins.storeDir' '/bar' --store "$REMOTE_STORE?store=/bar" + +# `builtins.currentSystem` + +## `system` alone affects by default +simpleTest 'builtins.currentSystem' 'foo' --system 'foo' +simpleTest 'builtins.currentSystem' 'bar' --system 'bar' + +## `system` affects if `eval-system` is an empty string +simpleTest 'builtins.currentSystem' 'foo' --system 'foo' --eval-system '' +simpleTest 'builtins.currentSystem' 'bar' --system 'bar' --eval-system '' + +## `eval-system` alone affects +simpleTest 'builtins.currentSystem' 'foo' --eval-system 'foo' +simpleTest 'builtins.currentSystem' 'bar' --eval-system 'bar' + +## `eval-system` overrides `system` +simpleTest 'builtins.currentSystem' 'bar' --system 'foo' --eval-system 'bar' +simpleTest 'builtins.currentSystem' 'baz' --system 'foo' --eval-system 'baz' diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 10b399d75..192e275e3 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -71,6 +71,7 @@ nix_tests = \ build-remote-trustless-should-fail-0.sh \ build-remote-with-mounted-ssh-ng.sh \ nar-access.sh \ + impure-eval.sh \ pure-eval.sh \ eval.sh \ repl.sh \ From 228e995cde0f059e4edebdfc8f46d3389d2dc135 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 14 Dec 2023 19:53:59 -0500 Subject: [PATCH 827/909] Add release not for `eval-system` --- doc/manual/rl-next/eval-system.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/manual/rl-next/eval-system.md diff --git a/doc/manual/rl-next/eval-system.md b/doc/manual/rl-next/eval-system.md new file mode 100644 index 000000000..a4696a56c --- /dev/null +++ b/doc/manual/rl-next/eval-system.md @@ -0,0 +1,12 @@ +--- +synopsis: Add new `eval-system` setting +prs: 4093 +--- + +Add a new `eval-system` option. +Unlike `system`, it just overrides the value of `builtins.currentSystem`. +This is more useful than overriding `system`, because you can build these derivations on remote builders which can work on the given system. +In contrast, `system` also effects scheduling which will cause Nix to build those derivations locally even if that doesn't make sense. + +`eval-system` only takes effect if it is non-empty. +If empty (the default) `system` is used as before, so there is no breakage. From 66d37b73383e40f0362b82a0e29c60d2913d689a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Luis=20Lafuente?= Date: Fri, 15 Dec 2023 12:41:38 +0100 Subject: [PATCH 828/909] Move clang dev deps to the nix devshell override --- flake.nix | 5 ++++- package.nix | 4 ---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index eb3846564..8c4436729 100644 --- a/flake.nix +++ b/flake.nix @@ -395,7 +395,7 @@ stdenvs))); devShells = let - makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; }).overrideAttrs (_: { + makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; }).overrideAttrs (attrs: { installFlags = "sysconfdir=$(out)/etc"; shellHook = '' PATH=$prefix/bin:$PATH @@ -405,6 +405,9 @@ # Make bash completion work. XDG_DATA_DIRS+=:$out/share ''; + nativeBuildInputs = attrs.nativeBuildInputs or [] + ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear + ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools; }); in forAllSystems (system: diff --git a/package.nix b/package.nix index 6ea5bf9c9..24395b484 100644 --- a/package.nix +++ b/package.nix @@ -5,7 +5,6 @@ , autoreconfHook , aws-sdk-cpp , boehmgc -, buildPackages , nlohmann_json , bison , boost @@ -208,9 +207,6 @@ in { # changelog ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d ++ lib.optional enableInternalAPIDocs doxygen - - ++ lib.optional stdenv.cc.isClang buildPackages.bear - ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) buildPackages.clang-tools ; buildInputs = lib.optionals doBuild [ From 5cb98095ba2c3de83d32c1729da7b9f6cfb1aeff Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Fri, 15 Dec 2023 23:56:17 -0800 Subject: [PATCH 829/909] Remove some blank lines from stack traces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This keeps hint messages, source location information, and source code snippets grouped together, while making stack traces shorter (so that more stack frames can be viewed on the same terminal). Before: error: … while evaluating the attribute 'body' at /Users/wiggles/nix/tests/functional/lang/eval-fail-assert.nix:4:3: 3| 4| body = x "x"; | ^ 5| } … from call site at /Users/wiggles/nix/tests/functional/lang/eval-fail-assert.nix:4:10: 3| 4| body = x "x"; | ^ 5| } … while calling 'x' at /Users/wiggles/nix/tests/functional/lang/eval-fail-assert.nix:2:7: 1| let { 2| x = arg: assert arg == "y"; 123; | ^ 3| error: assertion '(arg == "y")' failed at /Users/wiggles/nix/tests/functional/lang/eval-fail-assert.nix:2:12: 1| let { 2| x = arg: assert arg == "y"; 123; | ^ 3| After: error: … while evaluating the attribute 'body' at /Users/wiggles/nix/tests/functional/lang/eval-fail-assert.nix:4:3: 3| 4| body = x "x"; | ^ 5| } … from call site at /Users/wiggles/nix/tests/functional/lang/eval-fail-assert.nix:4:10: 3| 4| body = x "x"; | ^ 5| } … while calling 'x' at /Users/wiggles/nix/tests/functional/lang/eval-fail-assert.nix:2:7: 1| let { 2| x = arg: assert arg == "y"; 123; | ^ 3| error: assertion '(arg == "y")' failed at /Users/wiggles/nix/tests/functional/lang/eval-fail-assert.nix:2:12: 1| let { 2| x = arg: assert arg == "y"; 123; | ^ 3| --- src/libutil/error.cc | 3 +-- tests/functional/lang/eval-fail-abort.err.exp | 2 -- ...il-addDrvOutputDependencies-empty-context.err.exp | 2 -- ...dDrvOutputDependencies-multi-elem-context.err.exp | 2 -- ...dDrvOutputDependencies-wrong-element-kind.err.exp | 2 -- tests/functional/lang/eval-fail-assert.err.exp | 8 -------- .../functional/lang/eval-fail-attr-name-type.err.exp | 4 ---- .../eval-fail-bad-string-interpolation-1.err.exp | 2 -- .../eval-fail-bad-string-interpolation-3.err.exp | 2 -- .../eval-fail-bad-string-interpolation-4.err.exp | 2 -- tests/functional/lang/eval-fail-blackhole.err.exp | 4 ---- tests/functional/lang/eval-fail-call-primop.err.exp | 2 -- tests/functional/lang/eval-fail-deepseq.err.exp | 6 ------ .../lang/eval-fail-dup-dynamic-attrs.err.exp | 4 ---- ...al-fail-foldlStrict-strict-op-application.err.exp | 8 -------- .../lang/eval-fail-fromTOML-timestamps.err.exp | 2 -- .../lang/eval-fail-hashfile-missing.err.exp | 2 -- tests/functional/lang/eval-fail-list.err.exp | 2 -- tests/functional/lang/eval-fail-missing-arg.err.exp | 4 ---- tests/functional/lang/eval-fail-not-throws.err.exp | 4 ---- tests/functional/lang/eval-fail-path-slash.err.exp | 2 -- tests/functional/lang/eval-fail-recursion.err.exp | 4 ---- tests/functional/lang/eval-fail-remove.err.exp | 4 ---- tests/functional/lang/eval-fail-scope-5.err.exp | 8 -------- tests/functional/lang/eval-fail-seq.err.exp | 4 ---- tests/functional/lang/eval-fail-set.err.exp | 2 -- tests/functional/lang/eval-fail-substring.err.exp | 2 -- tests/functional/lang/eval-fail-to-path.err.exp | 2 -- tests/functional/lang/eval-fail-toJSON.err.exp | 12 ------------ .../functional/lang/eval-fail-undeclared-arg.err.exp | 4 ---- .../lang/eval-fail-using-set-as-attr-name.err.exp | 2 -- tests/functional/lang/parse-fail-dup-attrs-1.err.exp | 2 -- tests/functional/lang/parse-fail-dup-attrs-2.err.exp | 2 -- tests/functional/lang/parse-fail-dup-attrs-3.err.exp | 2 -- tests/functional/lang/parse-fail-dup-attrs-4.err.exp | 2 -- tests/functional/lang/parse-fail-dup-attrs-7.err.exp | 2 -- tests/functional/lang/parse-fail-dup-formals.err.exp | 2 -- .../functional/lang/parse-fail-eof-in-string.err.exp | 2 -- .../lang/parse-fail-mixed-nested-attrs1.err.exp | 2 -- .../lang/parse-fail-mixed-nested-attrs2.err.exp | 2 -- tests/functional/lang/parse-fail-patterns-1.err.exp | 2 -- .../lang/parse-fail-regression-20060610.err.exp | 2 -- tests/functional/lang/parse-fail-undef-var-2.err.exp | 2 -- tests/functional/lang/parse-fail-undef-var.err.exp | 2 -- tests/functional/lang/parse-fail-utf8.err.exp | 2 -- 45 files changed, 1 insertion(+), 140 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 72c346cb5..bc0194d59 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -173,10 +173,9 @@ static bool printUnknownLocations = getEnv("_NIX_EVAL_SHOW_UNKNOWN_LOCATIONS").h static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std::shared_ptr & pos) { bool hasPos = pos && *pos; if (hasPos) { - oss << "\n" << indent << ANSI_BLUE << "at " ANSI_WARNING << *pos << ANSI_NORMAL << ":"; + oss << indent << ANSI_BLUE << "at " ANSI_WARNING << *pos << ANSI_NORMAL << ":"; if (auto loc = pos->getCodeLines()) { - oss << "\n"; printCodeLines(oss, "", *pos, *loc); oss << "\n"; } diff --git a/tests/functional/lang/eval-fail-abort.err.exp b/tests/functional/lang/eval-fail-abort.err.exp index 345232d3f..20e7b9e18 100644 --- a/tests/functional/lang/eval-fail-abort.err.exp +++ b/tests/functional/lang/eval-fail-abort.err.exp @@ -1,8 +1,6 @@ error: … while calling the 'abort' builtin - at /pwd/lang/eval-fail-abort.nix:1:14: - 1| if true then abort "this should fail" else 1 | ^ 2| diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp index ad91a22aa..37e0bd9ee 100644 --- a/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp @@ -1,8 +1,6 @@ error: … while calling the 'addDrvOutputDependencies' builtin - at /pwd/lang/eval-fail-addDrvOutputDependencies-empty-context.nix:1:1: - 1| builtins.addDrvOutputDependencies "" | ^ 2| diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp index bb389db4e..6828e03c8 100644 --- a/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp @@ -1,8 +1,6 @@ error: … while calling the 'addDrvOutputDependencies' builtin - at /pwd/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix:18:4: - 17| 18| in builtins.addDrvOutputDependencies combo-path | ^ diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp index 070381118..72b5e6368 100644 --- a/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp @@ -1,8 +1,6 @@ error: … while calling the 'addDrvOutputDependencies' builtin - at /pwd/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix:9:4: - 8| 9| in builtins.addDrvOutputDependencies drv.outPath | ^ diff --git a/tests/functional/lang/eval-fail-assert.err.exp b/tests/functional/lang/eval-fail-assert.err.exp index aeecd8167..0656ec81c 100644 --- a/tests/functional/lang/eval-fail-assert.err.exp +++ b/tests/functional/lang/eval-fail-assert.err.exp @@ -1,35 +1,27 @@ error: … while evaluating the attribute 'body' - at /pwd/lang/eval-fail-assert.nix:4:3: - 3| 4| body = x "x"; | ^ 5| } … from call site - at /pwd/lang/eval-fail-assert.nix:4:10: - 3| 4| body = x "x"; | ^ 5| } … while calling 'x' - at /pwd/lang/eval-fail-assert.nix:2:7: - 1| let { 2| x = arg: assert arg == "y"; 123; | ^ 3| error: assertion '(arg == "y")' failed - at /pwd/lang/eval-fail-assert.nix:2:12: - 1| let { 2| x = arg: assert arg == "y"; 123; | ^ diff --git a/tests/functional/lang/eval-fail-attr-name-type.err.exp b/tests/functional/lang/eval-fail-attr-name-type.err.exp index 5f9a073dd..23cceb58a 100644 --- a/tests/functional/lang/eval-fail-attr-name-type.err.exp +++ b/tests/functional/lang/eval-fail-attr-name-type.err.exp @@ -1,17 +1,13 @@ error: … while evaluating the attribute 'puppy."${key}"' - at /pwd/lang/eval-fail-attr-name-type.nix:3:5: - 2| attrs = { 3| puppy.doggy = {}; | ^ 4| }; … while evaluating an attribute name - at /pwd/lang/eval-fail-attr-name-type.nix:7:17: - 6| in 7| attrs.puppy.${key} | ^ diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp index eb73e9a52..b461b2e02 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp @@ -1,8 +1,6 @@ error: … while evaluating a path segment - at /pwd/lang/eval-fail-bad-string-interpolation-1.nix:1:2: - 1| "${x: x}" | ^ 2| diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp index ac14f329b..95f4c2460 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp @@ -1,8 +1,6 @@ error: … while evaluating a path segment - at /pwd/lang/eval-fail-bad-string-interpolation-3.nix:1:3: - 1| ''${x: x}'' | ^ 2| diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp index 07843a480..4950f8ddb 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp @@ -1,8 +1,6 @@ error: … while evaluating a path segment - at /pwd/lang/eval-fail-bad-string-interpolation-4.nix:9:3: - 8| # The error message should not be too long. 9| ''${pkgs}'' | ^ diff --git a/tests/functional/lang/eval-fail-blackhole.err.exp b/tests/functional/lang/eval-fail-blackhole.err.exp index f0618d8ac..95e33a5fe 100644 --- a/tests/functional/lang/eval-fail-blackhole.err.exp +++ b/tests/functional/lang/eval-fail-blackhole.err.exp @@ -1,17 +1,13 @@ error: … while evaluating the attribute 'body' - at /pwd/lang/eval-fail-blackhole.nix:2:3: - 1| let { 2| body = x; | ^ 3| x = y; error: infinite recursion encountered - at /pwd/lang/eval-fail-blackhole.nix:3:7: - 2| body = x; 3| x = y; | ^ diff --git a/tests/functional/lang/eval-fail-call-primop.err.exp b/tests/functional/lang/eval-fail-call-primop.err.exp index 19b407c47..ae5b55ed4 100644 --- a/tests/functional/lang/eval-fail-call-primop.err.exp +++ b/tests/functional/lang/eval-fail-call-primop.err.exp @@ -1,8 +1,6 @@ error: … while calling the 'length' builtin - at /pwd/lang/eval-fail-call-primop.nix:1:1: - 1| builtins.length 1 | ^ 2| diff --git a/tests/functional/lang/eval-fail-deepseq.err.exp b/tests/functional/lang/eval-fail-deepseq.err.exp index 5e204ba73..11b62340d 100644 --- a/tests/functional/lang/eval-fail-deepseq.err.exp +++ b/tests/functional/lang/eval-fail-deepseq.err.exp @@ -1,24 +1,18 @@ error: … while calling the 'deepSeq' builtin - at /pwd/lang/eval-fail-deepseq.nix:1:1: - 1| builtins.deepSeq { x = abort "foo"; } 456 | ^ 2| … while evaluating the attribute 'x' - at /pwd/lang/eval-fail-deepseq.nix:1:20: - 1| builtins.deepSeq { x = abort "foo"; } 456 | ^ 2| … while calling the 'abort' builtin - at /pwd/lang/eval-fail-deepseq.nix:1:24: - 1| builtins.deepSeq { x = abort "foo"; } 456 | ^ 2| diff --git a/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp b/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp index c5fa67523..834f9c67b 100644 --- a/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp +++ b/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp @@ -1,17 +1,13 @@ error: … while evaluating the attribute 'set' - at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:3: - 1| { 2| set = { "${"" + "b"}" = 1; }; | ^ 3| set = { "${"b" + ""}" = 2; }; error: dynamic attribute 'b' already defined at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:11 - at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:3:11: - 2| set = { "${"" + "b"}" = 1; }; 3| set = { "${"b" + ""}" = 2; }; | ^ diff --git a/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp index 0069285fb..7cb08af8a 100644 --- a/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp +++ b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp @@ -1,35 +1,27 @@ error: … while calling the 'foldl'' builtin - at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:2:1: - 1| # Tests that the result of applying op is forced even if the value is never used 2| builtins.foldl' | ^ 3| (_: f: f null) … while calling anonymous lambda - at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:3:7: - 2| builtins.foldl' 3| (_: f: f null) | ^ 4| null … from call site - at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:3:10: - 2| builtins.foldl' 3| (_: f: f null) | ^ 4| null … while calling anonymous lambda - at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:5:6: - 4| null 5| [ (_: throw "Not the final value, but is still forced!") (_: 23) ] | ^ diff --git a/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp b/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp index 5b60d253d..73f9df8cc 100644 --- a/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp +++ b/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp @@ -1,8 +1,6 @@ error: … while calling the 'fromTOML' builtin - at /pwd/lang/eval-fail-fromTOML-timestamps.nix:1:1: - 1| builtins.fromTOML '' | ^ 2| key = "value" diff --git a/tests/functional/lang/eval-fail-hashfile-missing.err.exp b/tests/functional/lang/eval-fail-hashfile-missing.err.exp index 6d38608c0..1e4653927 100644 --- a/tests/functional/lang/eval-fail-hashfile-missing.err.exp +++ b/tests/functional/lang/eval-fail-hashfile-missing.err.exp @@ -1,8 +1,6 @@ error: … while calling the 'toString' builtin - at /pwd/lang/eval-fail-hashfile-missing.nix:4:3: - 3| in 4| toString (builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"])) | ^ diff --git a/tests/functional/lang/eval-fail-list.err.exp b/tests/functional/lang/eval-fail-list.err.exp index 24d682118..4320fc022 100644 --- a/tests/functional/lang/eval-fail-list.err.exp +++ b/tests/functional/lang/eval-fail-list.err.exp @@ -1,8 +1,6 @@ error: … while evaluating one of the elements to concatenate - at /pwd/lang/eval-fail-list.nix:1:2: - 1| 8++1 | ^ 2| diff --git a/tests/functional/lang/eval-fail-missing-arg.err.exp b/tests/functional/lang/eval-fail-missing-arg.err.exp index 61fabf0d5..3b162fe1b 100644 --- a/tests/functional/lang/eval-fail-missing-arg.err.exp +++ b/tests/functional/lang/eval-fail-missing-arg.err.exp @@ -1,16 +1,12 @@ error: … from call site - at /pwd/lang/eval-fail-missing-arg.nix:1:1: - 1| ({x, y, z}: x + y + z) {x = "foo"; z = "bar";} | ^ 2| error: function 'anonymous lambda' called without required argument 'y' - at /pwd/lang/eval-fail-missing-arg.nix:1:2: - 1| ({x, y, z}: x + y + z) {x = "foo"; z = "bar";} | ^ 2| diff --git a/tests/functional/lang/eval-fail-not-throws.err.exp b/tests/functional/lang/eval-fail-not-throws.err.exp index b290afb0a..fc81f7277 100644 --- a/tests/functional/lang/eval-fail-not-throws.err.exp +++ b/tests/functional/lang/eval-fail-not-throws.err.exp @@ -1,16 +1,12 @@ error: … in the argument of the not operator - at /pwd/lang/eval-fail-not-throws.nix:1:4: - 1| ! (throw "uh oh!") | ^ 2| … while calling the 'throw' builtin - at /pwd/lang/eval-fail-not-throws.nix:1:4: - 1| ! (throw "uh oh!") | ^ 2| diff --git a/tests/functional/lang/eval-fail-path-slash.err.exp b/tests/functional/lang/eval-fail-path-slash.err.exp index f0011c97f..e3531d352 100644 --- a/tests/functional/lang/eval-fail-path-slash.err.exp +++ b/tests/functional/lang/eval-fail-path-slash.err.exp @@ -1,7 +1,5 @@ error: path has a trailing slash - at /pwd/lang/eval-fail-path-slash.nix:6:12: - 5| # and https://nixos.org/nix-dev/2016-June/020829.html 6| /nix/store/ | ^ diff --git a/tests/functional/lang/eval-fail-recursion.err.exp b/tests/functional/lang/eval-fail-recursion.err.exp index af64133cb..19380dc65 100644 --- a/tests/functional/lang/eval-fail-recursion.err.exp +++ b/tests/functional/lang/eval-fail-recursion.err.exp @@ -1,16 +1,12 @@ error: … in the right operand of the update (//) operator - at /pwd/lang/eval-fail-recursion.nix:1:12: - 1| let a = {} // a; in a.foo | ^ 2| error: infinite recursion encountered - at /pwd/lang/eval-fail-recursion.nix:1:15: - 1| let a = {} // a; in a.foo | ^ 2| diff --git a/tests/functional/lang/eval-fail-remove.err.exp b/tests/functional/lang/eval-fail-remove.err.exp index e82cdac98..292b3c3f3 100644 --- a/tests/functional/lang/eval-fail-remove.err.exp +++ b/tests/functional/lang/eval-fail-remove.err.exp @@ -1,17 +1,13 @@ error: … while evaluating the attribute 'body' - at /pwd/lang/eval-fail-remove.nix:4:3: - 3| 4| body = (removeAttrs attrs ["x"]).x; | ^ 5| } error: attribute 'x' missing - at /pwd/lang/eval-fail-remove.nix:4:10: - 3| 4| body = (removeAttrs attrs ["x"]).x; | ^ diff --git a/tests/functional/lang/eval-fail-scope-5.err.exp b/tests/functional/lang/eval-fail-scope-5.err.exp index 22b6166f8..b0b05cad7 100644 --- a/tests/functional/lang/eval-fail-scope-5.err.exp +++ b/tests/functional/lang/eval-fail-scope-5.err.exp @@ -1,35 +1,27 @@ error: … while evaluating the attribute 'body' - at /pwd/lang/eval-fail-scope-5.nix:8:3: - 7| 8| body = f {}; | ^ 9| … from call site - at /pwd/lang/eval-fail-scope-5.nix:8:10: - 7| 8| body = f {}; | ^ 9| … while calling 'f' - at /pwd/lang/eval-fail-scope-5.nix:6:7: - 5| 6| f = {x ? y, y ? x}: x + y; | ^ 7| error: infinite recursion encountered - at /pwd/lang/eval-fail-scope-5.nix:6:12: - 5| 6| f = {x ? y, y ? x}: x + y; | ^ diff --git a/tests/functional/lang/eval-fail-seq.err.exp b/tests/functional/lang/eval-fail-seq.err.exp index 33a7e9491..3e3d71b15 100644 --- a/tests/functional/lang/eval-fail-seq.err.exp +++ b/tests/functional/lang/eval-fail-seq.err.exp @@ -1,16 +1,12 @@ error: … while calling the 'seq' builtin - at /pwd/lang/eval-fail-seq.nix:1:1: - 1| builtins.seq (abort "foo") 2 | ^ 2| … while calling the 'abort' builtin - at /pwd/lang/eval-fail-seq.nix:1:15: - 1| builtins.seq (abort "foo") 2 | ^ 2| diff --git a/tests/functional/lang/eval-fail-set.err.exp b/tests/functional/lang/eval-fail-set.err.exp index 0d0140508..6dd646e11 100644 --- a/tests/functional/lang/eval-fail-set.err.exp +++ b/tests/functional/lang/eval-fail-set.err.exp @@ -1,7 +1,5 @@ error: undefined variable 'x' - at /pwd/lang/eval-fail-set.nix:1:3: - 1| 8.x | ^ 2| diff --git a/tests/functional/lang/eval-fail-substring.err.exp b/tests/functional/lang/eval-fail-substring.err.exp index 5c58be29a..0457a826e 100644 --- a/tests/functional/lang/eval-fail-substring.err.exp +++ b/tests/functional/lang/eval-fail-substring.err.exp @@ -1,8 +1,6 @@ error: … while calling the 'substring' builtin - at /pwd/lang/eval-fail-substring.nix:1:1: - 1| builtins.substring (builtins.sub 0 1) 1 "x" | ^ 2| diff --git a/tests/functional/lang/eval-fail-to-path.err.exp b/tests/functional/lang/eval-fail-to-path.err.exp index 4ffa2cf6d..d6b17be99 100644 --- a/tests/functional/lang/eval-fail-to-path.err.exp +++ b/tests/functional/lang/eval-fail-to-path.err.exp @@ -1,8 +1,6 @@ error: … while calling the 'toPath' builtin - at /pwd/lang/eval-fail-to-path.nix:1:1: - 1| builtins.toPath "foo/bar" | ^ 2| diff --git a/tests/functional/lang/eval-fail-toJSON.err.exp b/tests/functional/lang/eval-fail-toJSON.err.exp index 4e618c203..4f6003437 100644 --- a/tests/functional/lang/eval-fail-toJSON.err.exp +++ b/tests/functional/lang/eval-fail-toJSON.err.exp @@ -1,25 +1,19 @@ error: … while calling the 'toJSON' builtin - at /pwd/lang/eval-fail-toJSON.nix:1:1: - 1| builtins.toJSON { | ^ 2| a.b = [ … while evaluating attribute 'a' - at /pwd/lang/eval-fail-toJSON.nix:2:3: - 1| builtins.toJSON { 2| a.b = [ | ^ 3| true … while evaluating attribute 'b' - at /pwd/lang/eval-fail-toJSON.nix:2:3: - 1| builtins.toJSON { 2| a.b = [ | ^ @@ -28,27 +22,21 @@ error: … while evaluating list element at index 3 … while evaluating attribute 'c' - at /pwd/lang/eval-fail-toJSON.nix:7:7: - 6| { 7| c.d = throw "hah no"; | ^ 8| } … while evaluating attribute 'd' - at /pwd/lang/eval-fail-toJSON.nix:7:7: - 6| { 7| c.d = throw "hah no"; | ^ 8| } … while calling the 'throw' builtin - at /pwd/lang/eval-fail-toJSON.nix:7:13: - 6| { 7| c.d = throw "hah no"; | ^ diff --git a/tests/functional/lang/eval-fail-undeclared-arg.err.exp b/tests/functional/lang/eval-fail-undeclared-arg.err.exp index 30db743c7..6e13a138e 100644 --- a/tests/functional/lang/eval-fail-undeclared-arg.err.exp +++ b/tests/functional/lang/eval-fail-undeclared-arg.err.exp @@ -1,16 +1,12 @@ error: … from call site - at /pwd/lang/eval-fail-undeclared-arg.nix:1:1: - 1| ({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";} | ^ 2| error: function 'anonymous lambda' called with unexpected argument 'y' - at /pwd/lang/eval-fail-undeclared-arg.nix:1:2: - 1| ({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";} | ^ 2| diff --git a/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp b/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp index 811d01b03..0a4f56ac5 100644 --- a/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp +++ b/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp @@ -1,8 +1,6 @@ error: … while evaluating an attribute name - at /pwd/lang/eval-fail-using-set-as-attr-name.nix:5:10: - 4| in 5| attr.${key} | ^ diff --git a/tests/functional/lang/parse-fail-dup-attrs-1.err.exp b/tests/functional/lang/parse-fail-dup-attrs-1.err.exp index 4fe6b7a1f..6c3a3510c 100644 --- a/tests/functional/lang/parse-fail-dup-attrs-1.err.exp +++ b/tests/functional/lang/parse-fail-dup-attrs-1.err.exp @@ -1,7 +1,5 @@ error: attribute 'x' already defined at «stdin»:1:3 - at «stdin»:3:3: - 2| y = 456; 3| x = 789; | ^ diff --git a/tests/functional/lang/parse-fail-dup-attrs-2.err.exp b/tests/functional/lang/parse-fail-dup-attrs-2.err.exp index 3aba2891f..fecdece20 100644 --- a/tests/functional/lang/parse-fail-dup-attrs-2.err.exp +++ b/tests/functional/lang/parse-fail-dup-attrs-2.err.exp @@ -1,7 +1,5 @@ error: attribute 'x' already defined at «stdin»:9:5 - at «stdin»:10:17: - 9| x = 789; 10| inherit (as) x; | ^ diff --git a/tests/functional/lang/parse-fail-dup-attrs-3.err.exp b/tests/functional/lang/parse-fail-dup-attrs-3.err.exp index 3aba2891f..fecdece20 100644 --- a/tests/functional/lang/parse-fail-dup-attrs-3.err.exp +++ b/tests/functional/lang/parse-fail-dup-attrs-3.err.exp @@ -1,7 +1,5 @@ error: attribute 'x' already defined at «stdin»:9:5 - at «stdin»:10:17: - 9| x = 789; 10| inherit (as) x; | ^ diff --git a/tests/functional/lang/parse-fail-dup-attrs-4.err.exp b/tests/functional/lang/parse-fail-dup-attrs-4.err.exp index ff68446a1..f85ffea51 100644 --- a/tests/functional/lang/parse-fail-dup-attrs-4.err.exp +++ b/tests/functional/lang/parse-fail-dup-attrs-4.err.exp @@ -1,7 +1,5 @@ error: attribute 'services.ssh.port' already defined at «stdin»:2:3 - at «stdin»:3:3: - 2| services.ssh.port = 22; 3| services.ssh.port = 23; | ^ diff --git a/tests/functional/lang/parse-fail-dup-attrs-7.err.exp b/tests/functional/lang/parse-fail-dup-attrs-7.err.exp index 512a499ca..98cea9dae 100644 --- a/tests/functional/lang/parse-fail-dup-attrs-7.err.exp +++ b/tests/functional/lang/parse-fail-dup-attrs-7.err.exp @@ -1,7 +1,5 @@ error: attribute 'x' already defined at «stdin»:6:12 - at «stdin»:7:12: - 6| inherit x; 7| inherit x; | ^ diff --git a/tests/functional/lang/parse-fail-dup-formals.err.exp b/tests/functional/lang/parse-fail-dup-formals.err.exp index 1d566fb33..d7c7e0237 100644 --- a/tests/functional/lang/parse-fail-dup-formals.err.exp +++ b/tests/functional/lang/parse-fail-dup-formals.err.exp @@ -1,6 +1,4 @@ error: duplicate formal function argument 'x' - at «stdin»:1:8: - 1| {x, y, x}: x | ^ diff --git a/tests/functional/lang/parse-fail-eof-in-string.err.exp b/tests/functional/lang/parse-fail-eof-in-string.err.exp index f9fa72312..b28d35950 100644 --- a/tests/functional/lang/parse-fail-eof-in-string.err.exp +++ b/tests/functional/lang/parse-fail-eof-in-string.err.exp @@ -1,7 +1,5 @@ error: syntax error, unexpected end of file, expecting '"' - at «stdin»:3:5: - 2| # Note that this file must not end with a newline. 3| a 1"$ | ^ diff --git a/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp index 32f776795..a4472156b 100644 --- a/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp +++ b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp @@ -1,7 +1,5 @@ error: attribute 'z' already defined at «stdin»:3:16 - at «stdin»:2:3: - 1| { 2| x.z = 3; | ^ diff --git a/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp index 0437cd50c..ead1f0dbd 100644 --- a/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp +++ b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp @@ -1,7 +1,5 @@ error: attribute 'y' already defined at «stdin»:3:9 - at «stdin»:2:3: - 1| { 2| x.y.y = 3; | ^ diff --git a/tests/functional/lang/parse-fail-patterns-1.err.exp b/tests/functional/lang/parse-fail-patterns-1.err.exp index 634a04aaa..6ba39d884 100644 --- a/tests/functional/lang/parse-fail-patterns-1.err.exp +++ b/tests/functional/lang/parse-fail-patterns-1.err.exp @@ -1,7 +1,5 @@ error: duplicate formal function argument 'args' - at «stdin»:1:1: - 1| args@{args, x, y, z}: x | ^ 2| diff --git a/tests/functional/lang/parse-fail-regression-20060610.err.exp b/tests/functional/lang/parse-fail-regression-20060610.err.exp index 167d01e85..d8875a6a5 100644 --- a/tests/functional/lang/parse-fail-regression-20060610.err.exp +++ b/tests/functional/lang/parse-fail-regression-20060610.err.exp @@ -1,7 +1,5 @@ error: undefined variable 'gcc' - at «stdin»:8:12: - 7| 8| body = ({ | ^ diff --git a/tests/functional/lang/parse-fail-undef-var-2.err.exp b/tests/functional/lang/parse-fail-undef-var-2.err.exp index 77c96bbd2..a58d8dca4 100644 --- a/tests/functional/lang/parse-fail-undef-var-2.err.exp +++ b/tests/functional/lang/parse-fail-undef-var-2.err.exp @@ -1,7 +1,5 @@ error: syntax error, unexpected ':', expecting '}' - at «stdin»:3:13: - 2| 3| f = {x, y : | ^ diff --git a/tests/functional/lang/parse-fail-undef-var.err.exp b/tests/functional/lang/parse-fail-undef-var.err.exp index 48e88747f..3d143d9af 100644 --- a/tests/functional/lang/parse-fail-undef-var.err.exp +++ b/tests/functional/lang/parse-fail-undef-var.err.exp @@ -1,7 +1,5 @@ error: undefined variable 'y' - at «stdin»:1:4: - 1| x: y | ^ 2| diff --git a/tests/functional/lang/parse-fail-utf8.err.exp b/tests/functional/lang/parse-fail-utf8.err.exp index 6087479a3..e83abdb9e 100644 --- a/tests/functional/lang/parse-fail-utf8.err.exp +++ b/tests/functional/lang/parse-fail-utf8.err.exp @@ -1,6 +1,4 @@ error: syntax error, unexpected invalid token, expecting end of file - at «stdin»:1:5: - 1| 123 | ^ From 7f5ed330e40d0aa2a2f907b2d4157329ff953cd2 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Sat, 16 Dec 2023 07:05:31 -0500 Subject: [PATCH 830/909] Document `Makefile` variables in `hacking.md` (#9620) --- doc/manual/src/contributing/hacking.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 4d3d66397..421ac981c 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -64,6 +64,15 @@ $ nix build You can also build Nix for one of the [supported platforms](#platforms). +## Makefile variables + +- `ENABLE_BUILD=yes` to enable building the C++ code. +- `ENABLE_TESTS=yes` to enable building the tests. +- `OPTIMIZE=1` to enable optimizations. +- `doc_generate=yes` to enable building the documentation (manual, man pages, etc.). + + The docs can take a while to build, so you may want to disable this for local development. + ## Building Nix To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: From c05d4fadd5f0943de0a00b17c85626d73152da66 Mon Sep 17 00:00:00 2001 From: David Arnold Date: Sat, 16 Dec 2023 23:07:17 +0100 Subject: [PATCH 831/909] fix: valid branch name --- src/libutil/url-parts.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 07bc8d0cd..e968eea4b 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -25,7 +25,7 @@ const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRege /// A Git ref (i.e. branch or tag name). /// \todo check that this is correct. -const static std::string refRegexS = "[a-zA-Z0-9@][a-zA-Z0-9_.\\/@-]*"; +const static std::string refRegexS = "[a-zA-Z0-9@][a-zA-Z0-9_.\\/@-+]*"; extern std::regex refRegex; /// Instead of defining what a good Git Ref is, we define what a bad Git Ref is From 4f9580085441a4255ce746a4cc498b45cc25a899 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 18 Dec 2023 11:41:52 +0100 Subject: [PATCH 832/909] add cross-reference --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index e28615cdc..b35dc37a1 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -631,7 +631,7 @@ public: At least one of the following condition must be met for Nix to accept copying a store object from another - Nix store (such as a substituter): + Nix store (such as a [substituter](#conf-substituters)): - the store object has been signed using a key in the trusted keys list - the [`require-sigs`](#conf-require-sigs) option has been set to `false` From d19a6675286a38edf8970459cbf454322f8151cb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 18 Dec 2023 13:54:40 +0100 Subject: [PATCH 833/909] CODEOWNERS: unsubscribe fricklerhandwerk (#9614) --- .github/CODEOWNERS | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 39d595199..526fecabf 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -10,16 +10,8 @@ # This file .github/CODEOWNERS @edolstra -# Public documentation -/doc @fricklerhandwerk -*.md @fricklerhandwerk - # Documentation of built-in functions -src/libexpr/primops.cc @fricklerhandwerk @roberth -# Documentation on experimental features -src/libutil/experimental-features.cc @fricklerhandwerk -# Documentation on configuration settings -src/libstore/globals.hh @fricklerhandwerk +src/libexpr/primops.cc @roberth # Libstore layer /src/libstore @thufschmitt From dfc876531f269950a4e183a4f77a813c421d7d64 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 4 Nov 2023 16:25:41 -0400 Subject: [PATCH 834/909] Organize content addressing, use `SourceAccessor` with `Store::addToStore` Co-authored-by: Robert Hensing --- perl/lib/Nix/Store.xs | 12 ++- src/libcmd/installable-value.cc | 2 +- src/libexpr/eval.cc | 2 +- src/libexpr/primops.cc | 2 +- src/libfetchers/cache.cc | 16 ++-- src/libfetchers/cache.hh | 6 +- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/git.cc | 4 +- src/libfetchers/github.cc | 4 +- src/libfetchers/input-accessor.cc | 39 ++++++--- src/libfetchers/input-accessor.hh | 8 +- src/libfetchers/mercurial.cc | 25 ++++-- src/libfetchers/tarball.cc | 14 +-- src/libstore/binary-cache-store.cc | 56 ++++++------ src/libstore/binary-cache-store.hh | 24 ++++-- src/libstore/build/local-derivation-goal.cc | 90 +++++++++---------- src/libstore/build/worker.cc | 4 +- src/libstore/content-address.cc | 28 ++++-- src/libstore/content-address.hh | 38 ++++----- src/libstore/daemon.cc | 19 +---- src/libstore/legacy-ssh-store.hh | 15 ++-- src/libstore/local-store.cc | 95 ++++++++------------- src/libstore/local-store.hh | 22 ++--- src/libstore/optimise-store.cc | 15 +++- src/libstore/remote-store.cc | 9 +- src/libstore/remote-store.hh | 11 ++- src/libstore/store-api.cc | 90 ++++++++++--------- src/libstore/store-api.hh | 36 +++++--- src/libstore/store-dir-config.hh | 14 +-- src/libutil/file-content-address.cc | 49 +++++++++++ src/libutil/file-content-address.hh | 56 ++++++++++++ src/libutil/hash.cc | 9 -- src/libutil/hash.hh | 7 +- src/nix-store/nix-store.cc | 20 ++++- src/nix/add-to-store.cc | 42 +++------ src/nix/hash.cc | 11 +-- src/nix/prefetch.cc | 7 +- tests/unit/libexpr/primops.cc | 2 +- 38 files changed, 515 insertions(+), 390 deletions(-) create mode 100644 src/libutil/file-content-address.cc create mode 100644 src/libutil/file-content-address.hh diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 82c7db608..4964b8a34 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -13,6 +13,7 @@ #include "globals.hh" #include "store-api.hh" #include "crypto.hh" +#include "posix-source-accessor.hh" #include #include @@ -205,7 +206,10 @@ void importPaths(int fd, int dontCheckSigs) SV * hashPath(char * algo, int base32, char * path) PPCODE: try { - Hash h = hashPath(parseHashAlgo(algo), path).first; + PosixSourceAccessor accessor; + Hash h = hashPath( + accessor, CanonPath::fromCwd(path), + FileIngestionMethod::Recursive, parseHashAlgo(algo)).first; auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { @@ -281,7 +285,11 @@ SV * addToStore(char * srcPath, int recursive, char * algo) PPCODE: try { auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashAlgo(algo)); + PosixSourceAccessor accessor; + auto path = store()->addToStore( + std::string(baseNameOf(srcPath)), + accessor, CanonPath::fromCwd(srcPath), + method, parseHashAlgo(algo)); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 08ad35105..bdc34bbe3 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -44,7 +44,7 @@ ref InstallableValue::require(ref installable) std::optional InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx) { if (v.type() == nPath) { - auto storePath = v.path().fetchToStore(state->store); + auto storePath = v.path().fetchToStore(*state->store); return {{ .path = DerivedPath::Opaque { .path = std::move(storePath), diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 1552e3e92..c9c25c898 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2317,7 +2317,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat auto dstPath = i != srcToStore.end() ? i->second : [&]() { - auto dstPath = path.fetchToStore(store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair); + auto dstPath = path.fetchToStore(*store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair); allowPath(dstPath); srcToStore.insert_or_assign(path, dstPath); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d78a28c73..75ee1e38d 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2229,7 +2229,7 @@ static void addPath( }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - auto dstPath = path.fetchToStore(state.store, name, method, filter.get(), state.repair); + auto dstPath = path.fetchToStore(*state.store, name, method, filter.get(), state.repair); if (expectedHash && expectedStorePath != dstPath) state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); state.allowAndSetStorePathString(dstPath, v); diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 63b05bdab..e071b4717 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -106,7 +106,7 @@ struct CacheImpl : Cache } void add( - ref store, + Store & store, const Attrs & inAttrs, const Attrs & infoAttrs, const StorePath & storePath, @@ -115,13 +115,13 @@ struct CacheImpl : Cache _state.lock()->add.use() (attrsToJSON(inAttrs).dump()) (attrsToJSON(infoAttrs).dump()) - (store->printStorePath(storePath)) + (store.printStorePath(storePath)) (locked) (time(0)).exec(); } std::optional> lookup( - ref store, + Store & store, const Attrs & inAttrs) override { if (auto res = lookupExpired(store, inAttrs)) { @@ -134,7 +134,7 @@ struct CacheImpl : Cache } std::optional lookupExpired( - ref store, + Store & store, const Attrs & inAttrs) override { auto state(_state.lock()); @@ -148,19 +148,19 @@ struct CacheImpl : Cache } auto infoJSON = stmt.getStr(0); - auto storePath = store->parseStorePath(stmt.getStr(1)); + auto storePath = store.parseStorePath(stmt.getStr(1)); auto locked = stmt.getInt(2) != 0; auto timestamp = stmt.getInt(3); - store->addTempRoot(storePath); - if (!store->isValidPath(storePath)) { + store.addTempRoot(storePath); + if (!store.isValidPath(storePath)) { // FIXME: we could try to substitute 'storePath'. debug("ignoring disappeared cache entry '%s'", inAttrsJSON); return {}; } debug("using cache entry '%s' -> '%s', '%s'", - inAttrsJSON, infoJSON, store->printStorePath(storePath)); + inAttrsJSON, infoJSON, store.printStorePath(storePath)); return Result { .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)), diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh index f70589267..791d77025 100644 --- a/src/libfetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -50,14 +50,14 @@ struct Cache /* Old cache for things that have a store path. */ virtual void add( - ref store, + Store & store, const Attrs & inAttrs, const Attrs & infoAttrs, const StorePath & storePath, bool locked) = 0; virtual std::optional> lookup( - ref store, + Store & store, const Attrs & inAttrs) = 0; struct Result @@ -68,7 +68,7 @@ struct Cache }; virtual std::optional lookupExpired( - ref store, + Store & store, const Attrs & inAttrs) = 0; }; diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 7ec1f9802..f309e5993 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -374,7 +374,7 @@ void InputScheme::clone(const Input & input, const Path & destDir) const std::pair InputScheme::fetch(ref store, const Input & input) { auto [accessor, input2] = getAccessor(store, input); - auto storePath = SourcePath(accessor).fetchToStore(store, input2.getName()); + auto storePath = SourcePath(accessor).fetchToStore(*store, input2.getName()); return {storePath, input2}; } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 5dac66930..01cd28427 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -368,14 +368,14 @@ struct GitInputScheme : InputScheme RepoInfo getRepoInfo(const Input & input) const { - auto checkHashType = [&](const std::optional & hash) + auto checkHashAlgorithm = [&](const std::optional & hash) { if (hash.has_value() && !(hash->algo == HashAlgorithm::SHA1 || hash->algo == HashAlgorithm::SHA256)) throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true)); }; if (auto rev = input.getRev()) - checkHashType(rev); + checkHashAlgorithm(rev); RepoInfo repoInfo; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 70acb9354..498e41357 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -201,7 +201,7 @@ struct GitArchiveInputScheme : InputScheme {"rev", rev->gitRev()}, }); - if (auto res = getCache()->lookup(store, lockedAttrs)) { + if (auto res = getCache()->lookup(*store, lockedAttrs)) { input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified")); return {std::move(res->second), input}; } @@ -213,7 +213,7 @@ struct GitArchiveInputScheme : InputScheme input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); getCache()->add( - store, + *store, lockedAttrs, { {"rev", rev->gitRev()}, diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index 1f793bf1d..a647f5915 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -5,10 +5,10 @@ namespace nix { StorePath InputAccessor::fetchToStore( - ref store, + Store & store, const CanonPath & path, std::string_view name, - FileIngestionMethod method, + ContentAddressMethod method, PathFilter * filter, RepairFlag repair) { @@ -20,10 +20,24 @@ StorePath InputAccessor::fetchToStore( if (!filter && fingerprint) { cacheKey = fetchers::Attrs{ {"_what", "fetchToStore"}, - {"store", store->storeDir}, + {"store", store.storeDir}, {"name", std::string(name)}, {"fingerprint", *fingerprint}, - {"method", (uint8_t) method}, + { + "method", + std::visit(overloaded { + [](const TextIngestionMethod &) { + return "text"; + }, + [](const FileIngestionMethod & fim) { + switch (fim) { + case FileIngestionMethod::Flat: return "flat"; + case FileIngestionMethod::Recursive: return "nar"; + default: assert(false); + } + }, + }, method.raw), + }, {"path", path.abs()} }; if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) { @@ -35,17 +49,14 @@ StorePath InputAccessor::fetchToStore( Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path))); - auto source = sinkToSource([&](Sink & sink) { - if (method == FileIngestionMethod::Recursive) - dumpPath(path, sink, filter ? *filter : defaultPathFilter); - else - readFile(path, sink); - }); + auto filter2 = filter ? *filter : defaultPathFilter; auto storePath = settings.readOnlyMode - ? store->computeStorePathFromDump(*source, name, method, HashAlgorithm::SHA256).first - : store->addToStoreFromDump(*source, name, method, HashAlgorithm::SHA256, repair); + ? store.computeStorePath( + name, *this, path, method, HashAlgorithm::SHA256, {}, filter2).first + : store.addToStore( + name, *this, path, method, HashAlgorithm::SHA256, {}, filter2, repair); if (cacheKey) fetchers::getCache()->add(store, *cacheKey, {}, storePath, true); @@ -60,9 +71,9 @@ std::ostream & operator << (std::ostream & str, const SourcePath & path) } StorePath SourcePath::fetchToStore( - ref store, + Store & store, std::string_view name, - FileIngestionMethod method, + ContentAddressMethod method, PathFilter * filter, RepairFlag repair) const { diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index f385e6231..d2a21cb4b 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -30,10 +30,10 @@ struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this store, + Store & store, const CanonPath & path, std::string_view name = "source", - FileIngestionMethod method = FileIngestionMethod::Recursive, + ContentAddressMethod method = FileIngestionMethod::Recursive, PathFilter * filter = nullptr, RepairFlag repair = NoRepair); }; @@ -116,9 +116,9 @@ struct SourcePath * Copy this `SourcePath` to the Nix store. */ StorePath fetchToStore( - ref store, + Store & store, std::string_view name = "source", - FileIngestionMethod method = FileIngestionMethod::Recursive, + ContentAddressMethod method = FileIngestionMethod::Recursive, PathFilter * filter = nullptr, RepairFlag repair = NoRepair) const; diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 6056b9a3c..9982389ab 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -6,6 +6,7 @@ #include "tarfile.hh" #include "store-api.hh" #include "url-parts.hh" +#include "posix-source-accessor.hh" #include "fetch-settings.hh" @@ -210,7 +211,12 @@ struct MercurialInputScheme : InputScheme return files.count(file); }; - auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, filter); + PosixSourceAccessor accessor; + auto storePath = store->addToStore( + input.getName(), + accessor, CanonPath { actualPath }, + FileIngestionMethod::Recursive, HashAlgorithm::SHA256, {}, + filter); return {std::move(storePath), input}; } @@ -218,7 +224,7 @@ struct MercurialInputScheme : InputScheme if (!input.getRef()) input.attrs.insert_or_assign("ref", "default"); - auto checkHashType = [&](const std::optional & hash) + auto checkHashAlgorithm = [&](const std::optional & hash) { if (hash.has_value() && hash->algo != HashAlgorithm::SHA1) throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(HashFormat::Base16, true)); @@ -227,7 +233,7 @@ struct MercurialInputScheme : InputScheme auto getLockedAttrs = [&]() { - checkHashType(input.getRev()); + checkHashAlgorithm(input.getRev()); return Attrs({ {"type", "hg"}, @@ -246,7 +252,7 @@ struct MercurialInputScheme : InputScheme }; if (input.getRev()) { - if (auto res = getCache()->lookup(store, getLockedAttrs())) + if (auto res = getCache()->lookup(*store, getLockedAttrs())) return makeResult(res->first, std::move(res->second)); } @@ -259,7 +265,7 @@ struct MercurialInputScheme : InputScheme {"ref", *input.getRef()}, }); - if (auto res = getCache()->lookup(store, unlockedAttrs)) { + if (auto res = getCache()->lookup(*store, unlockedAttrs)) { auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), HashAlgorithm::SHA1); if (!input.getRev() || input.getRev() == rev2) { input.attrs.insert_or_assign("rev", rev2.gitRev()); @@ -305,7 +311,7 @@ struct MercurialInputScheme : InputScheme auto revCount = std::stoull(tokens[1]); input.attrs.insert_or_assign("ref", tokens[2]); - if (auto res = getCache()->lookup(store, getLockedAttrs())) + if (auto res = getCache()->lookup(*store, getLockedAttrs())) return makeResult(res->first, std::move(res->second)); Path tmpDir = createTempDir(); @@ -315,7 +321,8 @@ struct MercurialInputScheme : InputScheme deletePath(tmpDir + "/.hg_archival.txt"); - auto storePath = store->addToStore(name, tmpDir); + PosixSourceAccessor accessor; + auto storePath = store->addToStore(name, accessor, CanonPath { tmpDir }); Attrs infoAttrs({ {"rev", input.getRev()->gitRev()}, @@ -324,14 +331,14 @@ struct MercurialInputScheme : InputScheme if (!_input.getRev()) getCache()->add( - store, + *store, unlockedAttrs, infoAttrs, storePath, false); getCache()->add( - store, + *store, getLockedAttrs(), infoAttrs, storePath, diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 086366180..3b7709440 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -8,6 +8,7 @@ #include "tarfile.hh" #include "types.hh" #include "split.hh" +#include "posix-source-accessor.hh" namespace nix::fetchers { @@ -26,7 +27,7 @@ DownloadFileResult downloadFile( {"name", name}, }); - auto cached = getCache()->lookupExpired(store, inAttrs); + auto cached = getCache()->lookupExpired(*store, inAttrs); auto useCached = [&]() -> DownloadFileResult { @@ -91,7 +92,7 @@ DownloadFileResult downloadFile( } getCache()->add( - store, + *store, inAttrs, infoAttrs, *storePath, @@ -99,7 +100,7 @@ DownloadFileResult downloadFile( if (url != res.effectiveUri) getCache()->add( - store, + *store, { {"type", "file"}, {"url", res.effectiveUri}, @@ -130,7 +131,7 @@ DownloadTarballResult downloadTarball( {"name", name}, }); - auto cached = getCache()->lookupExpired(store, inAttrs); + auto cached = getCache()->lookupExpired(*store, inAttrs); if (cached && !cached->expired) return { @@ -156,7 +157,8 @@ DownloadTarballResult downloadTarball( throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); auto topDir = tmpDir + "/" + members.begin()->name; lastModified = lstat(topDir).st_mtime; - unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, defaultPathFilter, NoRepair); + PosixSourceAccessor accessor; + unpackedStorePath = store->addToStore(name, accessor, CanonPath { topDir }, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, {}, defaultPathFilter, NoRepair); } Attrs infoAttrs({ @@ -168,7 +170,7 @@ DownloadTarballResult downloadTarball( infoAttrs.emplace("immutableUrl", *res.immutableUrl); getCache()->add( - store, + *store, inAttrs, infoAttrs, *unpackedStorePath, diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 2837e8934..19aa283fc 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -300,8 +300,13 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource }}); } -StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) +StorePath BinaryCacheStore::addToStoreFromDump( + Source & dump, + std::string_view name, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) { if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) unsupported("addToStoreFromDump"); @@ -309,15 +314,14 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n ValidPathInfo info { *this, name, - FixedOutputInfo { - .method = method, - .hash = nar.first, - .references = { + ContentAddressWithReferences::fromParts( + method, + nar.first, + { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, - }, - }, + }), nar.first, }; info.narSize = nar.second; @@ -399,42 +403,36 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, } StorePath BinaryCacheStore::addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashAlgorithm hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) + std::string_view name, + SourceAccessor & accessor, + const CanonPath & path, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + PathFilter & filter, + RepairFlag repair) { /* FIXME: Make BinaryCacheStore::addToStoreCommon support non-recursive+sha256 so we can just use the default implementation of this method in terms of addToStoreFromDump. */ - HashSink sink { hashAlgo }; - if (method == FileIngestionMethod::Recursive) { - dumpPath(srcPath, sink, filter); - } else { - readFile(srcPath, sink); - } - auto h = sink.finish().first; + auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first; auto source = sinkToSource([&](Sink & sink) { - dumpPath(srcPath, sink, filter); + accessor.dumpPath(path, sink, filter); }); return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, name, - FixedOutputInfo { - .method = method, - .hash = h, - .references = { + ContentAddressWithReferences::fromParts( + method, + h, + { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, - }, - }, + }), nar.first, }; info.narSize = nar.second; diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 395e1b479..dbe4ac180 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -123,17 +123,23 @@ public: void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs) override; - StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) override; + StorePath addToStoreFromDump( + Source & dump, + std::string_view name, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) override; StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashAlgorithm hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override; + std::string_view name, + SourceAccessor & accessor, + const CanonPath & srcPath, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + PathFilter & filter, + RepairFlag repair) override; StorePath addTextToStore( std::string_view name, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 802b39f84..e4828dd2f 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -20,6 +20,7 @@ #include "child.hh" #include "unix-domain-socket.hh" #include "posix-fs-canonicalise.hh" +#include "posix-source-accessor.hh" #include #include @@ -1290,13 +1291,14 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In { throw Error("queryPathFromHashPart"); } StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashAlgorithm hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override + std::string_view name, + SourceAccessor & accessor, + const CanonPath & srcPath, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + PathFilter & filter, + RepairFlag repair) override { throw Error("addToStore"); } void addToStore(const ValidPathInfo & info, Source & narSource, @@ -1318,14 +1320,14 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In } StorePath addToStoreFromDump( - Source & dump, - std::string_view name, - FileIngestionMethod method, - HashAlgorithm hashAlgo, - RepairFlag repair, - const StorePathSet & references) override + Source & dump, + std::string_view name, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) override { - auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references); + auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, references, repair); goal.addDependency(path); return path; } @@ -2453,8 +2455,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() throw BuildError( "output path %1% without valid stats info", actualPath); - if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } || - outputHash.method == ContentAddressMethod { TextIngestionMethod {} }) + if (outputHash.method.getFileIngestionMethod() == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0) @@ -2466,38 +2467,23 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() rewriteOutput(outputRewrites); /* FIXME optimize and deduplicate with addToStore */ std::string oldHashPart { scratchPath->hashPart() }; - HashModuloSink caSink {outputHash.hashAlgo, oldHashPart }; - std::visit(overloaded { - [&](const TextIngestionMethod &) { - readFile(actualPath, caSink); - }, - [&](const FileIngestionMethod & m2) { - switch (m2) { - case FileIngestionMethod::Recursive: - dumpPath(actualPath, caSink); - break; - case FileIngestionMethod::Flat: - readFile(actualPath, caSink); - break; - } - }, - }, outputHash.method.raw); - auto got = caSink.finish().first; + auto got = ({ + HashModuloSink caSink { outputHash.hashAlgo, oldHashPart }; + PosixSourceAccessor accessor; + dumpPath( + accessor, CanonPath { actualPath }, + caSink, + outputHash.method.getFileIngestionMethod()); + caSink.finish().first; + }); - auto optCA = ContentAddressWithReferences::fromPartsOpt( - outputHash.method, - std::move(got), - rewriteRefs()); - if (!optCA) { - // TODO track distinct failure modes separately (at the time of - // writing there is just one but `nullopt` is unclear) so this - // message can't get out of sync. - throw BuildError("output path '%s' has illegal content address, probably a spurious self-reference with text hashing"); - } ValidPathInfo newInfo0 { worker.store, outputPathName(drv->name, outputName), - std::move(*optCA), + ContentAddressWithReferences::fromParts( + outputHash.method, + std::move(got), + rewriteRefs()), Hash::dummy, }; if (*scratchPath != newInfo0.path) { @@ -2511,9 +2497,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() std::string(newInfo0.path.hashPart())}}); } - HashResult narHashAndSize = hashPath(HashAlgorithm::SHA256, actualPath); - newInfo0.narHash = narHashAndSize.first; - newInfo0.narSize = narHashAndSize.second; + { + PosixSourceAccessor accessor; + HashResult narHashAndSize = hashPath( + accessor, CanonPath { actualPath }, + FileIngestionMethod::Recursive, HashAlgorithm::SHA256); + newInfo0.narHash = narHashAndSize.first; + newInfo0.narSize = narHashAndSize.second; + } assert(newInfo0.ca); return newInfo0; @@ -2531,7 +2522,10 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() std::string { scratchPath->hashPart() }, std::string { requiredFinalPath.hashPart() }); rewriteOutput(outputRewrites); - auto narHashAndSize = hashPath(HashAlgorithm::SHA256, actualPath); + PosixSourceAccessor accessor; + HashResult narHashAndSize = hashPath( + accessor, CanonPath { actualPath }, + FileIngestionMethod::Recursive, HashAlgorithm::SHA256); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; auto refs = rewriteRefs(); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 9b8c36286..399ad47fd 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -519,7 +519,9 @@ bool Worker::pathContentsGood(const StorePath & path) if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(info->narHash.algo, store.printStorePath(path)); + HashResult current = hashPath( + *store.getFSAccessor(), CanonPath { store.printStorePath(path) }, + FileIngestionMethod::Recursive, info->narHash.algo); Hash nullHash(HashAlgorithm::SHA256); res = info->narHash == nullHash || info->narHash == current.first; } diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index f42a13126..fc408f5af 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -50,6 +50,18 @@ std::string ContentAddressMethod::render(HashAlgorithm ha) const }, raw); } +FileIngestionMethod ContentAddressMethod::getFileIngestionMethod() const +{ + return std::visit(overloaded { + [&](const TextIngestionMethod & th) { + return FileIngestionMethod::Flat; + }, + [&](const FileIngestionMethod & fim) { + return fim; + } + }, raw); +} + std::string ContentAddress::render() const { return std::visit(overloaded { @@ -79,7 +91,7 @@ static std::pair parseContentAddressMethodP prefix = *optPrefix; } - auto parseHashType_ = [&](){ + auto parseHashAlgorithm_ = [&](){ auto hashTypeRaw = splitPrefixTo(rest, ':'); if (!hashTypeRaw) throw UsageError("content address hash must be in form ':', but found: %s", wholeInput); @@ -90,7 +102,7 @@ static std::pair parseContentAddressMethodP // Switch on prefix if (prefix == "text") { // No parsing of the ingestion method, "text" only support flat. - HashAlgorithm hashAlgo = parseHashType_(); + HashAlgorithm hashAlgo = parseHashAlgorithm_(); return { TextIngestionMethod {}, std::move(hashAlgo), @@ -100,7 +112,7 @@ static std::pair parseContentAddressMethodP auto method = FileIngestionMethod::Flat; if (splitPrefix(rest, "r:")) method = FileIngestionMethod::Recursive; - HashAlgorithm hashAlgo = parseHashType_(); + HashAlgorithm hashAlgo = parseHashAlgorithm_(); return { std::move(method), std::move(hashAlgo), @@ -176,13 +188,13 @@ ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const Con }, ca.method.raw); } -std::optional ContentAddressWithReferences::fromPartsOpt( - ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept +ContentAddressWithReferences ContentAddressWithReferences::fromParts( + ContentAddressMethod method, Hash hash, StoreReferences refs) { return std::visit(overloaded { - [&](TextIngestionMethod _) -> std::optional { + [&](TextIngestionMethod _) -> ContentAddressWithReferences { if (refs.self) - return std::nullopt; + throw Error("self-reference not allowed with text hashing"); return ContentAddressWithReferences { TextInfo { .hash = std::move(hash), @@ -190,7 +202,7 @@ std::optional ContentAddressWithReferences::fromPa } }; }, - [&](FileIngestionMethod m2) -> std::optional { + [&](FileIngestionMethod m2) -> ContentAddressWithReferences { return ContentAddressWithReferences { FixedOutputInfo { .method = m2, diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 05234da38..6863ad260 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -4,6 +4,7 @@ #include #include "hash.hh" #include "path.hh" +#include "file-content-address.hh" #include "comparator.hh" #include "variant-wrapper.hh" @@ -31,22 +32,6 @@ namespace nix { */ struct TextIngestionMethod : std::monostate { }; -/** - * An enumeration of the main ways we can serialize file system - * objects. - */ -enum struct FileIngestionMethod : uint8_t { - /** - * Flat-file hashing. Directly ingest the contents of a single file - */ - Flat = 0, - /** - * Recursive (or NAR) hashing. Serializes the file-system object in Nix - * Archive format and ingest that - */ - Recursive = 1 -}; - /** * Compute the prefix to the hash algorithm which indicates how the * files were ingested. @@ -54,7 +39,7 @@ enum struct FileIngestionMethod : uint8_t { std::string makeFileIngestionPrefix(FileIngestionMethod m); /** - * An enumeration of all the ways we can serialize file system objects. + * An enumeration of all the ways we can content-address store objects. * * Just the type of a content address. Combine with the hash itself, and * we have a `ContentAddress` as defined below. Combine that, in turn, @@ -102,7 +87,15 @@ struct ContentAddressMethod * * The rough inverse of `parse()`. */ - std::string render(HashAlgorithm ha) const; + std::string render(HashAlgorithm ht) const; + + /** + * Get the underlying way to content-address file system objects. + * + * Different ways of hashing store objects may use the same method + * for hashing file systeme objects. + */ + FileIngestionMethod getFileIngestionMethod() const; }; @@ -266,11 +259,12 @@ struct ContentAddressWithReferences * * @param refs References to other store objects or oneself. * - * Do note that not all combinations are supported; `nullopt` is - * returns for invalid combinations. + * @note note that all combinations are supported. This is a + * *partial function* and exceptions will be thrown for invalid + * combinations. */ - static std::optional fromPartsOpt( - ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept; + static ContentAddressWithReferences fromParts( + ContentAddressMethod method, Hash hash, StoreReferences refs); ContentAddressMethod getMethod() const; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index a112d6d31..574263c68 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -403,22 +403,9 @@ static void performOp(TunnelLogger * logger, ref store, auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parse(camStr); auto hashAlgo = hashAlgo_; // work around clang bug FramedSource source(from); - // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. - return std::visit(overloaded { - [&](const TextIngestionMethod &) { - if (hashAlgo != HashAlgorithm::SHA256) - throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", - name, printHashAlgo(hashAlgo)); - // We could stream this by changing Store - std::string contents = source.drain(); - auto path = store->addTextToStore(name, contents, refs, repair); - return store->queryPathInfo(path); - }, - [&](const FileIngestionMethod & fim) { - auto path = store->addToStoreFromDump(source, name, fim, hashAlgo, repair, refs); - return store->queryPathInfo(path); - }, - }, contentAddressMethod.raw); + // TODO these two steps are essentially RemoteStore::addCAToStore. Move it up to Store. + auto path = store->addToStoreFromDump(source, name, contentAddressMethod, hashAlgo, refs, repair); + return store->queryPathInfo(path); }(); logger->stopWork(); diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index c40c256bb..8b142ba2a 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -59,13 +59,14 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor { unsupported("queryPathFromHashPart"); } StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashAlgorithm hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override + std::string_view name, + SourceAccessor & accessor, + const CanonPath & srcPath, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + PathFilter & filter, + RepairFlag repair) override { unsupported("addToStore"); } StorePath addTextToStore( diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7e82bae28..cd8bf24f8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -13,6 +13,7 @@ #include "compression.hh" #include "signals.hh" #include "posix-fs-canonicalise.hh" +#include "posix-source-accessor.hh" #include #include @@ -1088,11 +1089,22 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (info.ca) { auto & specified = *info.ca; - auto actualHash = hashCAPath( - specified.method, - specified.hash.algo, - info.path - ); + auto actualHash = ({ + HashModuloSink caSink { + specified.hash.algo, + std::string { info.path.hashPart() }, + }; + PosixSourceAccessor accessor; + dumpPath( + *getFSAccessor(false), + CanonPath { printStorePath(info.path) }, + caSink, + specified.method.getFileIngestionMethod()); + ContentAddress { + .method = specified.method, + .hash = caSink.finish().first, + }; + }); if (specified.hash != actualHash.hash) { throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), @@ -1115,8 +1127,13 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, } -StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name, - FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) +StorePath LocalStore::addToStoreFromDump( + Source & source0, + std::string_view name, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) { /* For computing the store path. */ auto hashSink = std::make_unique(hashAlgo); @@ -1166,25 +1183,21 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name delTempDir = std::make_unique(tempDir); tempPath = tempDir + "/x"; - if (method == FileIngestionMethod::Recursive) - restorePath(tempPath, bothSource); - else - writeFile(tempPath, bothSource); + restorePath(tempPath, bothSource, method.getFileIngestionMethod()); dump.clear(); } auto [hash, size] = hashSink->finish(); - ContentAddressWithReferences desc = FixedOutputInfo { - .method = method, - .hash = hash, - .references = { + auto desc = ContentAddressWithReferences::fromParts( + method, + hash, + { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, - }, - }; + }); auto dstPath = makeFixedOutputPathFromCA(name, desc); @@ -1207,11 +1220,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name if (inMemory) { StringSource dumpSource { dump }; - /* Restore from the NAR in memory. */ - if (method == FileIngestionMethod::Recursive) - restorePath(realPath, dumpSource); - else - writeFile(realPath, dumpSource); + /* Restore from the buffer in memory. */ + restorePath(realPath, dumpSource, method.getFileIngestionMethod()); } else { /* Move the temporary path we restored above. */ moveFile(tempPath, realPath); @@ -1389,7 +1399,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) for (auto & link : readDirectory(linksDir)) { printMsg(lvlTalkative, "checking contents of '%s'", link.name); Path linkPath = linksDir + "/" + link.name; - std::string hash = hashPath(HashAlgorithm::SHA256, linkPath).first.to_string(HashFormat::Nix32, false); + PosixSourceAccessor accessor; + std::string hash = hashPath( + accessor, CanonPath { linkPath }, + FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first.to_string(HashFormat::Nix32, false); if (hash != link.name) { printError("link '%s' was modified! expected hash '%s', got '%s'", linkPath, link.name, hash); @@ -1696,42 +1709,6 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id, } } -ContentAddress LocalStore::hashCAPath( - const ContentAddressMethod & method, const HashAlgorithm & hashAlgo, - const StorePath & path) -{ - return hashCAPath(method, hashAlgo, Store::toRealPath(path), path.hashPart()); -} - -ContentAddress LocalStore::hashCAPath( - const ContentAddressMethod & method, - const HashAlgorithm & hashAlgo, - const Path & path, - const std::string_view pathHash -) -{ - HashModuloSink caSink ( hashAlgo, std::string(pathHash) ); - std::visit(overloaded { - [&](const TextIngestionMethod &) { - readFile(path, caSink); - }, - [&](const FileIngestionMethod & m2) { - switch (m2) { - case FileIngestionMethod::Recursive: - dumpPath(path, caSink); - break; - case FileIngestionMethod::Flat: - readFile(path, caSink); - break; - } - }, - }, method.raw); - return ContentAddress { - .method = method, - .hash = caSink.finish().first, - }; -} - void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) { assert(drvPath.isDerivation()); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ee605b5a2..a8323fe5a 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -177,8 +177,13 @@ public: void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override; - StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) override; + StorePath addToStoreFromDump( + Source & dump, + std::string_view name, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) override; StorePath addTextToStore( std::string_view name, @@ -350,19 +355,6 @@ private: void signPathInfo(ValidPathInfo & info); void signRealisation(Realisation &); - // XXX: Make a generic `Store` method - ContentAddress hashCAPath( - const ContentAddressMethod & method, - const HashAlgorithm & hashAlgo, - const StorePath & path); - - ContentAddress hashCAPath( - const ContentAddressMethod & method, - const HashAlgorithm & hashAlgo, - const Path & path, - const std::string_view pathHash - ); - void addBuildLog(const StorePath & drvPath, std::string_view log) override; friend struct LocalDerivationGoal; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index b395453d1..a494e6ecc 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -2,6 +2,7 @@ #include "globals.hh" #include "signals.hh" #include "posix-fs-canonicalise.hh" +#include "posix-source-accessor.hh" #include #include @@ -146,7 +147,12 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, Also note that if `path' is a symlink, then we're hashing the contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ - Hash hash = hashPath(HashAlgorithm::SHA256, path).first; + Hash hash = ({ + PosixSourceAccessor accessor; + hashPath( + accessor, CanonPath { path }, + FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first; + }); debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true)); /* Check if this is a known hash. */ @@ -156,7 +162,12 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, if (pathExists(linkPath)) { auto stLink = lstat(linkPath); if (st.st_size != stLink.st_size - || (repair && hash != hashPath(HashAlgorithm::SHA256, linkPath).first)) + || (repair && hash != ({ + PosixSourceAccessor accessor; + hashPath( + accessor, CanonPath { linkPath }, + FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first; + }))) { // XXX: Consider overwriting linkPath with our valid version. warn("removing corrupted link '%s'", linkPath); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index dd6347468..567776b67 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -502,8 +502,13 @@ ref RemoteStore::addCAToStore( } -StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) +StorePath RemoteStore::addToStoreFromDump( + Source & dump, + std::string_view name, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) { return addCAToStore(dump, name, method, hashAlgo, references, repair)->path; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index f2e34c1a3..68824a737 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -82,10 +82,15 @@ public: RepairFlag repair); /** - * Add a content-addressable store path. Does not support references. `dump` will be drained. + * Add a content-addressable store path. `dump` will be drained. */ - StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; + StorePath addToStoreFromDump( + Source & dump, + std::string_view name, + ContentAddressMethod method = FileIngestionMethod::Recursive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + const StorePathSet & references = StorePathSet(), + RepairFlag repair = NoRepair) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7f35e74af..5b4c6c765 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -232,22 +232,28 @@ StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const } -std::pair StoreDirConfig::computeStorePathFromDump( - Source & dump, - std::string_view name, - FileIngestionMethod method, - HashAlgorithm hashAlgo, - const StorePathSet & references) const +std::pair StoreDirConfig::computeStorePath( + std::string_view name, + SourceAccessor & accessor, + const CanonPath & path, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + PathFilter & filter) const { - HashSink sink(hashAlgo); - dump.drainInto(sink); - auto h = sink.finish().first; - FixedOutputInfo caInfo { - .method = method, - .hash = h, - .references = {}, + auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first; + return { + makeFixedOutputPathFromCA( + name, + ContentAddressWithReferences::fromParts( + method, + h, + { + .others = references, + .self = false, + })), + h, }; - return std::make_pair(makeFixedOutputPath(name, caInfo), h); } @@ -264,22 +270,19 @@ StorePath StoreDirConfig::computeStorePathForText( StorePath Store::addToStore( - std::string_view name, - const Path & _srcPath, - FileIngestionMethod method, - HashAlgorithm hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) + std::string_view name, + SourceAccessor & accessor, + const CanonPath & path, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + PathFilter & filter, + RepairFlag repair) { - Path srcPath(absPath(_srcPath)); auto source = sinkToSource([&](Sink & sink) { - if (method == FileIngestionMethod::Recursive) - dumpPath(srcPath, sink, filter); - else - readFile(srcPath, sink); + dumpPath(accessor, path, sink, method.getFileIngestionMethod(), filter); }); - return addToStoreFromDump(*source, name, method, hashAlgo, repair, references); + return addToStoreFromDump(*source, name, method, hashAlgo, references, repair); } void Store::addMultipleToStore( @@ -404,9 +407,13 @@ digraph graphname { fileSink -> caHashSink } */ -ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, - FileIngestionMethod method, HashAlgorithm hashAlgo, - std::optional expectedCAHash) +ValidPathInfo Store::addToStoreSlow( + std::string_view name, + SourceAccessor & accessor, + const CanonPath & srcPath, + ContentAddressMethod method, HashAlgorithm hashAlgo, + const StorePathSet & references, + std::optional expectedCAHash) { HashSink narHashSink { HashAlgorithm::SHA256 }; HashSink caHashSink { hashAlgo }; @@ -425,7 +432,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, srcPath. The fact that we use scratchpadSink as a temporary buffer here is an implementation detail. */ auto fileSource = sinkToSource([&](Sink & scratchpadSink) { - dumpPath(srcPath, scratchpadSink); + accessor.dumpPath(srcPath, scratchpadSink); }); /* tapped provides the same data as fileSource, but we also write all the @@ -433,9 +440,11 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, TeeSource tapped { *fileSource, narSink }; NullParseSink blank; - auto & parseSink = method == FileIngestionMethod::Flat + auto & parseSink = method.getFileIngestionMethod() == FileIngestionMethod::Flat ? (ParseSink &) fileSink - : (ParseSink &) blank; + : method.getFileIngestionMethod() == FileIngestionMethod::Recursive + ? (ParseSink &) blank + : (abort(), (ParseSink &)*(ParseSink *)nullptr); // handled both cases /* The information that flows from tapped (besides being replicated in narSink), is now put in parseSink. */ @@ -452,21 +461,24 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, if (expectedCAHash && expectedCAHash != hash) throw Error("hash mismatch for '%s'", srcPath); + ValidPathInfo info { *this, name, - FixedOutputInfo { - .method = method, - .hash = hash, - .references = {}, - }, + ContentAddressWithReferences::fromParts( + method, + hash, + { + .others = references, + .self = false, + }), narHash, }; info.narSize = narSize; if (!isValidPath(info.path)) { auto source = sinkToSource([&](Sink & scratchpadSink) { - dumpPath(srcPath, scratchpadSink); + accessor.dumpPath(srcPath, scratchpadSink); }); addToStore(info, *source); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 2c883ce97..fc0a82a73 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -427,22 +427,28 @@ public: * libutil/archive.hh). */ virtual StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, - HashAlgorithm hashAlgo = HashAlgorithm::SHA256, - PathFilter & filter = defaultPathFilter, - RepairFlag repair = NoRepair, - const StorePathSet & references = StorePathSet()); + std::string_view name, + SourceAccessor & accessor, + const CanonPath & path, + ContentAddressMethod method = FileIngestionMethod::Recursive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + const StorePathSet & references = StorePathSet(), + PathFilter & filter = defaultPathFilter, + RepairFlag repair = NoRepair); /** * Copy the contents of a path to the store and register the * validity the resulting path, using a constant amount of * memory. */ - ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, - std::optional expectedCAHash = {}); + ValidPathInfo addToStoreSlow( + std::string_view name, + SourceAccessor & accessor, + const CanonPath & path, + ContentAddressMethod method = FileIngestionMethod::Recursive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + const StorePathSet & references = StorePathSet(), + std::optional expectedCAHash = {}); /** * Like addToStore(), but the contents of the path are contained @@ -453,9 +459,13 @@ public: * * \todo remove? */ - virtual StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, RepairFlag repair = NoRepair, - const StorePathSet & references = StorePathSet()) + virtual StorePath addToStoreFromDump( + Source & dump, + std::string_view name, + ContentAddressMethod method = FileIngestionMethod::Recursive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + const StorePathSet & references = StorePathSet(), + RepairFlag repair = NoRepair) { unsupported("addToStoreFromDump"); } /** diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index 8dafca096..0fc8ded9c 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -91,15 +91,17 @@ struct StoreDirConfig : public Config StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; /** - * Read-only variant of addToStoreFromDump(). It returns the store - * path to which a NAR or flat file would be written. + * Read-only variant of addToStore(). It returns the store + * path for the given file sytem object. */ - std::pair computeStorePathFromDump( - Source & dump, + std::pair computeStorePath( std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, + SourceAccessor & accessor, + const CanonPath & path, + ContentAddressMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, - const StorePathSet & references = {}) const; + const StorePathSet & references = {}, + PathFilter & filter = defaultPathFilter) const; /** * Preparatory part of addTextToStore(). diff --git a/src/libutil/file-content-address.cc b/src/libutil/file-content-address.cc new file mode 100644 index 000000000..9917986f6 --- /dev/null +++ b/src/libutil/file-content-address.cc @@ -0,0 +1,49 @@ +#include "file-content-address.hh" +#include "archive.hh" + +namespace nix { + +void dumpPath( + SourceAccessor & accessor, const CanonPath & path, + Sink & sink, + FileIngestionMethod method, + PathFilter & filter) +{ + switch (method) { + case FileIngestionMethod::Flat: + accessor.readFile(path, sink); + break; + case FileIngestionMethod::Recursive: + accessor.dumpPath(path, sink, filter); + break; + } +} + + +void restorePath( + const Path & path, + Source & source, + FileIngestionMethod method) +{ + switch (method) { + case FileIngestionMethod::Flat: + writeFile(path, source); + break; + case FileIngestionMethod::Recursive: + restorePath(path, source); + break; + } +} + + +HashResult hashPath( + SourceAccessor & accessor, const CanonPath & path, + FileIngestionMethod method, HashAlgorithm ht, + PathFilter & filter) +{ + HashSink sink { ht }; + dumpPath(accessor, path, sink, method, filter); + return sink.finish(); +} + +} diff --git a/src/libutil/file-content-address.hh b/src/libutil/file-content-address.hh new file mode 100644 index 000000000..8e93f5847 --- /dev/null +++ b/src/libutil/file-content-address.hh @@ -0,0 +1,56 @@ +#pragma once +///@file + +#include "source-accessor.hh" +#include "fs-sink.hh" +#include "util.hh" + +namespace nix { + +/** + * An enumeration of the main ways we can serialize file system + * objects. + */ +enum struct FileIngestionMethod : uint8_t { + /** + * Flat-file hashing. Directly ingest the contents of a single file + */ + Flat = 0, + /** + * Recursive (or NAR) hashing. Serializes the file-system object in + * Nix Archive format and ingest that. + */ + Recursive = 1, +}; + +/** + * Dump a serialization of the given file system object. + */ +void dumpPath( + SourceAccessor & accessor, const CanonPath & path, + Sink & sink, + FileIngestionMethod method, + PathFilter & filter = defaultPathFilter); + +/** + * Restore a serialization of the given file system object. + * + * @TODO use an arbitrary `ParseSink`. + */ +void restorePath( + const Path & path, + Source & source, + FileIngestionMethod method); + +/** + * Compute the hash of the given file system object according to the + * given method. + * + * The hash is defined as (essentially) hashString(ht, dumpPath(path)). + */ +HashResult hashPath( + SourceAccessor & accessor, const CanonPath & path, + FileIngestionMethod method, HashAlgorithm ht, + PathFilter & filter = defaultPathFilter); + +} diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 30456ae5c..502afbda2 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -367,15 +367,6 @@ HashResult HashSink::currentHash() } -HashResult hashPath( - HashAlgorithm ha, const Path & path, PathFilter & filter) -{ - HashSink sink(ha); - dumpPath(path, sink, filter); - return sink.finish(); -} - - Hash compressHash(const Hash & hash, unsigned int newSize) { Hash h(hash.algo); diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 7bed9e2bd..2fe9a53f5 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -168,14 +168,11 @@ Hash hashString(HashAlgorithm ha, std::string_view s); Hash hashFile(HashAlgorithm ha, const Path & path); /** - * Compute the hash of the given path, serializing as a Nix Archive and - * then hashing that. + * The final hash and the number of bytes digested. * - * The hash is defined as (essentially) hashString(ht, dumpPath(path)). + * @todo Convert to proper struct */ typedef std::pair HashResult; -HashResult hashPath(HashAlgorithm ha, const Path & path, - PathFilter & filter = defaultPathFilter); /** * Compress a hash to the specified number of bytes by cyclically diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index d361dc0ac..0a0a3ab1a 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -13,6 +13,7 @@ #include "shared.hh" #include "graphml.hh" #include "legacy.hh" +#include "posix-source-accessor.hh" #include "path-with-outputs.hh" #include "posix-fs-canonicalise.hh" @@ -175,8 +176,12 @@ static void opAdd(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); + PosixSourceAccessor accessor; for (auto & i : opArgs) - cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i))); + cout << fmt("%s\n", store->printStorePath(store->addToStore( + std::string(baseNameOf(i)), + accessor, + CanonPath::fromCwd(i)))); } @@ -196,8 +201,14 @@ static void opAddFixed(Strings opFlags, Strings opArgs) HashAlgorithm hashAlgo = parseHashAlgo(opArgs.front()); opArgs.pop_front(); + PosixSourceAccessor accessor; for (auto & i : opArgs) - std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), i, method, hashAlgo).path)); + std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow( + baseNameOf(i), + accessor, + CanonPath::fromCwd(i), + method, + hashAlgo).path)); } @@ -541,7 +552,10 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) if (canonicalise) canonicalisePathMetaData(store->printStorePath(info->path), {}); if (!hashGiven) { - HashResult hash = hashPath(HashAlgorithm::SHA256, store->printStorePath(info->path)); + HashResult hash = hashPath( + *store->getFSAccessor(false), CanonPath { store->printStorePath(info->path) }, + + FileIngestionMethod::Recursive, HashAlgorithm::SHA256); info->narHash = hash.first; info->narSize = hash.second; } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 02de796b5..64a43ecfa 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "store-api.hh" #include "archive.hh" +#include "posix-source-accessor.hh" using namespace nix; @@ -20,7 +21,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; - FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive; + ContentAddressMethod caMethod = FileIngestionMethod::Recursive; CmdAddToStore() { @@ -48,7 +49,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand )", .labels = {"hash-mode"}, .handler = {[this](std::string s) { - this->ingestionMethod = parseIngestionMethod(s); + this->caMethod = parseIngestionMethod(s); }}, }); } @@ -57,36 +58,17 @@ struct CmdAddToStore : MixDryRun, StoreCommand { if (!namePart) namePart = baseNameOf(path); - StringSink sink; - dumpPath(path, sink); + PosixSourceAccessor accessor; - auto narHash = hashString(HashAlgorithm::SHA256, sink.s); + auto path2 = CanonPath::fromCwd(path); - Hash hash = narHash; - if (ingestionMethod == FileIngestionMethod::Flat) { - HashSink hsink(HashAlgorithm::SHA256); - readFile(path, hsink); - hash = hsink.finish().first; - } + auto storePath = dryRun + ? store->computeStorePath( + *namePart, accessor, path2, caMethod, HashAlgorithm::SHA256, {}).first + : store->addToStoreSlow( + *namePart, accessor, path2, caMethod, HashAlgorithm::SHA256, {}).path; - ValidPathInfo info { - *store, - std::move(*namePart), - FixedOutputInfo { - .method = std::move(ingestionMethod), - .hash = std::move(hash), - .references = {}, - }, - narHash, - }; - info.narSize = sink.s.size(); - - if (!dryRun) { - auto source = StringSource(sink.s); - store->addToStore(info, source); - } - - logger->cout("%s", store->printStorePath(info.path)); + logger->cout("%s", store->printStorePath(storePath)); } }; @@ -110,7 +92,7 @@ struct CmdAddFile : CmdAddToStore { CmdAddFile() { - ingestionMethod = FileIngestionMethod::Flat; + caMethod = FileIngestionMethod::Flat; } std::string description() override diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 0bba3b7d2..83694306e 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -5,6 +5,7 @@ #include "shared.hh" #include "references.hh" #include "archive.hh" +#include "posix-source-accessor.hh" using namespace nix; @@ -88,14 +89,8 @@ struct CmdHashBase : Command else hashSink = std::make_unique(ha); - switch (mode) { - case FileIngestionMethod::Flat: - readFile(path, *hashSink); - break; - case FileIngestionMethod::Recursive: - dumpPath(path, *hashSink); - break; - } + PosixSourceAccessor accessor; + dumpPath(accessor, CanonPath::fromCwd(path), *hashSink, mode); Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index bbfeb8aa4..b5d619006 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -9,6 +9,7 @@ #include "attr-path.hh" #include "eval-inline.hh" #include "legacy.hh" +#include "posix-source-accessor.hh" #include @@ -122,7 +123,11 @@ std::tuple prefetchFile( Activity act(*logger, lvlChatty, actUnknown, fmt("adding '%s' to the store", url)); - auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashAlgo, expectedHash); + PosixSourceAccessor accessor; + auto info = store->addToStoreSlow( + *name, + accessor, CanonPath::fromCwd(tmpFile), + ingestionMethod, hashAlgo, {}, expectedHash); storePath = info.path; assert(info.ca); hash = info.ca->hash; diff --git a/tests/unit/libexpr/primops.cc b/tests/unit/libexpr/primops.cc index 31b1b49ae..6d7649b3c 100644 --- a/tests/unit/libexpr/primops.cc +++ b/tests/unit/libexpr/primops.cc @@ -604,7 +604,7 @@ namespace nix { ASSERT_THAT(v, IsStringEq("401b09eab3c013d4ca54922bb802bec8fd5318192b0a75f201d8b3727429080fb337591abd3e44453b954555b7a0812e1081c39b740293f765eae731f5a65ed1")); } - TEST_F(PrimOpTest, hashStringInvalidHashType) { + TEST_F(PrimOpTest, hashStringInvalidHashAlgorithm) { ASSERT_THROW(eval("builtins.hashString \"foobar\" \"asdf\""), Error); } From ed26b186fbed9e5f8df2453a5b1aec0c18b11401 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 8 Nov 2023 21:11:48 -0500 Subject: [PATCH 835/909] Remove now-redundant text-hashing store methods `addTextToStore` and `computeStorePathFromDump` are now redundant. Co-authored-by: Robert Hensing --- src/libexpr/primops.cc | 10 ++- src/libstore/binary-cache-store.cc | 71 +++++++++++---------- src/libstore/binary-cache-store.hh | 6 -- src/libstore/build/local-derivation-goal.cc | 11 ---- src/libstore/content-address.hh | 8 +-- src/libstore/daemon.cc | 5 +- src/libstore/derivations.cc | 10 ++- src/libstore/dummy-store.cc | 7 -- src/libstore/legacy-ssh-store.hh | 7 -- src/libstore/local-store.cc | 52 --------------- src/libstore/local-store.hh | 6 -- src/libstore/remote-store.cc | 10 --- src/libstore/remote-store.hh | 6 -- src/libstore/store-api.cc | 34 +++------- src/libstore/store-api.hh | 10 --- src/libstore/store-dir-config.hh | 23 ------- src/nix-env/user-env.cc | 13 ++-- src/nix/develop.cc | 6 +- 18 files changed, 83 insertions(+), 212 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 75ee1e38d..8b689f0c8 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2072,8 +2072,14 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val } auto storePath = settings.readOnlyMode - ? state.store->computeStorePathForText(name, contents, refs) - : state.store->addTextToStore(name, contents, refs, state.repair); + ? state.store->makeFixedOutputPathFromCA(name, TextInfo { + .hash = hashString(HashAlgorithm::SHA256, contents), + .references = std::move(refs), + }) + : ({ + StringSource s { contents }; + state.store->addToStoreFromDump(s, name, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair); + }); /* Note: we don't need to add `context' to the context of the result, since `storePath' itself has references to the paths diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 19aa283fc..8a3052433 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -12,6 +12,7 @@ #include "thread-pool.hh" #include "callback.hh" #include "signals.hh" +#include "archive.hh" #include #include @@ -308,15 +309,47 @@ StorePath BinaryCacheStore::addToStoreFromDump( const StorePathSet & references, RepairFlag repair) { - if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) - unsupported("addToStoreFromDump"); - return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { + std::optional caHash; + std::string nar; + + if (auto * dump2p = dynamic_cast(&dump)) { + auto & dump2 = *dump2p; + // Hack, this gives us a "replayable" source so we can compute + // multiple hashes more easily. + caHash = hashString(HashAlgorithm::SHA256, dump2.s); + switch (method.getFileIngestionMethod()) { + case FileIngestionMethod::Recursive: + // The dump is already NAR in this case, just use it. + nar = dump2.s; + break; + case FileIngestionMethod::Flat: + // The dump is Flat, so we need to convert it to NAR with a + // single file. + StringSink s; + dumpString(dump2.s, s); + nar = std::move(s.s); + break; + } + } else { + // Otherwise, we have to do th same hashing as NAR so our single + // hash will suffice for both purposes. + if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) + unsupported("addToStoreFromDump"); + } + StringSource narDump { nar }; + + // Use `narDump` if we wrote to `nar`. + Source & narDump2 = nar.size() > 0 + ? static_cast(narDump) + : dump; + + return addToStoreCommon(narDump2, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, name, ContentAddressWithReferences::fromParts( method, - nar.first, + caHash ? *caHash : nar.first, { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus @@ -440,36 +473,6 @@ StorePath BinaryCacheStore::addToStore( })->path; } -StorePath BinaryCacheStore::addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) -{ - auto textHash = hashString(HashAlgorithm::SHA256, s); - auto path = makeTextPath(name, TextInfo { { textHash }, references }); - - if (!repair && isValidPath(path)) - return path; - - StringSink sink; - dumpString(s, sink); - StringSource source(sink.s); - return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { - ValidPathInfo info { - *this, - std::string { name }, - TextInfo { - .hash = textHash, - .references = references, - }, - nar.first, - }; - info.narSize = nar.second; - return info; - })->path; -} - void BinaryCacheStore::queryRealisationUncached(const DrvOutput & id, Callback> callback) noexcept { diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index dbe4ac180..98e43ee6a 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -141,12 +141,6 @@ public: PathFilter & filter, RepairFlag repair) override; - StorePath addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) override; - void registerDrvOutput(const Realisation & info) override; void queryRealisationUncached(const DrvOutput &, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index e4828dd2f..b01d9e237 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1308,17 +1308,6 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In goal.addDependency(info.path); } - StorePath addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair = NoRepair) override - { - auto path = next->addTextToStore(name, s, references, repair); - goal.addDependency(path); - return path; - } - StorePath addToStoreFromDump( Source & dump, std::string_view name, diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 6863ad260..f0973412b 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -109,11 +109,11 @@ struct ContentAddressMethod * serialisation methods (flat file vs NAR). Thus, ‘ca’ has one of the * following forms: * - * - ‘text:sha256:’: For paths - * computed by Store::makeTextPath() / Store::addTextToStore(). + * - `TextIngestionMethod`: + * ‘text:sha256:’ * - * - ‘fixed:::’: For paths computed by - * Store::makeFixedOutputPath() / Store::addToStore(). + * - `FixedIngestionMethod`: + * ‘fixed:::’ */ struct ContentAddress { diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 574263c68..923ea6447 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -483,7 +483,10 @@ static void performOp(TunnelLogger * logger, ref store, std::string s = readString(from); auto refs = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); - auto path = store->addTextToStore(suffix, s, refs, NoRepair); + auto path = ({ + StringSource source { s }; + store->addToStoreFromDump(source, suffix, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, NoRepair); + }); logger->stopWork(); to << store->printStorePath(path); break; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index c35150b57..8a7d660ff 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -143,8 +143,14 @@ StorePath writeDerivation(Store & store, auto suffix = std::string(drv.name) + drvExtension; auto contents = drv.unparse(store, false); return readOnly || settings.readOnlyMode - ? store.computeStorePathForText(suffix, contents, references) - : store.addTextToStore(suffix, contents, references, repair); + ? store.makeFixedOutputPathFromCA(suffix, TextInfo { + .hash = hashString(HashAlgorithm::SHA256, contents), + .references = std::move(references), + }) + : ({ + StringSource s { contents }; + store.addToStoreFromDump(s, suffix, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair); + }); } diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 821cda399..f52a309d1 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -58,13 +58,6 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store RepairFlag repair, CheckSigsFlag checkSigs) override { unsupported("addToStore"); } - StorePath addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) override - { unsupported("addTextToStore"); } - void narFromPath(const StorePath & path, Sink & sink) override { unsupported("narFromPath"); } diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index 8b142ba2a..c5a3ce677 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -69,13 +69,6 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor RepairFlag repair) override { unsupported("addToStore"); } - StorePath addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) override - { unsupported("addTextToStore"); } - private: void putBuildSettings(Connection & conn); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index cd8bf24f8..df1de7752 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1257,58 +1257,6 @@ StorePath LocalStore::addToStoreFromDump( } -StorePath LocalStore::addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, RepairFlag repair) -{ - auto hash = hashString(HashAlgorithm::SHA256, s); - auto dstPath = makeTextPath(name, TextInfo { - .hash = hash, - .references = references, - }); - - addTempRoot(dstPath); - - if (repair || !isValidPath(dstPath)) { - - auto realPath = Store::toRealPath(dstPath); - - PathLocks outputLock({realPath}); - - if (repair || !isValidPath(dstPath)) { - - deletePath(realPath); - - autoGC(); - - writeFile(realPath, s); - - canonicalisePathMetaData(realPath, {}); - - StringSink sink; - dumpString(s, sink); - auto narHash = hashString(HashAlgorithm::SHA256, sink.s); - - optimisePath(realPath, repair); - - ValidPathInfo info { dstPath, narHash }; - info.narSize = sink.s.size(); - info.references = references; - info.ca = { - .method = TextIngestionMethod {}, - .hash = hash, - }; - registerValidPath(info); - } - - outputLock.setDeletion(true); - } - - return dstPath; -} - - /* Create a temporary directory in the store that won't be garbage-collected until the returned FD is closed. */ std::pair LocalStore::createTempDirInStore() diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index a8323fe5a..ba56d3ead 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -185,12 +185,6 @@ public: const StorePathSet & references, RepairFlag repair) override; - StorePath addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) override; - void addTempRoot(const StorePath & path) override; private: diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 567776b67..4d0113594 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -608,16 +608,6 @@ void RemoteStore::addMultipleToStore( } -StorePath RemoteStore::addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) -{ - StringSource source(s); - return addCAToStore(source, name, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair)->path; -} - void RemoteStore::registerDrvOutput(const Realisation & info) { auto conn(getConnection()); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 68824a737..87704985b 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -106,12 +106,6 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; - StorePath addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) override; - void registerDrvOutput(const Realisation & info) override; void queryRealisationUncached(const DrvOutput &, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5b4c6c765..c2516afb5 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -205,25 +205,19 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed } -StorePath StoreDirConfig::makeTextPath(std::string_view name, const TextInfo & info) const -{ - assert(info.hash.algo == HashAlgorithm::SHA256); - return makeStorePath( - makeType(*this, "text", StoreReferences { - .others = info.references, - .self = false, - }), - info.hash, - name); -} - - StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const { // New template return std::visit(overloaded { [&](const TextInfo & ti) { - return makeTextPath(name, ti); + assert(ti.hash.algo == HashAlgorithm::SHA256); + return makeStorePath( + makeType(*this, "text", StoreReferences { + .others = ti.references, + .self = false, + }), + ti.hash, + name); }, [&](const FixedOutputInfo & foi) { return makeFixedOutputPath(name, foi); @@ -257,18 +251,6 @@ std::pair StoreDirConfig::computeStorePath( } -StorePath StoreDirConfig::computeStorePathForText( - std::string_view name, - std::string_view s, - const StorePathSet & references) const -{ - return makeTextPath(name, TextInfo { - .hash = hashString(HashAlgorithm::SHA256, s), - .references = references, - }); -} - - StorePath Store::addToStore( std::string_view name, SourceAccessor & accessor, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index fc0a82a73..96a7ebd7b 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -468,16 +468,6 @@ public: RepairFlag repair = NoRepair) { unsupported("addToStoreFromDump"); } - /** - * Like addToStore, but the contents written to the output path is a - * regular file containing the given string. - */ - virtual StorePath addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair = NoRepair) = 0; - /** * Add a mapping indicating that `deriver!outputName` maps to the output path * `output`. diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index 0fc8ded9c..7ca8c2665 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -86,8 +86,6 @@ struct StoreDirConfig : public Config StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; - StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; /** @@ -102,27 +100,6 @@ struct StoreDirConfig : public Config HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = {}, PathFilter & filter = defaultPathFilter) const; - - /** - * Preparatory part of addTextToStore(). - * - * !!! Computation of the path should take the references given to - * addTextToStore() into account, otherwise we have a (relatively - * minor) security hole: a caller can register a source file with - * bogus references. If there are too many references, the path may - * not be garbage collected when it has to be (not really a problem, - * the caller could create a root anyway), or it may be garbage - * collected when it shouldn't be (more serious). - * - * Hashing the references would solve this (bogus references would - * simply yield a different store path, so other users wouldn't be - * affected), but it has some backwards compatibility issues (the - * hashing scheme changes), so I'm not doing that for now. - */ - StorePath computeStorePathForText( - std::string_view name, - std::string_view s, - const StorePathSet & references) const; }; } diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 34f6bd005..5d01fbf10 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -104,10 +104,15 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* Also write a copy of the list of user environment elements to the store; we need it for future modifications of the environment. */ - std::ostringstream str; - manifest.print(state.symbols, str, true); - auto manifestFile = state.store->addTextToStore("env-manifest.nix", - str.str(), references); + auto manifestFile = ({ + std::ostringstream str; + manifest.print(state.symbols, str, true); + // TODO with C++20 we can use str.view() instead and avoid copy. + std::string str2 = str.str(); + StringSource source { str2 }; + state.store->addToStoreFromDump( + source, "env-manifest.nix", TextIngestionMethod {}, HashAlgorithm::SHA256, references); + }); /* Get the environment builder expression. */ Value envBuilder; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 606b044b0..8db2de491 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -223,7 +223,11 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore if (builder != "bash") throw Error("'nix develop' only works on derivations that use 'bash' as their builder"); - auto getEnvShPath = evalStore->addTextToStore("get-env.sh", getEnvSh, {}); + auto getEnvShPath = ({ + StringSource source { getEnvSh }; + evalStore->addToStoreFromDump( + source, "get-env.sh", TextIngestionMethod {}, HashAlgorithm::SHA256, {}); + }); drv.args = {store->printStorePath(getEnvShPath)}; From b1c559eabccc8890c74f4e520b89e800f6e7ef7e Mon Sep 17 00:00:00 2001 From: tomberek Date: Mon, 18 Dec 2023 10:45:57 -0500 Subject: [PATCH 836/909] docs: add link to project board to PRs (#9630) * docs: add link to project board to PRs * Update .github/PULL_REQUEST_TEMPLATE.md Co-authored-by: Valentin Gagarin * fix wording * add note on the process --------- Co-authored-by: Valentin Gagarin --- .github/PULL_REQUEST_TEMPLATE.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 217b19108..d12a4d36c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -10,6 +10,8 @@ -# Priorities +# Priorities and Process Add :+1: to [pull requests you find important](https://github.com/NixOS/nix/pulls?q=is%3Aopen+sort%3Areactions-%2B1-desc). + +The Nix maintainer team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19) to [schedule and track reviews](https://github.com/NixOS/nix/tree/master/maintainers#project-board-protocol). From 7feabf7d44c960563350a246358d4e36bd598d60 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 14 Dec 2023 00:05:03 -0500 Subject: [PATCH 837/909] Split `--disable-tests`, fix cross builds It might seem obnoxious to have yet more configure flags, but I found controlling both the unit and functional tests with one flag was quite confusing because they are so different: - unit tests depending on building, functional tests don't (e.g. when we test already-built Nix) - unit tests can be installed, functional tests cannot - unit tests neeed extra libraries (GTest, RapidCheck), functional tests need extra executables (jq). - unit tests are run by `make check`, functional tests are run by `make installcheck` Really on a technical level, they seem wholly independent. Only on a human level ("they are both are tests") do they have anything in common. I had messed up the logic in cross builds because of this. Now I split the flag in two (and cleaned up a few other inconsistencies), and the logic fixed itself. Co-Authored-By: Robert Hensing --- Makefile | 30 ++++++-- Makefile.config.in | 7 +- configure.ac | 43 ++++++++---- doc/internal-api/local.mk | 14 +--- doc/manual/local.mk | 6 +- doc/manual/src/contributing/hacking.md | 5 +- .../src/installation/prerequisites-source.md | 2 +- mk/disable-tests.mk | 12 ---- package.nix | 69 ++++++++----------- 9 files changed, 93 insertions(+), 95 deletions(-) delete mode 100644 mk/disable-tests.mk diff --git a/Makefile b/Makefile index 3dae8b394..c62216df8 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ makefiles = \ misc/upstart/local.mk endif -ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes) +ifeq ($(ENABLE_UNIT_TESTS), yes) makefiles += \ tests/unit/libutil/local.mk \ tests/unit/libutil-support/local.mk \ @@ -32,9 +32,14 @@ makefiles += \ tests/unit/libstore-support/local.mk \ tests/unit/libexpr/local.mk \ tests/unit/libexpr-support/local.mk +else +.PHONY: check +check: + @echo "Unit tests are disabled. Configure without '--disable-unit-tests', or avoid calling 'make check'." + @exit 1 endif -ifeq ($(ENABLE_TESTS), yes) +ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes) makefiles += \ tests/functional/local.mk \ tests/functional/ca/local.mk \ @@ -42,8 +47,10 @@ makefiles += \ tests/functional/test-libstoreconsumer/local.mk \ tests/functional/plugins/local.mk else -makefiles += \ - mk/disable-tests.mk +.PHONY: installcheck +installcheck: + @echo "Functional tests are disabled. Configure without '--disable-functional-tests', or avoid calling 'make installcheck'." + @exit 1 endif OPTIMIZE = 1 @@ -59,9 +66,22 @@ include mk/lib.mk # Must be included after `mk/lib.mk` so rules refer to variables defined # by the library. Rules are not "lazy" like variables, unfortunately. -ifeq ($(ENABLE_BUILD), yes) +ifeq ($(ENABLE_DOC_GEN),yes) $(eval $(call include-sub-makefile, doc/manual/local.mk)) +else +.PHONY: manual-html manpages +manual-html manpages: + @echo "Generated docs are disabled. Configure without '--disable-doc-gen', or avoid calling 'make manpages' and 'make manual-html'." + @exit 1 endif + +ifeq ($(ENABLE_INTERNAL_API_DOCS),yes) $(eval $(call include-sub-makefile, doc/internal-api/local.mk)) +else +.PHONY: internal-api-html +internal-api-html: + @echo "Internal API docs are disabled. Configure with '--enable-internal-api-docs', or avoid calling 'make internal-api-html'." + @exit 1 +endif GLOBAL_CXXFLAGS += -g -Wall -include $(buildprefix)config.h -std=c++2a -I src diff --git a/Makefile.config.in b/Makefile.config.in index c85e028c2..21a9f41ec 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -9,8 +9,11 @@ CXXFLAGS = @CXXFLAGS@ CXXLTO = @CXXLTO@ EDITLINE_LIBS = @EDITLINE_LIBS@ ENABLE_BUILD = @ENABLE_BUILD@ +ENABLE_DOC_GEN = @ENABLE_DOC_GEN@ +ENABLE_FUNCTIONAL_TESTS = @ENABLE_FUNCTIONAL_TESTS@ +ENABLE_INTERNAL_API_DOCS = @ENABLE_INTERNAL_API_DOCS@ ENABLE_S3 = @ENABLE_S3@ -ENABLE_TESTS = @ENABLE_TESTS@ +ENABLE_UNIT_TESTS = @ENABLE_UNIT_TESTS@ GTEST_LIBS = @GTEST_LIBS@ HAVE_LIBCPUID = @HAVE_LIBCPUID@ HAVE_SECCOMP = @HAVE_SECCOMP@ @@ -36,12 +39,10 @@ checkbindir = @checkbindir@ checklibdir = @checklibdir@ datadir = @datadir@ datarootdir = @datarootdir@ -doc_generate = @doc_generate@ docdir = @docdir@ embedded_sandbox_shell = @embedded_sandbox_shell@ exec_prefix = @exec_prefix@ includedir = @includedir@ -internal_api_docs = @internal_api_docs@ libdir = @libdir@ libexecdir = @libexecdir@ localstatedir = @localstatedir@ diff --git a/configure.ac b/configure.ac index a949f9df2..1bc4f17b0 100644 --- a/configure.ac +++ b/configure.ac @@ -138,20 +138,38 @@ AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]), ENABLE_BUILD=$enableval, ENABLE_BUILD=yes) AC_SUBST(ENABLE_BUILD) -# Building without tests is useful for bootstrapping with a smaller footprint +# Building without unit tests is useful for bootstrapping with a smaller footprint # or running the tests in a separate derivation. Otherwise, we do compile and # run them. -AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]), - ENABLE_TESTS=$enableval, ENABLE_TESTS=yes) -AC_SUBST(ENABLE_TESTS) -# Building without API docs is the default as Nix' C++ interfaces are internal and unstable. -AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]), - internal_api_docs=$enableval, internal_api_docs=no) -AC_SUBST(internal_api_docs) +AC_ARG_ENABLE(unit-tests, AS_HELP_STRING([--disable-unit-tests],[Do not build the tests]), + ENABLE_UNIT_TESTS=$enableval, ENABLE_UNIT_TESTS=$ENABLE_BUILD) +AC_SUBST(ENABLE_UNIT_TESTS) AS_IF( - [test "$ENABLE_BUILD" == "yes" || test "$ENABLE_TEST" == "yes"], + [test "$ENABLE_BUILD" == "no" && test "$ENABLE_UNIT_TESTS" == "yes"], + [AC_MSG_ERROR([Cannot enable unit tests when building overall is disabled. Please do not pass '--enable-unit-tests' or do not pass '--disable-build'.])]) + +AC_ARG_ENABLE(functional-tests, AS_HELP_STRING([--disable-functional-tests],[Do not build the tests]), + ENABLE_FUNCTIONAL_TESTS=$enableval, ENABLE_FUNCTIONAL_TESTS=yes) +AC_SUBST(ENABLE_FUNCTIONAL_TESTS) + +# documentation generation switch +AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation generation]), + ENABLE_DOC_GEN=$enableval, ENABLE_DOC_GEN=$ENABLE_BUILD) +AC_SUBST(ENABLE_DOC_GEN) + +AS_IF( + [test "$ENABLE_BUILD" == "no" && test "$ENABLE_GENERATED_DOCS" == "yes"], + [AC_MSG_ERROR([Cannot enable generated docs when building overall is disabled. Please do not pass '--enable-doc-gen' or do not pass '--disable-build'.])]) + +# Building without API docs is the default as Nix' C++ interfaces are internal and unstable. +AC_ARG_ENABLE(internal-api-docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]), + ENABLE_INTERNAL_API_DOCS=$enableval, ENABLE_INTERNAL_API_DOCS=no) +AC_SUBST(ENABLE_INTERNAL_API_DOCS) + +AS_IF( + [test "$ENABLE_FUNCTIONAL_TESTS" == "yes" || test "$ENABLE_DOC_GEN" == "yes"], [NEED_PROG(jq, jq)]) AS_IF([test "$ENABLE_BUILD" == "yes"],[ @@ -317,7 +335,7 @@ if test "$gc" = yes; then AC_DEFINE(HAVE_BOEHMGC, 1, [Whether to use the Boehm garbage collector.]) fi -AS_IF([test "$ENABLE_TESTS" == "yes"],[ +AS_IF([test "$ENABLE_UNIT_TESTS" == "yes"],[ # Look for gtest. PKG_CHECK_MODULES([GTEST], [gtest_main]) @@ -349,11 +367,6 @@ AC_LANG_POP(C++) # Look for nlohmann/json. PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9]) -# documentation generation switch -AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation generation]), - doc_generate=$enableval, doc_generate=yes) -AC_SUBST(doc_generate) - # Look for lowdown library. PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"]) diff --git a/doc/internal-api/local.mk b/doc/internal-api/local.mk index 890f341b7..bf2c4dede 100644 --- a/doc/internal-api/local.mk +++ b/doc/internal-api/local.mk @@ -1,19 +1,7 @@ -.PHONY: internal-api-html - -ifeq ($(internal_api_docs), yes) - $(docdir)/internal-api/html/index.html $(docdir)/internal-api/latex: $(d)/doxygen.cfg mkdir -p $(docdir)/internal-api { cat $< ; echo "OUTPUT_DIRECTORY=$(docdir)/internal-api" ; } | doxygen - # Generate the HTML API docs for Nix's unstable internal interfaces. +.PHONY: internal-api-html internal-api-html: $(docdir)/internal-api/html/index.html - -else - -# Make a nicer error message -internal-api-html: - @echo "Internal API docs are disabled. Configure with '--enable-internal-api-docs', or avoid calling 'make internal-api-html'." - @exit 1 - -endif diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 456000d3d..b77168885 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -1,5 +1,3 @@ -ifeq ($(doc_generate),yes) - # The version of Nix used to generate the doc. Can also be # `$(nix_INSTALL_PATH)` or just `nix` (to grap ambient from the `PATH`), # if one prefers. @@ -180,6 +178,8 @@ manual-html: $(docdir)/manual/index.html install: $(docdir)/manual/index.html # Generate 'nix' manpages. +.PHONY: manpages +manpages: $(mandir)/man1/nix3-manpages install: $(mandir)/man1/nix3-manpages man: doc/manual/generated/man1/nix3-manpages all: doc/manual/generated/man1/nix3-manpages @@ -225,5 +225,3 @@ $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/ @rm -rf $(DESTDIR)$(docdir)/manual @mv $(DESTDIR)$(docdir)/manual.tmp/html $(DESTDIR)$(docdir)/manual @rm -rf $(DESTDIR)$(docdir)/manual.tmp - -endif diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 421ac981c..9478c424d 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -67,9 +67,10 @@ You can also build Nix for one of the [supported platforms](#platforms). ## Makefile variables - `ENABLE_BUILD=yes` to enable building the C++ code. -- `ENABLE_TESTS=yes` to enable building the tests. +- `ENABLE_DOC_GEN=yes` to enable building the documentation (manual, man pages, etc.). +- `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests. +- `ENABLE_UNIT_TESTS=yes` to enable building the unit tests. - `OPTIMIZE=1` to enable optimizations. -- `doc_generate=yes` to enable building the documentation (manual, man pages, etc.). The docs can take a while to build, so you may want to disable this for local development. diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md index d4babf1ea..807e82517 100644 --- a/doc/manual/src/installation/prerequisites-source.md +++ b/doc/manual/src/installation/prerequisites-source.md @@ -72,7 +72,7 @@ This is an optional dependency and can be disabled by providing a `--disable-cpuid` to the `configure` script. - - Unless `./configure --disable-tests` is specified, GoogleTest (GTest) and + - Unless `./configure --disable-unit-tests` is specified, GoogleTest (GTest) and RapidCheck are required, which are available at and respectively. diff --git a/mk/disable-tests.mk b/mk/disable-tests.mk deleted file mode 100644 index f72f84412..000000000 --- a/mk/disable-tests.mk +++ /dev/null @@ -1,12 +0,0 @@ -# This file is only active for `./configure --disable-tests`. -# Running `make check` or `make installcheck` would indicate a mistake in the -# caller. - -installcheck: - @echo "Tests are disabled. Configure without '--disable-tests', or avoid calling 'make installcheck'." - @exit 1 - -# This currently has little effect. -check: - @echo "Tests are disabled. Configure without '--disable-tests', or avoid calling 'make check'." - @exit 1 diff --git a/package.nix b/package.nix index 24395b484..370820c40 100644 --- a/package.nix +++ b/package.nix @@ -104,30 +104,6 @@ let inherit doBuild doCheck doInstallCheck; }; - filesets = { - baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.; - - configureFiles = fileset.unions [ - ./.version - ./configure.ac - ./m4 - # TODO: do we really need README.md? It doesn't seem used in the build. - ./README.md - ]; - - topLevelBuildFiles = fileset.unions [ - ./local.mk - ./Makefile - ./Makefile.config.in - ./mk - ]; - - functionalTestFiles = fileset.unions [ - ./tests/functional - (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) - ]; - }; - mkDerivation = if withCoverageChecks then @@ -151,32 +127,44 @@ mkDerivation (finalAttrs: let # to be run later, requiresthe unit tests to be built. buildUnitTests = doCheck || installUnitTests; - anySortOfTesting = buildUnitTests || doInstallCheck; - in { inherit pname version; src = let - + baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.; in fileset.toSource { root = ./.; - fileset = fileset.intersect filesets.baseFiles (fileset.unions ([ - filesets.configureFiles - filesets.topLevelBuildFiles - ./doc/internal-api + fileset = fileset.intersect baseFiles (fileset.unions ([ + # For configure + ./.version + ./configure.ac + ./m4 + # TODO: do we really need README.md? It doesn't seem used in the build. + ./README.md + # For make, regardless of what we are building + ./local.mk + ./Makefile + ./Makefile.config.in + ./mk + (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) ] ++ lib.optionals doBuild [ ./boehmgc-coroutine-sp-fallback.diff ./doc ./misc ./precompiled-headers.h ./src - ./tests/unit ./COPYING ./scripts/local.mk - ] ++ lib.optionals anySortOfTesting [ - filesets.functionalTestFiles + ] ++ lib.optionals buildUnitTests [ + ./doc/manual + ] ++ lib.optionals enableInternalAPIDocs [ + ./doc/internal-api + ] ++ lib.optionals buildUnitTests [ + ./tests/unit + ] ++ lib.optionals doInstallCheck [ + ./tests/functional ])); }; @@ -277,7 +265,8 @@ in { configureFlags = [ "--sysconfdir=/etc" (lib.enableFeature doBuild "build") - (lib.enableFeature anySortOfTesting "tests") + (lib.enableFeature buildUnitTests "unit-tests") + (lib.enableFeature doInstallCheck "functional-tests") (lib.enableFeature enableInternalAPIDocs "internal-api-docs") (lib.enableFeature enableManual "doc-gen") (lib.enableFeature installUnitTests "install-unit-tests") @@ -310,10 +299,7 @@ in { ''; postInstall = lib.optionalString doBuild ( - '' - mkdir -p $doc/nix-support - echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products - '' + lib.optionalString stdenv.hostPlatform.isStatic '' + lib.optionalString stdenv.hostPlatform.isStatic '' mkdir -p $out/nix-support echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products '' + lib.optionalString stdenv.isDarwin '' @@ -322,7 +308,10 @@ in { $out/lib/libboost_context.dylib \ $out/lib/libnixutil.dylib '' - ) + lib.optionalString enableInternalAPIDocs '' + ) + lib.optionalString enableManual '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc manual ''${!outputDoc}/share/doc/nix/manual" >> ''${!outputDoc}/nix-support/hydra-build-products + '' + lib.optionalString enableInternalAPIDocs '' mkdir -p ''${!outputDoc}/nix-support echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products ''; From 123ef6a9967d5ca8ed4052d84128ff0e98950532 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Mon, 18 Dec 2023 10:19:25 -0800 Subject: [PATCH 838/909] Fix warnings when running checks `nix flake check` had these warnings: trace: warning: Module argument `nodes.client.config` is deprecated. Use `nodes.client` instead. trace: warning: Module argument `nodes.client.config` is deprecated. Use `nodes.client` instead. trace: warning: The option `services.openssh.permitRootLogin' defined in `/nix/store/3m3hfpmbjdf4w39qfjami7ljhvhczay1-source/tests/nixos/nix-copy.nix' has been renamed to `services.openssh.settings.PermitRootLogin'. trace: warning: Module argument `nodes.http_dns.config` is deprecated. Use `nodes.http_dns` instead. trace: warning: Module argument `nodes.github.config` is deprecated. Use `nodes.github` instead. trace: warning: Module argument `nodes.sourcehut.config` is deprecated. Use `nodes.sourcehut` instead. --- tests/nixos/github-flakes.nix | 2 +- tests/nixos/nix-copy.nix | 2 +- tests/nixos/nss-preload.nix | 4 ++-- tests/nixos/remote-builds-ssh-ng.nix | 2 +- tests/nixos/remote-builds.nix | 8 ++++---- tests/nixos/sourcehut-flakes.nix | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index 62ae8871b..a51689445 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -144,7 +144,7 @@ in virtualisation.memorySize = 4096; nix.settings.substituters = lib.mkForce [ ]; nix.extraOptions = "experimental-features = nix-command flakes"; - networking.hosts.${(builtins.head nodes.github.config.networking.interfaces.eth1.ipv4.addresses).address} = + networking.hosts.${(builtins.head nodes.github.networking.interfaces.eth1.ipv4.addresses).address} = [ "channels.nixos.org" "api.github.com" "github.com" ]; security.pki.certificateFiles = [ "${cert}/ca.crt" ]; }; diff --git a/tests/nixos/nix-copy.nix b/tests/nixos/nix-copy.nix index 2981cc2b8..7db5197aa 100644 --- a/tests/nixos/nix-copy.nix +++ b/tests/nixos/nix-copy.nix @@ -36,7 +36,7 @@ in { server = { config, pkgs, ... }: { services.openssh.enable = true; - services.openssh.permitRootLogin = "yes"; + services.openssh.settings.PermitRootLogin = "yes"; users.users.root.password = "foobar"; virtualisation.writableStore = true; virtualisation.additionalPaths = [ pkgB pkgC ]; diff --git a/tests/nixos/nss-preload.nix b/tests/nixos/nss-preload.nix index cef62e95b..00505d114 100644 --- a/tests/nixos/nss-preload.nix +++ b/tests/nixos/nss-preload.nix @@ -84,8 +84,8 @@ in client = { lib, nodes, pkgs, ... }: { networking.useDHCP = false; networking.nameservers = [ - (lib.head nodes.http_dns.config.networking.interfaces.eth1.ipv6.addresses).address - (lib.head nodes.http_dns.config.networking.interfaces.eth1.ipv4.addresses).address + (lib.head nodes.http_dns.networking.interfaces.eth1.ipv6.addresses).address + (lib.head nodes.http_dns.networking.interfaces.eth1.ipv4.addresses).address ]; networking.interfaces.eth1.ipv6.addresses = [ { address = "fd21::10"; prefixLength = 64; } diff --git a/tests/nixos/remote-builds-ssh-ng.nix b/tests/nixos/remote-builds-ssh-ng.nix index b59dde9bf..20a43803d 100644 --- a/tests/nixos/remote-builds-ssh-ng.nix +++ b/tests/nixos/remote-builds-ssh-ng.nix @@ -81,7 +81,7 @@ in client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") # Perform a build - out = client.succeed("nix-build ${expr nodes.client.config 1} 2> build-output") + out = client.succeed("nix-build ${expr nodes.client 1} 2> build-output") # Verify that the build was done on the builder builder.succeed(f"test -e {out.strip()}") diff --git a/tests/nixos/remote-builds.nix b/tests/nixos/remote-builds.nix index 1c96cc787..ad7f509db 100644 --- a/tests/nixos/remote-builds.nix +++ b/tests/nixos/remote-builds.nix @@ -90,22 +90,22 @@ in # Perform a build and check that it was performed on the builder. out = client.succeed( - "nix-build ${expr nodes.client.config 1} 2> build-output", + "nix-build ${expr nodes.client 1} 2> build-output", "grep -q Hello build-output" ) builder1.succeed(f"test -e {out}") # And a parallel build. - paths = client.succeed(r'nix-store -r $(nix-instantiate ${expr nodes.client.config 2})\!out $(nix-instantiate ${expr nodes.client.config 3})\!out') + paths = client.succeed(r'nix-store -r $(nix-instantiate ${expr nodes.client 2})\!out $(nix-instantiate ${expr nodes.client 3})\!out') out1, out2 = paths.split() builder1.succeed(f"test -e {out1} -o -e {out2}") builder2.succeed(f"test -e {out1} -o -e {out2}") # And a failing build. - client.fail("nix-build ${expr nodes.client.config 5}") + client.fail("nix-build ${expr nodes.client 5}") # Test whether the build hook automatically skips unavailable builders. builder1.block() - client.succeed("nix-build ${expr nodes.client.config 4}") + client.succeed("nix-build ${expr nodes.client 4}") ''; } diff --git a/tests/nixos/sourcehut-flakes.nix b/tests/nixos/sourcehut-flakes.nix index 6e8d884a0..04f3590e1 100644 --- a/tests/nixos/sourcehut-flakes.nix +++ b/tests/nixos/sourcehut-flakes.nix @@ -108,7 +108,7 @@ in flake-registry = https://git.sr.ht/~NixOS/flake-registry/blob/master/flake-registry.json ''; environment.systemPackages = [ pkgs.jq ]; - networking.hosts.${(builtins.head nodes.sourcehut.config.networking.interfaces.eth1.ipv4.addresses).address} = + networking.hosts.${(builtins.head nodes.sourcehut.networking.interfaces.eth1.ipv4.addresses).address} = [ "git.sr.ht" ]; security.pki.certificateFiles = [ "${cert}/ca.crt" ]; }; From ba0087316acc2aba999cabe5e1a159da636b2569 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Mon, 18 Dec 2023 12:59:58 -0800 Subject: [PATCH 839/909] package: don't set sysconfdir in devShells --- flake.nix | 2 +- package.nix | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 8c4436729..a8fc105e8 100644 --- a/flake.nix +++ b/flake.nix @@ -395,7 +395,7 @@ stdenvs))); devShells = let - makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; }).overrideAttrs (attrs: { + makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: { installFlags = "sysconfdir=$(out)/etc"; shellHook = '' PATH=$prefix/bin:$PATH diff --git a/package.nix b/package.nix index 370820c40..b5ff45083 100644 --- a/package.nix +++ b/package.nix @@ -87,6 +87,9 @@ , test-daemon ? null , test-client ? null +# Avoid setting things that would interfere with a functioning devShell +, forDevShell ? false + # Not a real argument, just the only way to approximate let-binding some # stuff for argument defaults. , __forDefaults ? { @@ -263,13 +266,14 @@ in { ); configureFlags = [ - "--sysconfdir=/etc" (lib.enableFeature doBuild "build") (lib.enableFeature buildUnitTests "unit-tests") (lib.enableFeature doInstallCheck "functional-tests") (lib.enableFeature enableInternalAPIDocs "internal-api-docs") (lib.enableFeature enableManual "doc-gen") (lib.enableFeature installUnitTests "install-unit-tests") + ] ++ lib.optionals (!forDevShell) [ + "--sysconfdir=/etc" ] ++ lib.optionals installUnitTests [ "--with-check-bin-dir=${builtins.placeholder "check"}/bin" "--with-check-lib-dir=${builtins.placeholder "check"}/lib" From 6f4930382bb61f0a9b2a9e5b0080977a4dd03866 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Mon, 18 Dec 2023 13:34:08 -0800 Subject: [PATCH 840/909] Document more `Makefile` variables --- doc/manual/src/contributing/hacking.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 9478c424d..dce0422dc 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -66,13 +66,24 @@ You can also build Nix for one of the [supported platforms](#platforms). ## Makefile variables +You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run +`make install`. + +You may want to set `MAKEFLAGS="-e -j $NIX_BUILD_CORES"` to allow environment +variables to override `Makefile` variables. + - `ENABLE_BUILD=yes` to enable building the C++ code. - `ENABLE_DOC_GEN=yes` to enable building the documentation (manual, man pages, etc.). + + The docs can take a while to build, so you may want to disable this for local development. - `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests. - `ENABLE_UNIT_TESTS=yes` to enable building the unit tests. - `OPTIMIZE=1` to enable optimizations. - - The docs can take a while to build, so you may want to disable this for local development. +- `libraries=libutil programs=` to only build a specific library (this will + fail in the linking phase if you don't have the other libraries built, but is + useful for checking types). +- `libraries= programs=nix` to only build a specific program (this will not, in + general, work, because the programs need the libraries). ## Building Nix From 0cee56db1ace13a1f4b856c800950b2fb04df993 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Mon, 18 Dec 2023 14:04:25 -0800 Subject: [PATCH 841/909] Fix `logging.sh` test on macOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On macOS in the `nix develop` shell, `make tests/functional/logging.sh.test` errors: ++(logging.sh:18) mktemp +(logging.sh:18) builder=/var/folders/z5/fclwwdms3r1gq4k4p3pkvvc00000gn/T/tmp.StuabKUhMh +(logging.sh:19) echo -e '#!/bin/sh\nmkdir $out' +++(logging.sh:22) mktemp -d ++(logging.sh:22) nix-build -E 'with import ./config.nix; mkDerivation { name = "fnord"; builder = /var/folders/z5/fclwwdms3r1gq4k4p3pkvvc00000gn/T/tmp.StuabKUhMh; }' --out-link /var/folders/z5/fclwwdms3r1gq4k4p3pkvvc00000gn/T/tmp.oaKcy0NXqC/result error: … while calling the 'derivationStrict' builtin at :9:12: 8| 9| strict = derivationStrict drvAttrs; | ^ 10| … while evaluating derivation 'fnord' whose name attribute is located at «string»:1:42 … while evaluating attribute 'args' of derivation 'fnord' at /Users/wiggles/nix/tests/functional/config.nix:23:7: 22| builder = shell; 23| args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' | ^ 24| if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; error: path '/var' is a symlink +(logging.sh:22) outp= ++(logging.sh:22) onError ++(/Users/wiggles/nix/tests/functional/common/vars-and-functions.sh:237) set +x logging.sh: test failed at: main in logging.sh:22 This is because `mktemp` returns a path like `/var/folders/z5/fclwwdms3r1gq4k4p3pkvvc00000gn/T/tmp.qDY24l6bIM`, where `/var` is a symlink to `/private/var`. Then, we attempt to use that path as a `builder`, which errors because symlinks are impure or whatever. Anyways, we can fix this by using `realpath "$(mktemp)"` instead of `mktemp` directly. NB: This error doesn't seem to happen when I run the tests through `nix flake check`. I'm not sure if Nix does something to `TMP` in that case. --- tests/functional/logging.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/logging.sh b/tests/functional/logging.sh index 1481b9b36..1ccc21d0b 100644 --- a/tests/functional/logging.sh +++ b/tests/functional/logging.sh @@ -15,7 +15,7 @@ nix-build dependencies.nix --no-out-link --compress-build-log [ "$(nix-store -l $path)" = FOO ] # test whether empty logs work fine with `nix log`. -builder="$(mktemp)" +builder="$(realpath "$(mktemp)")" echo -e "#!/bin/sh\nmkdir \$out" > "$builder" outp="$(nix-build -E \ 'with import ./config.nix; mkDerivation { name = "fnord"; builder = '"$builder"'; }' \ From 23fb19cb18709ed097274d427a1024ae08789ed3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 22:36:31 +0000 Subject: [PATCH 842/909] build(deps): bump zeebe-io/backport-action from 2.2.0 to 2.3.0 Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 2.2.0 to 2.3.0. - [Release notes](https://github.com/zeebe-io/backport-action/releases) - [Commits](https://github.com/zeebe-io/backport-action/compare/v2.2.0...v2.3.0) --- updated-dependencies: - dependency-name: zeebe-io/backport-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 85ddcfad3..f003114ba 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v2.2.0 + uses: zeebe-io/backport-action@v2.3.0 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} From fa4bbe53e837a138c382468601cd769736f7d1dc Mon Sep 17 00:00:00 2001 From: Mel Zuser Date: Mon, 18 Dec 2023 15:02:26 -0800 Subject: [PATCH 843/909] installer: allow overriding of NIX_FIRST_BUILD_ID on darwin because there are often already users in the 300 range and it's painful to work around. revives #6466 --- scripts/install-darwin-multi-user.sh | 6 ++++-- scripts/install-multi-user.sh | 19 +++++++++++++++---- scripts/install-systemd-multi-user.sh | 4 ++++ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 0326d3415..766f81bde 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -3,11 +3,13 @@ set -eu set -o pipefail +# System specific settings +export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-301}" +export NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d" + readonly NIX_DAEMON_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist # create by default; set 0 to DIY, use a symlink, etc. readonly NIX_VOLUME_CREATE=${NIX_VOLUME_CREATE:-1} # now default -NIX_FIRST_BUILD_UID="301" -NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d" # caution: may update times on / if not run as normal non-root user read_only_root() { diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index a08f62333..ad3ee8881 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -25,9 +25,9 @@ readonly RED='\033[31m' readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32} readonly NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-30000}" readonly NIX_BUILD_GROUP_NAME="nixbld" -# darwin installer needs to override these -NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}" -NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d" +# each system specific installer must set these: +# NIX_FIRST_BUILD_UID +# NIX_BUILD_USER_NAME_TEMPLATE # Please don't change this. We don't support it, because the # default shell profile that comes with Nix doesn't support it. readonly NIX_ROOT="/nix" @@ -707,6 +707,12 @@ EOF fi } +check_required_system_specific_settings() { + if [ -z "${NIX_FIRST_BUILD_UID+x}" ] || [ -z "${NIX_BUILD_USER_NAME_TEMPLATE+x}" ]; then + failure "Internal error: System specific installer for $(uname) ($1) does not export required settings." + fi +} + welcome_to_nix() { local -r NIX_UID_RANGES="${NIX_FIRST_BUILD_UID}..$((NIX_FIRST_BUILD_UID + NIX_USER_COUNT - 1))" local -r RANGE_TEXT=$(echo -ne "${BLUE}(uids [${NIX_UID_RANGES}])${ESC}") @@ -726,7 +732,9 @@ manager. This will happen in a few stages: if you are ready to continue. 3. Create the system users ${RANGE_TEXT} and groups ${GROUP_TEXT} - that the Nix daemon uses to run builds. + that the Nix daemon uses to run builds. To create system users + in a different range, exit and run this tool again with + NIX_FIRST_BUILD_UID set. 4. Perform the basic installation of the Nix files daemon. @@ -968,13 +976,16 @@ main() { if is_os_darwin; then # shellcheck source=./install-darwin-multi-user.sh . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh" + check_required_system_specific_settings "install-darwin-multi-user.sh" elif is_os_linux; then # shellcheck source=./install-systemd-multi-user.sh . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" # most of this works on non-systemd distros also + check_required_system_specific_settings "install-systemd-multi-user.sh" else failure "Sorry, I don't know what to do on $(uname)" fi + welcome_to_nix if ! is_root; then diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 07b34033a..202a9bb54 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -3,6 +3,10 @@ set -eu set -o pipefail +# System specific settings +export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}" +export NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d" + readonly SERVICE_SRC=/lib/systemd/system/nix-daemon.service readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service From 7526b7ded6d5884cefcd4c71e0a33962d883ae78 Mon Sep 17 00:00:00 2001 From: Andrew Marshall Date: Mon, 18 Dec 2023 19:33:20 -0500 Subject: [PATCH 844/909] Allow access to /dev/stderr in Darwin sandbox We allow /dev/stdout, so why not this? Since it is process-local, anyway, should not be possible to escape sandbox using it. --- src/libstore/build/sandbox-defaults.sb | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/build/sandbox-defaults.sb b/src/libstore/build/sandbox-defaults.sb index 77f013aea..25ec11285 100644 --- a/src/libstore/build/sandbox-defaults.sb +++ b/src/libstore/build/sandbox-defaults.sb @@ -68,6 +68,7 @@ R""( (allow file* (literal "/dev/null") (literal "/dev/random") + (literal "/dev/stderr") (literal "/dev/stdin") (literal "/dev/stdout") (literal "/dev/tty") From 0218e4e6c386e4c432520506568420c3cc384e47 Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 10 Dec 2023 04:15:51 +0100 Subject: [PATCH 845/909] memset less in addToStoreFromDump MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resizing a std::string clears the newly added bytes, which is not necessary here and comes with a ~1.4% slowdown on our test nixos config. 〉 nix eval --raw --impure --expr 'with import {}; system' before: Time (mean ± σ): 4.486 s ± 0.003 s [User: 3.978 s, System: 0.507 s] Range (min … max): 4.482 s … 4.492 s 10 runs after: Time (mean ± σ): 4.429 s ± 0.002 s [User: 3.929 s, System: 0.500 s] Range (min … max): 4.427 s … 4.433 s 10 runs --- src/libstore/local-store.cc | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7e82bae28..d903bb061 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -18,6 +18,8 @@ #include #include +#include +#include #include #include #include @@ -1130,7 +1132,11 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name path. */ bool inMemory = false; - std::string dump; + struct Free { + void operator()(void* v) { free(v); } + }; + std::unique_ptr dumpBuffer(nullptr); + std::string_view dump; /* Fill out buffer, and decide whether we are working strictly in memory based on whether we break out because the buffer is full @@ -1139,13 +1145,18 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto oldSize = dump.size(); constexpr size_t chunkSize = 65536; auto want = std::min(chunkSize, settings.narBufferSize - oldSize); - dump.resize(oldSize + want); + if (auto tmp = realloc(dumpBuffer.get(), oldSize + want)) { + dumpBuffer.release(); + dumpBuffer.reset((char*) tmp); + } else { + throw std::bad_alloc(); + } auto got = 0; Finally cleanup([&]() { - dump.resize(oldSize + got); + dump = {dumpBuffer.get(), dump.size() + got}; }); try { - got = source.read(dump.data() + oldSize, want); + got = source.read(dumpBuffer.get() + oldSize, want); } catch (EndOfFile &) { inMemory = true; break; @@ -1171,7 +1182,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name else writeFile(tempPath, bothSource); - dump.clear(); + dumpBuffer.reset(); + dump = {}; } auto [hash, size] = hashSink->finish(); From 78353deb028fcc700776db9d92dcae45d68fb85f Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 10 Dec 2023 08:24:45 +0100 Subject: [PATCH 846/909] encode black holes as tApp values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit checking for isBlackhole in the forceValue hot path is rather more expensive than necessary, and with a little bit of trickery we can move such handling into the isApp case. small performance benefit, but under some circumstances we've seen 2% improvement as well. 〉 nix eval --raw --impure --expr 'with import {}; system' before: Time (mean ± σ): 4.429 s ± 0.002 s [User: 3.929 s, System: 0.500 s] Range (min … max): 4.427 s … 4.433 s 10 runs after: Time (mean ± σ): 4.396 s ± 0.002 s [User: 3.894 s, System: 0.501 s] Range (min … max): 4.393 s … 4.399 s 10 runs --- src/libexpr/eval-inline.hh | 13 +++++++---- src/libexpr/eval.cc | 44 +++++++++++++++++++++----------------- src/libexpr/eval.hh | 8 +++++++ src/libexpr/nixexpr.hh | 7 ++++++ src/libexpr/primops.cc | 23 ++++++++++++++++++++ src/libexpr/primops.hh | 6 ++++++ src/libexpr/value.hh | 24 ++++++++++++++------- 7 files changed, 93 insertions(+), 32 deletions(-) diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index c37b1d62b..9d08f1938 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -104,11 +104,16 @@ void EvalState::forceValue(Value & v, Callable getPos) } } else if (v.isApp()) { - PosIdx pos = getPos(); - callFunction(*v.app.left, *v.app.right, v, pos); + try { + callFunction(*v.app.left, *v.app.right, v, noPos); + } catch (InfiniteRecursionError & e) { + // only one black hole can *throw* in any given eval stack so we need not + // check whether the position is set already. + if (v.isBlackhole()) + e.err.errPos = positions[getPos()]; + throw; + } } - else if (v.isBlackhole()) - error("infinite recursion encountered").atPos(getPos()).template debugThrow(); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 9e494148e..71c151f96 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -162,7 +162,17 @@ void Value::print(const SymbolTable &symbols, std::ostream &str, break; case tThunk: case tApp: - str << ""; + if (!isBlackhole()) { + str << ""; + } else { + // Although we know for sure that it's going to be an infinite recursion + // when this value is accessed _in the current context_, it's likely + // that the user will misinterpret a simpler «infinite recursion» output + // as a definitive statement about the value, while in fact it may be + // a valid value after `builtins.trace` and perhaps some other steps + // have completed. + str << "«potential infinite recursion»"; + } break; case tLambda: str << ""; @@ -179,15 +189,6 @@ void Value::print(const SymbolTable &symbols, std::ostream &str, case tFloat: str << fpoint; break; - case tBlackhole: - // Although we know for sure that it's going to be an infinite recursion - // when this value is accessed _in the current context_, it's likely - // that the user will misinterpret a simpler «infinite recursion» output - // as a definitive statement about the value, while in fact it may be - // a valid value after `builtins.trace` and perhaps some other steps - // have completed. - str << "«potential infinite recursion»"; - break; default: printError("Nix evaluator internal error: Value::print(): invalid value type %1%", internalType); abort(); @@ -256,8 +257,7 @@ std::string showType(const Value & v) return fmt("the partially applied built-in function '%s'", std::string(getPrimOp(v)->primOp->name)); case tExternal: return v.external->showType(); case tThunk: return "a thunk"; - case tApp: return "a function application"; - case tBlackhole: return "a black hole"; + case tApp: return v.isBlackhole() ? "a black hole" : "a function application"; default: return std::string(showType(v.type())); } @@ -1621,15 +1621,17 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & return; } else { /* We have all the arguments, so call the primop. */ - auto name = vCur.primOp->name; + auto * fn = vCur.primOp; nrPrimOpCalls++; - if (countCalls) primOpCalls[name]++; + // This will count black holes, but that's ok, because unrecoverable errors are rare. + if (countCalls) primOpCalls[fn->name]++; try { - vCur.primOp->fun(*this, vCur.determinePos(noPos), args, vCur); + fn->fun(*this, vCur.determinePos(noPos), args, vCur); } catch (Error & e) { - addErrorTrace(e, pos, "while calling the '%1%' builtin", name); + if (!fn->hideInDiagnostics) + addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); throw; } @@ -1666,18 +1668,20 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & for (size_t i = 0; i < argsLeft; ++i) vArgs[argsDone + i] = args[i]; - auto name = primOp->primOp->name; + auto fn = primOp->primOp; nrPrimOpCalls++; - if (countCalls) primOpCalls[name]++; + // This will count black holes, but that's ok, because unrecoverable errors are rare. + if (countCalls) primOpCalls[fn->name]++; try { // TODO: // 1. Unify this and above code. Heavily redundant. // 2. Create a fake env (arg1, arg2, etc.) and a fake expr (arg1: arg2: etc: builtins.name arg1 arg2 etc) // so the debugger allows to inspect the wrong parameters passed to the builtin. - primOp->primOp->fun(*this, vCur.determinePos(noPos), vArgs, vCur); + fn->fun(*this, vCur.determinePos(noPos), vArgs, vCur); } catch (Error & e) { - addErrorTrace(e, pos, "while calling the '%1%' builtin", name); + if (!fn->hideInDiagnostics) + addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); throw; } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index f3f6d35b9..e5e401ab6 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -77,6 +77,14 @@ struct PrimOp */ std::optional experimentalFeature; + /** + * Whether to hide this primop in diagnostics. + * + * Used to hide the fact that black holes are primop applications from + * stack traces. + */ + bool hideInDiagnostics; + /** * Validity check to be performed by functions that introduce primops, * such as RegisterPrimOp() and Value::mkPrimOp(). diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 020286815..cf6fd1a8d 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -21,6 +21,13 @@ MakeError(TypeError, EvalError); MakeError(UndefinedVarError, Error); MakeError(MissingArgumentError, EvalError); +class InfiniteRecursionError : public EvalError +{ + friend class EvalState; +public: + using EvalError::EvalError; +}; + /** * Position objects. */ diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 89d5492da..d46eccd34 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4263,6 +4263,29 @@ static RegisterPrimOp primop_splitVersion({ }); +static void prim_blackHoleFn(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + state.error("infinite recursion encountered") + .debugThrow(); +} + +static PrimOp primop_blackHole = { + .name = "«blackHole»", + .args = {}, + .fun = prim_blackHoleFn, + .hideInDiagnostics = true, +}; + +static Value makeBlackHole() +{ + Value v; + v.mkPrimOp(&primop_blackHole); + return v; +} + +Value prim_blackHole = makeBlackHole(); + + /************************************************************* * Primop registration *************************************************************/ diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 45486608f..244eada86 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -51,4 +51,10 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Valu */ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v); +/** + * Placeholder value for black holes, used to represent black holes as + * applications of this value to the evaluated thunks. + */ +extern Value prim_blackHole; + } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 30b3d4934..52cd0f901 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -32,7 +32,6 @@ typedef enum { tThunk, tApp, tLambda, - tBlackhole, tPrimOp, tPrimOpApp, tExternal, @@ -151,7 +150,7 @@ public: // type() == nThunk inline bool isThunk() const { return internalType == tThunk; }; inline bool isApp() const { return internalType == tApp; }; - inline bool isBlackhole() const { return internalType == tBlackhole; }; + inline bool isBlackhole() const; // type() == nFunction inline bool isLambda() const { return internalType == tLambda; }; @@ -248,7 +247,7 @@ public: case tLambda: case tPrimOp: case tPrimOpApp: return nFunction; case tExternal: return nExternal; case tFloat: return nFloat; - case tThunk: case tApp: case tBlackhole: return nThunk; + case tThunk: case tApp: return nThunk; } if (invalidIsThunk) return nThunk; @@ -356,11 +355,7 @@ public: lambda.fun = f; } - inline void mkBlackhole() - { - internalType = tBlackhole; - // Value will be overridden anyways - } + inline void mkBlackhole(); void mkPrimOp(PrimOp * p); @@ -447,6 +442,19 @@ public: }; +extern Value prim_blackHole; + +inline bool Value::isBlackhole() const +{ + return internalType == tApp && app.left == &prim_blackHole; +} + +inline void Value::mkBlackhole() +{ + mkApp(&prim_blackHole, &prim_blackHole); +} + + #if HAVE_BOEHMGC typedef std::vector> ValueVector; typedef std::map, traceable_allocator>> ValueMap; From 74c134914c747b1df6385cab5d2298f66a87b61f Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 10 Dec 2023 09:25:20 +0100 Subject: [PATCH 847/909] compare string values with strcmp string_view()ification calls strlen() first, which we don't need here. --- src/libexpr/eval.cc | 2 +- src/libexpr/primops.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 71c151f96..8e89ddcf1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2436,7 +2436,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v return v1.boolean == v2.boolean; case nString: - return v1.string_view().compare(v2.string_view()) == 0; + return strcmp(v1.c_str(), v2.c_str()) == 0; case nPath: return diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d46eccd34..b7e903667 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -586,7 +586,7 @@ struct CompareValues case nFloat: return v1->fpoint < v2->fpoint; case nString: - return v1->string_view().compare(v2->string_view()) < 0; + return strcmp(v1->c_str(), v2->c_str()) < 0; case nPath: // Note: we don't take the accessor into account // since it's not obvious how to compare them in a @@ -2401,7 +2401,7 @@ static void prim_attrNames(EvalState & state, const PosIdx pos, Value * * args, (v.listElems()[n++] = state.allocValue())->mkString(state.symbols[i.name]); std::sort(v.listElems(), v.listElems() + n, - [](Value * v1, Value * v2) { return v1->string_view().compare(v2->string_view()) < 0; }); + [](Value * v1, Value * v2) { return strcmp(v1->c_str(), v2->c_str()) < 0; }); } static RegisterPrimOp primop_attrNames({ From cc4038d54177c944340607c7d141680e66ff92a7 Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 10 Dec 2023 09:49:38 +0100 Subject: [PATCH 848/909] use std::tie() for macro-generated operators MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit as written the comparisons generate copies, even though it looks as though they shouldn't. before: Time (mean ± σ): 4.396 s ± 0.002 s [User: 3.894 s, System: 0.501 s] Range (min … max): 4.393 s … 4.399 s 10 runs after: Time (mean ± σ): 4.260 s ± 0.003 s [User: 3.754 s, System: 0.505 s] Range (min … max): 4.257 s … 4.266 s 10 runs --- src/libcmd/built-path.cc | 4 ++-- src/libstore/derived-path.cc | 8 ++------ src/libutil/comparator.hh | 4 ++-- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index 8e2efc7c3..c5eb93c5d 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -12,9 +12,9 @@ namespace nix { bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ { \ const MY_TYPE* me = this; \ - auto fields1 = std::make_tuple(*me->drvPath, me->FIELD); \ + auto fields1 = std::tie(*me->drvPath, me->FIELD); \ me = &other; \ - auto fields2 = std::make_tuple(*me->drvPath, me->FIELD); \ + auto fields2 = std::tie(*me->drvPath, me->FIELD); \ return fields1 COMPARATOR fields2; \ } #define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 3105dbc93..a7b404321 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -12,9 +12,9 @@ namespace nix { bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ { \ const MY_TYPE* me = this; \ - auto fields1 = std::make_tuple(*me->drvPath, me->FIELD); \ + auto fields1 = std::tie(*me->drvPath, me->FIELD); \ me = &other; \ - auto fields2 = std::make_tuple(*me->drvPath, me->FIELD); \ + auto fields2 = std::tie(*me->drvPath, me->FIELD); \ return fields1 COMPARATOR fields2; \ } #define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ @@ -22,13 +22,9 @@ namespace nix { CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) -#define FIELD_TYPE std::string CMP(SingleDerivedPath, SingleDerivedPathBuilt, output) -#undef FIELD_TYPE -#define FIELD_TYPE OutputsSpec CMP(SingleDerivedPath, DerivedPathBuilt, outputs) -#undef FIELD_TYPE #undef CMP #undef CMP_ONE diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index a4d20a675..cbc2bb4fd 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -13,9 +13,9 @@ #define GENERATE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE, ...) \ PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const { \ __VA_OPT__(const MY_TYPE * me = this;) \ - auto fields1 = std::make_tuple( __VA_ARGS__ ); \ + auto fields1 = std::tie( __VA_ARGS__ ); \ __VA_OPT__(me = &other;) \ - auto fields2 = std::make_tuple( __VA_ARGS__ ); \ + auto fields2 = std::tie( __VA_ARGS__ ); \ return fields1 COMPARATOR fields2; \ } #define GENERATE_EQUAL(prefix, qualification, my_type, args...) \ From 2e0321912a9efa352160eb1e57e6b7b88e517d0d Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 10 Dec 2023 12:59:51 +0100 Subject: [PATCH 849/909] use aligned flex tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ~2% speedup on parsing without eval, less (but still significant) on system eval. having flex generate faster parsers leads to very strange misparses. maybe re2c is worth investigating. before: Time (mean ± σ): 4.260 s ± 0.003 s [User: 3.754 s, System: 0.505 s] Range (min … max): 4.257 s … 4.266 s 10 runs after: Time (mean ± σ): 4.231 s ± 0.004 s [User: 3.725 s, System: 0.504 s] Range (min … max): 4.226 s … 4.240 s 10 runs --- src/libexpr/lexer.l | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index a3a8608d9..9a35dd594 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -1,4 +1,5 @@ %option reentrant bison-bridge bison-locations +%option align %option noyywrap %option never-interactive %option stack From b78e77b34c14b0f127b22e252309527e84967dcc Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 10 Dec 2023 13:00:18 +0100 Subject: [PATCH 850/909] use custom location type in the parser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ~1% parser speedup from not using TLS indirections, less on system eval. this could have also gone in flex yyextra data, but that's significantly slower for some reason (albeit still faster than thread locals). before: Time (mean ± σ): 4.231 s ± 0.004 s [User: 3.725 s, System: 0.504 s] Range (min … max): 4.226 s … 4.240 s 10 runs after: Time (mean ± σ): 4.224 s ± 0.005 s [User: 3.711 s, System: 0.512 s] Range (min … max): 4.218 s … 4.234 s 10 runs --- src/libexpr/lexer.l | 9 +++------ src/libexpr/parser.y | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 9a35dd594..df2cbd06f 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -36,9 +36,6 @@ static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data) #define CUR_POS makeCurPos(*yylloc, data) -// backup to recover from yyless(0) -thread_local YYLTYPE prev_yylloc; - static void initLoc(YYLTYPE * loc) { loc->first_line = loc->last_line = 1; @@ -47,7 +44,7 @@ static void initLoc(YYLTYPE * loc) static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) { - prev_yylloc = *loc; + loc->stash(); loc->first_line = loc->last_line; loc->first_column = loc->last_column; @@ -231,7 +228,7 @@ or { return OR_KW; } {HPATH_START}\$\{ { PUSH_STATE(PATH_START); yyless(0); - *yylloc = prev_yylloc; + yylloc->unstash(); } {PATH_SEG} { @@ -287,7 +284,7 @@ or { return OR_KW; } context (it may be ')', ';', or something of that sort) */ POP_STATE(); yyless(0); - *yylloc = prev_yylloc; + yylloc->unstash(); return PATH_END; } diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 16ad8af2e..b331776f0 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -28,6 +28,31 @@ namespace nix { +#define YYLTYPE ::nix::ParserLocation + struct ParserLocation + { + int first_line, first_column; + int last_line, last_column; + + // backup to recover from yyless(0) + int stashed_first_line, stashed_first_column; + int stashed_last_line, stashed_last_column; + + void stash() { + stashed_first_line = first_line; + stashed_first_column = first_column; + stashed_last_line = last_line; + stashed_last_column = last_column; + } + + void unstash() { + first_line = stashed_first_line; + first_column = stashed_first_column; + last_line = stashed_last_line; + last_column = stashed_last_column; + } + }; + struct ParseData { EvalState & state; From f9aee2f2c41652b3b76d16a874fdded4e6d28d92 Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 10 Dec 2023 10:34:55 +0100 Subject: [PATCH 851/909] don't malloc/memset posix accessor buffer it's relatively small and fits on the stack nicely, and we don't need it initialized either. --- src/libutil/posix-source-accessor.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 15ff76e59..5f26fa67b 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -25,7 +25,7 @@ void PosixSourceAccessor::readFile( off_t left = st.st_size; - std::vector buf(64 * 1024); + std::array buf; while (left) { checkInterrupt(); ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size())); From 69ed4aee612e247f2d6ebbb44aba743c4282e00e Mon Sep 17 00:00:00 2001 From: pennae Date: Mon, 11 Dec 2023 15:48:24 +0100 Subject: [PATCH 852/909] remove lazy-pos forceValue almost all uses of this are interactive, except for deepSeq. deepSeq is going to be expensive and rare enough to not care much about, and Value::determinePos should usually be cheap enough to not be too much of a burden in any case. --- src/libcmd/installable-flake.cc | 2 +- src/libcmd/repl.cc | 4 ++-- src/libexpr/eval-inline.hh | 10 +--------- src/libexpr/eval.cc | 2 +- src/libexpr/eval.hh | 3 --- src/libexpr/get-drvs.cc | 4 ++-- src/nix-build/nix-build.cc | 2 +- src/nix-env/user-env.cc | 2 +- src/nix-instantiate/nix-instantiate.cc | 2 +- 9 files changed, 10 insertions(+), 21 deletions(-) diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 2f428cb7e..ddec7537b 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -52,7 +52,7 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); assert(aOutputs); - state.forceValue(*aOutputs->value, [&]() { return aOutputs->value->determinePos(noPos); }); + state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos)); return aOutputs->value; } diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 0986296ad..97d709ff4 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -888,7 +888,7 @@ void NixRepl::evalString(std::string s, Value & v) { Expr * e = parseString(s); e->eval(*state, *env, v); - state->forceValue(v, [&]() { return v.determinePos(noPos); }); + state->forceValue(v, v.determinePos(noPos)); } @@ -907,7 +907,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m str.flush(); checkInterrupt(); - state->forceValue(v, [&]() { return v.determinePos(noPos); }); + state->forceValue(v, v.determinePos(noPos)); switch (v.type()) { diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 9d08f1938..8a9ebb77a 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -81,15 +81,7 @@ Env & EvalState::allocEnv(size_t size) } -[[gnu::always_inline]] void EvalState::forceValue(Value & v, const PosIdx pos) -{ - forceValue(v, [&]() { return pos; }); -} - - -template -void EvalState::forceValue(Value & v, Callable getPos) { if (v.isThunk()) { Env * env = v.thunk.env; @@ -110,7 +102,7 @@ void EvalState::forceValue(Value & v, Callable getPos) // only one black hole can *throw* in any given eval stack so we need not // check whether the position is set already. if (v.isBlackhole()) - e.err.errPos = positions[getPos()]; + e.err.errPos = positions[pos]; throw; } } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 8e89ddcf1..4dc5af97a 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2044,7 +2044,7 @@ void EvalState::forceValueDeep(Value & v) recurse = [&](Value & v) { if (!seen.insert(&v).second) return; - forceValue(v, [&]() { return v.determinePos(noPos); }); + forceValue(v, v.determinePos(noPos)); if (v.type() == nAttrs) { for (auto & i : *v.attrs) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e5e401ab6..4c7ea1d98 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -473,9 +473,6 @@ public: */ inline void forceValue(Value & v, const PosIdx pos); - template - inline void forceValue(Value & v, Callable getPos); - /** * Force a value, then recursively force list elements and * attributes. diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index d4e946d81..a6441871c 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -198,7 +198,7 @@ StringSet DrvInfo::queryMetaNames() bool DrvInfo::checkMeta(Value & v) { - state->forceValue(v, [&]() { return v.determinePos(noPos); }); + state->forceValue(v, v.determinePos(noPos)); if (v.type() == nList) { for (auto elem : v.listItems()) if (!checkMeta(*elem)) return false; @@ -304,7 +304,7 @@ static bool getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures) { try { - state.forceValue(v, [&]() { return v.determinePos(noPos); }); + state.forceValue(v, v.determinePos(noPos)); if (!state.isDerivation(v)) return true; /* Remove spurious duplicates (e.g., a set like `rec { x = diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 01da028d8..4465e2f90 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -350,7 +350,7 @@ static void main_nix_build(int argc, char * * argv) takesNixShellAttr(vRoot) ? *autoArgsWithInNixShell : *autoArgs, vRoot ).first); - state->forceValue(v, [&]() { return v.determinePos(noPos); }); + state->forceValue(v, v.determinePos(noPos)); getDerivations( *state, v, diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 34f6bd005..fe5b89b3f 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -128,7 +128,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* Evaluate it. */ debug("evaluating user environment builder"); - state.forceValue(topLevel, [&]() { return topLevel.determinePos(noPos); }); + state.forceValue(topLevel, topLevel.determinePos(noPos)); NixStringContext context; Attr & aDrvPath(*topLevel.attrs->find(state.sDrvPath)); auto topLevelDrv = state.coerceToStorePath(aDrvPath.pos, *aDrvPath.value, context, ""); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 86b9be17d..ab590b3a6 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -40,7 +40,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, for (auto & i : attrPaths) { Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot).first); - state.forceValue(v, [&]() { return v.determinePos(noPos); }); + state.forceValue(v, v.determinePos(noPos)); NixStringContext context; if (evalOnly) { From f9db4de0f3758e0f730a5d98348e7cc40082104a Mon Sep 17 00:00:00 2001 From: pennae Date: Mon, 11 Dec 2023 15:54:16 +0100 Subject: [PATCH 853/909] force-inline forceValue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit forceValue is extremely hot. interestingly adding likeliness annotations to the branches does not seem to make a difference. before: Time (mean ± σ): 4.224 s ± 0.005 s [User: 3.711 s, System: 0.512 s] Range (min … max): 4.218 s … 4.234 s 10 runs after: Time (mean ± σ): 4.140 s ± 0.009 s [User: 3.647 s, System: 0.492 s] Range (min … max): 4.130 s … 4.152 s 10 runs --- src/libexpr/eval-inline.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 8a9ebb77a..d48871628 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -81,6 +81,7 @@ Env & EvalState::allocEnv(size_t size) } +[[gnu::always_inline]] void EvalState::forceValue(Value & v, const PosIdx pos) { if (v.isThunk()) { From 2b0e95e7aabd075f95cbfb1607330b2284b01918 Mon Sep 17 00:00:00 2001 From: pennae Date: Mon, 11 Dec 2023 16:23:08 +0100 Subject: [PATCH 854/909] use singleton expr to generate black hole errors this also reduces forceValue code size and removes the need for hideInDiagnostics. coopting thunk forcing like this has the additional benefit of clarifying how these errors can happen in the first place. --- src/libexpr/eval-inline.hh | 14 +++----------- src/libexpr/eval.cc | 35 +++++++++++++++++++++++++++-------- src/libexpr/eval.hh | 10 ++-------- src/libexpr/nixexpr.cc | 2 ++ src/libexpr/nixexpr.hh | 10 ++++++++++ src/libexpr/primops.cc | 23 ----------------------- src/libexpr/primops.hh | 6 ------ src/libexpr/value.hh | 12 +++++++----- 8 files changed, 51 insertions(+), 61 deletions(-) diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index d48871628..52aa75b5f 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -93,20 +93,12 @@ void EvalState::forceValue(Value & v, const PosIdx pos) expr->eval(*this, *env, v); } catch (...) { v.mkThunk(env, expr); + tryFixupBlackHolePos(v, pos); throw; } } - else if (v.isApp()) { - try { - callFunction(*v.app.left, *v.app.right, v, noPos); - } catch (InfiniteRecursionError & e) { - // only one black hole can *throw* in any given eval stack so we need not - // check whether the position is set already. - if (v.isBlackhole()) - e.err.errPos = positions[pos]; - throw; - } - } + else if (v.isApp()) + callFunction(*v.app.left, *v.app.right, v, pos); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 4dc5af97a..0c35b3713 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -256,8 +256,8 @@ std::string showType(const Value & v) case tPrimOpApp: return fmt("the partially applied built-in function '%s'", std::string(getPrimOp(v)->primOp->name)); case tExternal: return v.external->showType(); - case tThunk: return "a thunk"; - case tApp: return v.isBlackhole() ? "a black hole" : "a function application"; + case tThunk: return v.isBlackhole() ? "a black hole" : "a thunk"; + case tApp: return "a function application"; default: return std::string(showType(v.type())); } @@ -1624,14 +1624,12 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & auto * fn = vCur.primOp; nrPrimOpCalls++; - // This will count black holes, but that's ok, because unrecoverable errors are rare. if (countCalls) primOpCalls[fn->name]++; try { fn->fun(*this, vCur.determinePos(noPos), args, vCur); } catch (Error & e) { - if (!fn->hideInDiagnostics) - addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); + addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); throw; } @@ -1670,7 +1668,6 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & auto fn = primOp->primOp; nrPrimOpCalls++; - // This will count black holes, but that's ok, because unrecoverable errors are rare. if (countCalls) primOpCalls[fn->name]++; try { @@ -1680,8 +1677,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & // so the debugger allows to inspect the wrong parameters passed to the builtin. fn->fun(*this, vCur.determinePos(noPos), vArgs, vCur); } catch (Error & e) { - if (!fn->hideInDiagnostics) - addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); + addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); throw; } @@ -2035,6 +2031,29 @@ void ExprPos::eval(EvalState & state, Env & env, Value & v) } +void ExprBlackHole::eval(EvalState & state, Env & env, Value & v) +{ + state.error("infinite recursion encountered") + .debugThrow(); +} + +// always force this to be separate, otherwise forceValue may inline it and take +// a massive perf hit +[[gnu::noinline]] +void EvalState::tryFixupBlackHolePos(Value & v, PosIdx pos) +{ + if (!v.isBlackhole()) + return; + auto e = std::current_exception(); + try { + std::rethrow_exception(e); + } catch (InfiniteRecursionError & e) { + e.err.errPos = positions[pos]; + } catch (...) { + } +} + + void EvalState::forceValueDeep(Value & v) { std::set seen; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 4c7ea1d98..56bc5e48f 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -77,14 +77,6 @@ struct PrimOp */ std::optional experimentalFeature; - /** - * Whether to hide this primop in diagnostics. - * - * Used to hide the fact that black holes are primop applications from - * stack traces. - */ - bool hideInDiagnostics; - /** * Validity check to be performed by functions that introduce primops, * such as RegisterPrimOp() and Value::mkPrimOp(). @@ -473,6 +465,8 @@ public: */ inline void forceValue(Value & v, const PosIdx pos); + void tryFixupBlackHolePos(Value & v, PosIdx pos); + /** * Force a value, then recursively force list elements and * attributes. diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 22be8e68c..84860b30f 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -9,6 +9,8 @@ namespace nix { +ExprBlackHole eBlackHole; + struct PosAdapter : AbstractPos { Pos::Origin origin; diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index cf6fd1a8d..1e57fec7a 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -462,6 +462,16 @@ struct ExprPos : Expr COMMON_METHODS }; +/* only used to mark thunks as black holes. */ +struct ExprBlackHole : Expr +{ + void show(const SymbolTable & symbols, std::ostream & str) const override {} + void eval(EvalState & state, Env & env, Value & v) override; + void bindVars(EvalState & es, const std::shared_ptr & env) override {} +}; + +extern ExprBlackHole eBlackHole; + /* Static environments are used to map variable names onto (level, displacement) pairs used to obtain the value of the variable at diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index b7e903667..2a71747a0 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4263,29 +4263,6 @@ static RegisterPrimOp primop_splitVersion({ }); -static void prim_blackHoleFn(EvalState & state, const PosIdx pos, Value * * args, Value & v) -{ - state.error("infinite recursion encountered") - .debugThrow(); -} - -static PrimOp primop_blackHole = { - .name = "«blackHole»", - .args = {}, - .fun = prim_blackHoleFn, - .hideInDiagnostics = true, -}; - -static Value makeBlackHole() -{ - Value v; - v.mkPrimOp(&primop_blackHole); - return v; -} - -Value prim_blackHole = makeBlackHole(); - - /************************************************************* * Primop registration *************************************************************/ diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 244eada86..45486608f 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -51,10 +51,4 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Valu */ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v); -/** - * Placeholder value for black holes, used to represent black holes as - * applications of this value to the evaluated thunks. - */ -extern Value prim_blackHole; - } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 52cd0f901..d9860e921 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -61,6 +61,7 @@ class Bindings; struct Env; struct Expr; struct ExprLambda; +struct ExprBlackHole; struct PrimOp; class Symbol; class PosIdx; @@ -442,16 +443,17 @@ public: }; -extern Value prim_blackHole; +extern ExprBlackHole eBlackHole; -inline bool Value::isBlackhole() const +bool Value::isBlackhole() const { - return internalType == tApp && app.left == &prim_blackHole; + return internalType == tThunk && thunk.expr == (Expr*) &eBlackHole; } -inline void Value::mkBlackhole() +void Value::mkBlackhole() { - mkApp(&prim_blackHole, &prim_blackHole); + internalType = tThunk; + thunk.expr = (Expr*) &eBlackHole; } From 26d60b837ca84856ceef18627b2354d26f002eb1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 20 Dec 2023 03:23:49 -0500 Subject: [PATCH 855/909] Move down fallback targets in `Makefile` This ensures `lib.mk` still defines `default` as the first target. This fixes some builds. --- Makefile | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index c62216df8..1fdb6e897 100644 --- a/Makefile +++ b/Makefile @@ -32,11 +32,6 @@ makefiles += \ tests/unit/libstore-support/local.mk \ tests/unit/libexpr/local.mk \ tests/unit/libexpr-support/local.mk -else -.PHONY: check -check: - @echo "Unit tests are disabled. Configure without '--disable-unit-tests', or avoid calling 'make check'." - @exit 1 endif ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes) @@ -46,11 +41,6 @@ makefiles += \ tests/functional/dyn-drv/local.mk \ tests/functional/test-libstoreconsumer/local.mk \ tests/functional/plugins/local.mk -else -.PHONY: installcheck -installcheck: - @echo "Functional tests are disabled. Configure without '--disable-functional-tests', or avoid calling 'make installcheck'." - @exit 1 endif OPTIMIZE = 1 @@ -64,9 +54,25 @@ endif include mk/lib.mk +# Must be included after `mk/lib.mk` so isn't the default target. +ifneq ($(ENABLE_UNIT_TESTS), yes) +.PHONY: check +check: + @echo "Unit tests are disabled. Configure without '--disable-unit-tests', or avoid calling 'make check'." + @exit 1 +endif + +ifneq ($(ENABLE_FUNCTIONAL_TESTS), yes) +.PHONY: installcheck +installcheck: + @echo "Functional tests are disabled. Configure without '--disable-functional-tests', or avoid calling 'make installcheck'." + @exit 1 +endif + # Must be included after `mk/lib.mk` so rules refer to variables defined # by the library. Rules are not "lazy" like variables, unfortunately. -ifeq ($(ENABLE_DOC_GEN),yes) + +ifeq ($(ENABLE_DOC_GEN), yes) $(eval $(call include-sub-makefile, doc/manual/local.mk)) else .PHONY: manual-html manpages @@ -75,7 +81,7 @@ manual-html manpages: @exit 1 endif -ifeq ($(ENABLE_INTERNAL_API_DOCS),yes) +ifeq ($(ENABLE_INTERNAL_API_DOCS), yes) $(eval $(call include-sub-makefile, doc/internal-api/local.mk)) else .PHONY: internal-api-html From ea454d8687b96376b221d7bdb1085968867c2496 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 20 Dec 2023 03:24:38 +0100 Subject: [PATCH 856/909] Undeprecate isNull There's no good reason to deprecate it: - For consistency reasons it should continue to exist, such that all primitive types have a corresponding `builtins.is*` primop. - There's no implementation cost to continuing to have this function - It costs users time to try to migrate away from it, e.g. https://github.com/NixOS/nixpkgs/pull/219747 and https://github.com/NixOS/nixpkgs/pull/275548 - Using it can give easier-to-read code like `all isNull list` Co-authored-by: Robert Hensing --- src/libexpr/primops.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8b689f0c8..1ca4a2541 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -438,9 +438,7 @@ static RegisterPrimOp primop_isNull({ .doc = R"( Return `true` if *e* evaluates to `null`, and `false` otherwise. - > **Warning** - > - > This function is *deprecated*; just write `e == null` instead. + This is equivalent to `e == null`. )", .fun = prim_isNull, }); From d77a39a314871b9c9a0a4d09b153c40ea9c8aaca Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Dec 2023 16:22:34 +0100 Subject: [PATCH 857/909] Fix indent --- src/libutil/url-name.cc | 45 +++++++++++++++++++++-------------------- src/libutil/url-name.hh | 2 +- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/src/libutil/url-name.cc b/src/libutil/url-name.cc index f94383e32..7e51aa2e1 100644 --- a/src/libutil/url-name.cc +++ b/src/libutil/url-name.cc @@ -13,35 +13,36 @@ static const std::regex gitProviderRegex("github|gitlab|sourcehut"); static const std::regex gitSchemeRegex("git($|\\+.*)"); static const std::regex defaultOutputRegex(".*\\.default($|\\^.*)"); -std::optional getNameFromURL(ParsedURL url) { - std::smatch match; +std::optional getNameFromURL(const ParsedURL & url) +{ + std::smatch match; - /* If there is a dir= argument, use its value */ - if (url.query.count("dir") > 0) - return url.query.at("dir"); + /* If there is a dir= argument, use its value */ + if (url.query.count("dir") > 0) + return url.query.at("dir"); - /* If the fragment isn't a "default" and contains two attribute elements, use the last one */ - if (std::regex_match(url.fragment, match, lastAttributeRegex)) - return match.str(1); + /* If the fragment isn't a "default" and contains two attribute elements, use the last one */ + if (std::regex_match(url.fragment, match, lastAttributeRegex)) + return match.str(1); - /* If this is a github/gitlab/sourcehut flake, use the repo name */ - if (std::regex_match(url.scheme, gitProviderRegex) && std::regex_match(url.path, match, secondPathSegmentRegex)) - return match.str(1); + /* If this is a github/gitlab/sourcehut flake, use the repo name */ + if (std::regex_match(url.scheme, gitProviderRegex) && std::regex_match(url.path, match, secondPathSegmentRegex)) + return match.str(1); - /* If it is a regular git flake, use the directory name */ - if (std::regex_match(url.scheme, gitSchemeRegex) && std::regex_match(url.path, match, lastPathSegmentRegex)) - return match.str(1); + /* If it is a regular git flake, use the directory name */ + if (std::regex_match(url.scheme, gitSchemeRegex) && std::regex_match(url.path, match, lastPathSegmentRegex)) + return match.str(1); - /* If everything failed but there is a non-default fragment, use it in full */ - if (!url.fragment.empty() && !std::regex_match(url.fragment, defaultOutputRegex)) - return url.fragment; + /* If everything failed but there is a non-default fragment, use it in full */ + if (!url.fragment.empty() && !std::regex_match(url.fragment, defaultOutputRegex)) + return url.fragment; - /* If there is no fragment, take the last element of the path */ - if (std::regex_match(url.path, match, lastPathSegmentRegex)) - return match.str(1); + /* If there is no fragment, take the last element of the path */ + if (std::regex_match(url.path, match, lastPathSegmentRegex)) + return match.str(1); - /* If even that didn't work, the URL does not contain enough info to determine a useful name */ - return {}; + /* If even that didn't work, the URL does not contain enough info to determine a useful name */ + return {}; } } diff --git a/src/libutil/url-name.hh b/src/libutil/url-name.hh index 188b951e5..6f32754d2 100644 --- a/src/libutil/url-name.hh +++ b/src/libutil/url-name.hh @@ -15,6 +15,6 @@ namespace nix { * flake output, for example because it is empty or "default". * Otherwise returns the extracted name. */ -std::optional getNameFromURL(ParsedURL url); +std::optional getNameFromURL(const ParsedURL & url); } From 14508ade289b884ab2ceb0645a549a69fba82cab Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Dec 2023 16:25:55 +0100 Subject: [PATCH 858/909] Typo --- src/nix/profile.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 98fa165e8..1d89815e2 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -119,7 +119,7 @@ struct ProfileManifest if (pathExists(manifestPath)) { auto json = nlohmann::json::parse(readFile(manifestPath)); - /* Keep track of alreay found names to allow preventing duplicates */ + /* Keep track of already found names to allow preventing duplicates. */ std::set foundNames; auto version = json.value("version", 0); From 942d635102810a310c747cee66d9e9f343e6b4c3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Dec 2023 16:33:53 +0100 Subject: [PATCH 859/909] Fix release notes --- doc/manual/rl-next/nix-profile-names.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/rl-next/nix-profile-names.md b/doc/manual/rl-next/nix-profile-names.md index 2f4a589a7..f5953bd72 100644 --- a/doc/manual/rl-next/nix-profile-names.md +++ b/doc/manual/rl-next/nix-profile-names.md @@ -1,5 +1,5 @@ --- -synopsis: nix profile: Allow referring to elements by human-readable name +synopsis: "`nix profile` now allows referring to elements by human-readable name" prs: 8678 --- From 5ed1884875cc6a6e9330b6c5a2f24c35e685f5a0 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Thu, 21 Dec 2023 10:14:54 -0800 Subject: [PATCH 860/909] libcmd: Installable::toStorePaths -> Installable::toStorePathSet --- src/libcmd/installables.cc | 4 ++-- src/libcmd/installables.hh | 2 +- src/nix/develop.cc | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 6b3c82374..be9ebe9ca 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -715,7 +715,7 @@ BuiltPaths Installable::toBuiltPaths( } } -StorePathSet Installable::toStorePaths( +StorePathSet Installable::toStorePathSet( ref evalStore, ref store, Realise mode, OperateOn operateOn, @@ -735,7 +735,7 @@ StorePath Installable::toStorePath( Realise mode, OperateOn operateOn, ref installable) { - auto paths = toStorePaths(evalStore, store, mode, operateOn, {installable}); + auto paths = toStorePathSet(evalStore, store, mode, operateOn, {installable}); if (paths.size() != 1) throw Error("argument '%s' should evaluate to one store path", installable->what()); diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index e087f935c..c8ad41388 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -165,7 +165,7 @@ struct Installable const Installables & installables, BuildMode bMode = bmNormal); - static std::set toStorePaths( + static std::set toStorePathSet( ref evalStore, ref store, Realise mode, diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 8db2de491..974020951 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -376,7 +376,7 @@ struct Common : InstallableCommand, MixProfile for (auto & [installable_, dir_] : redirects) { auto dir = absPath(dir_); auto installable = parseInstallable(store, installable_); - auto builtPaths = Installable::toStorePaths( + auto builtPaths = Installable::toStorePathSet( getEvalStore(), store, Realise::Nothing, OperateOn::Output, {installable}); for (auto & path: builtPaths) { auto from = store->printStorePath(path); @@ -631,7 +631,7 @@ struct CmdDevelop : Common, MixEnvironment bool found = false; - for (auto & path : Installable::toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, {bashInstallable})) { + for (auto & path : Installable::toStorePathSet(getEvalStore(), store, Realise::Outputs, OperateOn::Output, {bashInstallable})) { auto s = store->printStorePath(path) + "/bin/bash"; if (pathExists(s)) { shell = s; From 1fb43d1eee6f398686523c0bb80adb987c584c61 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Wed, 20 Dec 2023 10:25:22 -0800 Subject: [PATCH 861/909] tests: add a test for command line ordering --- tests/functional/shell-hello.nix | 16 ++++++++++++++++ tests/functional/shell.sh | 8 ++++++++ 2 files changed, 24 insertions(+) diff --git a/tests/functional/shell-hello.nix b/tests/functional/shell-hello.nix index 3fdd3501d..dfe66ef93 100644 --- a/tests/functional/shell-hello.nix +++ b/tests/functional/shell-hello.nix @@ -23,4 +23,20 @@ with import ./config.nix; chmod +x $dev/bin/hello2 ''; }; + + salve-mundi = mkDerivation { + name = "salve-mundi"; + outputs = [ "out" ]; + meta.outputsToInstall = [ "out" ]; + buildCommand = + '' + mkdir -p $out/bin + + cat > $out/bin/hello < Date: Mon, 18 Dec 2023 15:22:09 -0800 Subject: [PATCH 862/909] nix shell: reflect command line order in PATH order Prior to this change, Nix would prepend every installable to the PATH list in order to ensure that installables appeared before the current PATH from the ambient environment. With this change, all the installables are still prepended to the PATH, but in the same order as they appear on the command line. This means that the first of two packages that expose an executable `hello` would appear in the PATH first, and thus be executed first. See the test in the prior commit for a more concrete example. --- src/libcmd/installables.cc | 14 ++++++++++++++ src/libcmd/installables.hh | 7 +++++++ src/nix/run.cc | 9 ++++++--- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index be9ebe9ca..736c41a1e 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -729,6 +729,20 @@ StorePathSet Installable::toStorePathSet( return outPaths; } +StorePaths Installable::toStorePaths( + ref evalStore, + ref store, + Realise mode, OperateOn operateOn, + const Installables & installables) +{ + StorePaths outPaths; + for (auto & path : toBuiltPaths(evalStore, store, mode, operateOn, installables)) { + auto thisOutPaths = path.outPaths(); + outPaths.insert(outPaths.end(), thisOutPaths.begin(), thisOutPaths.end()); + } + return outPaths; +} + StorePath Installable::toStorePath( ref evalStore, ref store, diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index c8ad41388..95e8841ca 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -172,6 +172,13 @@ struct Installable OperateOn operateOn, const Installables & installables); + static std::vector toStorePaths( + ref evalStore, + ref store, + Realise mode, + OperateOn operateOn, + const Installables & installables); + static StorePath toStorePath( ref evalStore, ref store, diff --git a/src/nix/run.cc b/src/nix/run.cc index efc0c56a1..9bca5b9d0 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -114,7 +114,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment setEnviron(); - auto unixPath = tokenizeString(getEnv("PATH").value_or(""), ":"); + std::vector pathAdditions; while (!todo.empty()) { auto path = todo.front(); @@ -122,7 +122,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment if (!done.insert(path).second) continue; if (true) - unixPath.push_front(store->printStorePath(path) + "/bin"); + pathAdditions.push_back(store->printStorePath(path) + "/bin"); auto propPath = CanonPath(store->printStorePath(path)) + "nix-support" + "propagated-user-env-packages"; if (auto st = accessor->maybeLstat(propPath); st && st->type == SourceAccessor::tRegular) { @@ -131,7 +131,10 @@ struct CmdShell : InstallablesCommand, MixEnvironment } } - setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1); + auto unixPath = tokenizeString(getEnv("PATH").value_or(""), ":"); + unixPath.insert(unixPath.begin(), pathAdditions.begin(), pathAdditions.end()); + auto unixPathString = concatStringsSep(":", unixPath); + setenv("PATH", unixPathString.c_str(), 1); Strings args; for (auto & arg : command) args.push_back(arg); From 8c4ea12f11511519726737cc39bc5b4e089b9f33 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 21 Dec 2023 21:03:06 +0100 Subject: [PATCH 863/909] libutil/url-parts.hh: comment --- src/libutil/url-parts.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index e968eea4b..a3b4f5b99 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -25,6 +25,7 @@ const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRege /// A Git ref (i.e. branch or tag name). /// \todo check that this is correct. +/// This regex incomplete. See https://git-scm.com/docs/git-check-ref-format const static std::string refRegexS = "[a-zA-Z0-9@][a-zA-Z0-9_.\\/@-+]*"; extern std::regex refRegex; From 4f47152209a81a2bef421467ca4bec00023eec04 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 21 Dec 2023 23:11:25 +0100 Subject: [PATCH 864/909] libutil/url-parts.hh: Fix regex Regex syntax is awful. --- src/libutil/url-parts.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index a3b4f5b99..4bb37ea9b 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -26,7 +26,7 @@ const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRege /// A Git ref (i.e. branch or tag name). /// \todo check that this is correct. /// This regex incomplete. See https://git-scm.com/docs/git-check-ref-format -const static std::string refRegexS = "[a-zA-Z0-9@][a-zA-Z0-9_.\\/@-+]*"; +const static std::string refRegexS = "[a-zA-Z0-9@][a-zA-Z0-9_.\\/@+-]*"; extern std::regex refRegex; /// Instead of defining what a good Git Ref is, we define what a bad Git Ref is From 26d7b0c793b389b71218fc38e613d3f75ad72299 Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Thu, 21 Dec 2023 22:45:21 +0100 Subject: [PATCH 865/909] Move url-name utility to libexpr/flake --- src/{libutil => libexpr/flake}/url-name.cc | 0 src/{libutil => libexpr/flake}/url-name.hh | 0 src/nix/profile.cc | 2 +- tests/unit/{libutil => libexpr/flake}/url-name.cc | 2 +- tests/unit/libexpr/local.mk | 3 ++- 5 files changed, 4 insertions(+), 3 deletions(-) rename src/{libutil => libexpr/flake}/url-name.cc (100%) rename src/{libutil => libexpr/flake}/url-name.hh (100%) rename tests/unit/{libutil => libexpr/flake}/url-name.cc (99%) diff --git a/src/libutil/url-name.cc b/src/libexpr/flake/url-name.cc similarity index 100% rename from src/libutil/url-name.cc rename to src/libexpr/flake/url-name.cc diff --git a/src/libutil/url-name.hh b/src/libexpr/flake/url-name.hh similarity index 100% rename from src/libutil/url-name.hh rename to src/libexpr/flake/url-name.hh diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 1d89815e2..abd56e4f4 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -11,7 +11,7 @@ #include "profiles.hh" #include "names.hh" #include "url.hh" -#include "url-name.hh" +#include "flake/url-name.hh" #include #include diff --git a/tests/unit/libutil/url-name.cc b/tests/unit/libexpr/flake/url-name.cc similarity index 99% rename from tests/unit/libutil/url-name.cc rename to tests/unit/libexpr/flake/url-name.cc index f637efa89..84d32837c 100644 --- a/tests/unit/libutil/url-name.cc +++ b/tests/unit/libexpr/flake/url-name.cc @@ -1,4 +1,4 @@ -#include "url-name.hh" +#include "flake/url-name.hh" #include namespace nix { diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk index 5743880d7..25810ad9c 100644 --- a/tests/unit/libexpr/local.mk +++ b/tests/unit/libexpr/local.mk @@ -16,7 +16,8 @@ endif libexpr-tests_SOURCES := \ $(wildcard $(d)/*.cc) \ - $(wildcard $(d)/value/*.cc) + $(wildcard $(d)/value/*.cc) \ + $(wildcard $(d)/flake/*.cc) libexpr-tests_EXTRA_INCLUDES = \ -I tests/unit/libexpr-support \ From 4b4111866358b39d5bfc352fa58040c8a54a2759 Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Fri, 22 Dec 2023 09:38:13 +0100 Subject: [PATCH 866/909] Move flakeref tests to new flake/ subdirectory --- tests/unit/libexpr/{ => flake}/flakeref.cc | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/unit/libexpr/{ => flake}/flakeref.cc (100%) diff --git a/tests/unit/libexpr/flakeref.cc b/tests/unit/libexpr/flake/flakeref.cc similarity index 100% rename from tests/unit/libexpr/flakeref.cc rename to tests/unit/libexpr/flake/flakeref.cc From 936a3642264ac159f3f9093710be3465b70e0e89 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 22 Dec 2023 16:35:58 +0100 Subject: [PATCH 867/909] getNameFromURL(): Support uppercase characters in attribute names In particular, this makes it handle 'legacyPackages' correctly. --- src/libexpr/flake/url-name.cc | 2 +- tests/unit/libexpr/flake/url-name.cc | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libexpr/flake/url-name.cc b/src/libexpr/flake/url-name.cc index 7e51aa2e1..753f197d5 100644 --- a/src/libexpr/flake/url-name.cc +++ b/src/libexpr/flake/url-name.cc @@ -4,7 +4,7 @@ namespace nix { -static const std::string attributeNamePattern("[a-z0-9_-]+"); +static const std::string attributeNamePattern("[a-zA-Z0-9_-]+"); static const std::regex lastAttributeRegex("(?:" + attributeNamePattern + "\\.)*(?!default)(" + attributeNamePattern +")(\\^.*)?"); static const std::string pathSegmentPattern("[a-zA-Z0-9_-]+"); static const std::regex lastPathSegmentRegex(".*/(" + pathSegmentPattern +")"); diff --git a/tests/unit/libexpr/flake/url-name.cc b/tests/unit/libexpr/flake/url-name.cc index 84d32837c..85387b323 100644 --- a/tests/unit/libexpr/flake/url-name.cc +++ b/tests/unit/libexpr/flake/url-name.cc @@ -5,11 +5,13 @@ namespace nix { /* ----------- tests for url-name.hh --------------------------------------------------*/ - TEST(getNameFromURL, getsNameFromURL) { + TEST(getNameFromURL, getNameFromURL) { ASSERT_EQ(getNameFromURL(parseURL("path:/home/user/project")), "project"); ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#packages.x86_64-linux.hello")), "hello"); - ASSERT_EQ(getNameFromURL(parseURL("path:.#nonStandardAttr.mylaptop")), "nonStandardAttr.mylaptop"); - ASSERT_EQ(getNameFromURL(parseURL("path:./repos/myflake#nonStandardAttr.mylaptop")), "nonStandardAttr.mylaptop"); + ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#legacyPackages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#packages.x86_64-linux.Hello")), "Hello"); + ASSERT_EQ(getNameFromURL(parseURL("path:.#nonStandardAttr.mylaptop")), "mylaptop"); + ASSERT_EQ(getNameFromURL(parseURL("path:./repos/myflake#nonStandardAttr.mylaptop")), "mylaptop"); ASSERT_EQ(getNameFromURL(parseURL("path:./nixpkgs#packages.x86_64-linux.complex^bin,man")), "complex"); ASSERT_EQ(getNameFromURL(parseURL("path:./myproj#packages.x86_64-linux.default^*")), "myproj"); From 9cb287657bec5a969d8bb1678d598d9fa820e60b Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Sat, 23 Dec 2023 17:15:09 -0500 Subject: [PATCH 868/909] remote-store test: Break out IFD expression into a separate file --- tests/functional/ifd.nix | 10 ++++++++++ tests/functional/remote-store.sh | 13 +------------ 2 files changed, 11 insertions(+), 12 deletions(-) create mode 100644 tests/functional/ifd.nix diff --git a/tests/functional/ifd.nix b/tests/functional/ifd.nix new file mode 100644 index 000000000..d0b9b54ad --- /dev/null +++ b/tests/functional/ifd.nix @@ -0,0 +1,10 @@ +with import ./config.nix; +import ( + mkDerivation { + name = "foo"; + bla = import ./dependencies.nix {}; + buildCommand = " + echo \\\"hi\\\" > $out + "; + } +) diff --git a/tests/functional/remote-store.sh b/tests/functional/remote-store.sh index 5c7bfde46..dc80f8b55 100644 --- a/tests/functional/remote-store.sh +++ b/tests/functional/remote-store.sh @@ -19,18 +19,7 @@ else fi # Test import-from-derivation through the daemon. -[[ $(nix eval --impure --raw --expr ' - with import ./config.nix; - import ( - mkDerivation { - name = "foo"; - bla = import ./dependencies.nix {}; - buildCommand = " - echo \\\"hi\\\" > $out - "; - } - ) -') = hi ]] +[[ $(nix eval --impure --raw --file ./ifd.nix) = hi ]] storeCleared=1 NIX_REMOTE_=$NIX_REMOTE $SHELL ./user-envs.sh From c3942ef85ffbd83391410fbf012f1de366d2463c Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Sat, 23 Dec 2023 21:26:12 -0500 Subject: [PATCH 869/909] Build IFD in the build store when using eval-store. Previously, IFDs would be built within the eval store, even though one is typically using `--eval-store` precisely to *avoid* local builds. Because the resulting Nix expression must be copied back to the eval store in order to be imported, this requires the eval store to trust the build store's signatures. --- doc/manual/rl-next/ifd-eval-store.md | 8 ++++++++ src/libexpr/primops.cc | 19 +++++++++++++------ tests/functional/eval-store.sh | 8 ++++++++ 3 files changed, 29 insertions(+), 6 deletions(-) create mode 100644 doc/manual/rl-next/ifd-eval-store.md diff --git a/doc/manual/rl-next/ifd-eval-store.md b/doc/manual/rl-next/ifd-eval-store.md new file mode 100644 index 000000000..835e7e7a3 --- /dev/null +++ b/doc/manual/rl-next/ifd-eval-store.md @@ -0,0 +1,8 @@ +--- +synopsis: import-from-derivation builds the derivation in the build store +prs: 9661 +--- + +When using `--eval-store`, `import`ing from a derivation will now result in the derivation being built on the build store, i.e. the store specified in the `store` Nix option. + +Because the resulting Nix expression must be copied back to the eval store in order to be imported, this requires the eval store to trust the build store's signatures. diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a1502da45..58826b3bd 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -84,14 +84,14 @@ StringMap EvalState::realiseContext(const NixStringContext & context) /* Build/substitute the context. */ std::vector buildReqs; for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d }); - store->buildPaths(buildReqs); + buildStore->buildPaths(buildReqs, bmNormal, store); + + StorePathSet outputsToCopyAndAllow; for (auto & drv : drvs) { - auto outputs = resolveDerivedPath(*store, drv); + auto outputs = resolveDerivedPath(*buildStore, drv, &*store); for (auto & [outputName, outputPath] : outputs) { - /* Add the output of this derivations to the allowed - paths. */ - allowPath(store->toRealPath(outputPath)); + outputsToCopyAndAllow.insert(outputPath); /* Get all the output paths corresponding to the placeholders we had */ if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { @@ -101,12 +101,19 @@ StringMap EvalState::realiseContext(const NixStringContext & context) .drvPath = drv.drvPath, .output = outputName, }).render(), - store->printStorePath(outputPath) + buildStore->printStorePath(outputPath) ); } } } + if (store != buildStore) copyClosure(*buildStore, *store, outputsToCopyAndAllow); + for (auto & outputPath : outputsToCopyAndAllow) { + /* Add the output of this derivations to the allowed + paths. */ + allowPath(store->toRealPath(outputPath)); + } + return res; } diff --git a/tests/functional/eval-store.sh b/tests/functional/eval-store.sh index ec99fd953..9937ecbce 100644 --- a/tests/functional/eval-store.sh +++ b/tests/functional/eval-store.sh @@ -40,3 +40,11 @@ if [[ ! -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then (! ls $NIX_STORE_DIR/*.drv) fi ls $eval_store/nix/store/*.drv + +clearStore +rm -rf "$eval_store" + +# Confirm that import-from-derivation builds on the build store +[[ $(nix eval --eval-store "$eval_store?require-sigs=false" --impure --raw --file ./ifd.nix) = hi ]] +ls $NIX_STORE_DIR/*dependencies-top/foobar +(! ls $eval_store/nix/store/*dependencies-top/foobar) From e2399fc94935c9bc1ae6670c5d445214e039ac84 Mon Sep 17 00:00:00 2001 From: Brian Le Date: Tue, 26 Dec 2023 17:12:28 -0500 Subject: [PATCH 870/909] Change "dervation" typos to "derivation" --- doc/manual/src/language/derivations.md | 2 +- src/libstore/remote-store.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 2aded5527..cbb30d074 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -274,7 +274,7 @@ The [`builder`](#attr-builder) is executed as follows: directory (typically, `/nix/store`). - `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs` - is set to `true` for the dervation. A detailed explanation of this + is set to `true` for the derivation. A detailed explanation of this behavior can be found in the [section about structured attrs](./advanced-attributes.md#adv-attr-structuredAttrs). diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 4d0113594..f0df646ca 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -186,7 +186,7 @@ void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, if (m.find("parsing derivation") != std::string::npos && m.find("expected string") != std::string::npos && m.find("Derive([") != std::string::npos) - throw Error("%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw dervation is in the form '%s'", std::move(m), "DrvWithVersion(..)"); + throw Error("%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw derivation is in the form '%s'", std::move(m), "DrvWithVersion(..)"); } throw; } From b6313f64f7be11e0fe74b17cb31dbbf12b2e7725 Mon Sep 17 00:00:00 2001 From: DavHau Date: Wed, 27 Dec 2023 19:57:27 +0700 Subject: [PATCH 871/909] saner default for log-lines: change to 25 This seems to be a much saner default. 10 lines are just not enough in so many cases. --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index b35dc37a1..c12998f8e 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -144,7 +144,7 @@ public: */ bool verboseBuild = true; - Setting logLines{this, 10, "log-lines", + Setting logLines{this, 25, "log-lines", "The number of lines of the tail of " "the log to show if a build fails."}; From 99a691c8a1abffd30077bd5f005cb8d4bbafae5c Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 24 Dec 2023 21:14:08 +0100 Subject: [PATCH 872/909] don't use istreams in hot paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit istream sentry objects are very expensive for single-character operations, and since we don't configure exception masks for the istreams used here they don't even do anything. all we need is end-of-string checks and an advancing position in an immutable memory buffer, both of which can be had for much cheaper than istreams allow. the effect of this change is most apparent on empty stores. before: Benchmark 1: nix eval --raw --impure --expr 'with import {}; system' Time (mean ± σ): 7.167 s ± 0.013 s [User: 5.528 s, System: 1.431 s] Range (min … max): 7.147 s … 7.182 s 10 runs after: Benchmark 1: nix eval --raw --impure --expr 'with import {}; system' Time (mean ± σ): 6.963 s ± 0.011 s [User: 5.330 s, System: 1.421 s] Range (min … max): 6.943 s … 6.974 s 10 runs --- src/libstore/derivations.cc | 47 +++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 8a7d660ff..973ce5211 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -154,18 +154,39 @@ StorePath writeDerivation(Store & store, } -/* Read string `s' from stream `str'. */ -static void expect(std::istream & str, std::string_view s) -{ - for (auto & c : s) { - if (str.get() != c) - throw FormatError("expected string '%1%'", s); +namespace { +/** + * This mimics std::istream to some extent. We use this much smaller implementation + * instead of plain istreams because the sentry object overhead is too high. + */ +struct StringViewStream { + std::string_view remaining; + + int peek() const { + return remaining.empty() ? EOF : remaining[0]; } + + int get() { + if (remaining.empty()) return EOF; + char c = remaining[0]; + remaining.remove_prefix(1); + return c; + } +}; +} + + +/* Read string `s' from stream `str'. */ +static void expect(StringViewStream & str, std::string_view s) +{ + if (!str.remaining.starts_with(s)) + throw FormatError("expected string '%1%'", s); + str.remaining.remove_prefix(s.size()); } /* Read a C-style string from stream `str'. */ -static std::string parseString(std::istream & str) +static std::string parseString(StringViewStream & str) { std::string res; expect(str, "\""); @@ -187,7 +208,7 @@ static void validatePath(std::string_view s) { throw FormatError("bad path '%1%' in derivation", s); } -static Path parsePath(std::istream & str) +static Path parsePath(StringViewStream & str) { auto s = parseString(str); validatePath(s); @@ -195,7 +216,7 @@ static Path parsePath(std::istream & str) } -static bool endOfList(std::istream & str) +static bool endOfList(StringViewStream & str) { if (str.peek() == ',') { str.get(); @@ -209,7 +230,7 @@ static bool endOfList(std::istream & str) } -static StringSet parseStrings(std::istream & str, bool arePaths) +static StringSet parseStrings(StringViewStream & str, bool arePaths) { StringSet res; expect(str, "["); @@ -267,7 +288,7 @@ static DerivationOutput parseDerivationOutput( } static DerivationOutput parseDerivationOutput( - const StoreDirConfig & store, std::istringstream & str, + const StoreDirConfig & store, StringViewStream & str, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings) { expect(str, ","); const auto pathS = parseString(str); @@ -297,7 +318,7 @@ enum struct DerivationATermVersion { static DerivedPathMap::ChildNode parseDerivedPathMapNode( const StoreDirConfig & store, - std::istringstream & str, + StringViewStream & str, DerivationATermVersion version) { DerivedPathMap::ChildNode node; @@ -349,7 +370,7 @@ Derivation parseDerivation( Derivation drv; drv.name = name; - std::istringstream str(std::move(s)); + StringViewStream str{s}; expect(str, "D"); DerivationATermVersion version; switch (str.peek()) { From 2cfc4ace35d1c8cca917c487be3cfddfcf3bba01 Mon Sep 17 00:00:00 2001 From: pennae Date: Tue, 26 Dec 2023 17:40:55 +0100 Subject: [PATCH 873/909] malloc/memset even less MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit more buffers that can be uninitialized and on the stack. small difference, but still worth doing. before: Benchmark 1: nix eval --raw --impure --expr 'with import {}; system' Time (mean ± σ): 6.963 s ± 0.011 s [User: 5.330 s, System: 1.421 s] Range (min … max): 6.943 s … 6.974 s 10 runs after: Benchmark 1: nix eval --raw --impure --expr 'with import {}; system' Time (mean ± σ): 6.952 s ± 0.015 s [User: 5.294 s, System: 1.452 s] Range (min … max): 6.926 s … 6.974 s 10 runs --- src/libutil/archive.cc | 2 +- src/libutil/file-system.cc | 2 +- src/libutil/serialise.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 465df2073..712ea51c7 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -140,7 +140,7 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path) sink.preallocateContents(size); uint64_t left = size; - std::vector buf(65536); + std::array buf; while (left) { checkInterrupt(); diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index c96effff9..4cac35ace 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -307,7 +307,7 @@ void writeFile(const Path & path, Source & source, mode_t mode, bool sync) if (!fd) throw SysError("opening file '%1%'", path); - std::vector buf(64 * 1024); + std::array buf; try { while (true) { diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index f465bd0de..76b378e18 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -82,7 +82,7 @@ void Source::operator () (std::string_view data) void Source::drainInto(Sink & sink) { std::string s; - std::vector buf(8192); + std::array buf; while (true) { size_t n; try { From 7434caca0545bd6194bb52eebf6fdf0424755eb0 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Fri, 15 Dec 2023 11:52:21 -0800 Subject: [PATCH 874/909] Fix segfault on infinite recursion in some cases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes a segfault on infinite function call recursion (rather than infinite thunk recursion) by tracking the function call depth in `EvalState`. Additionally, to avoid printing extremely long stack traces, stack frames are now deduplicated, with a `(19997 duplicate traces omitted)` message. This should only really be triggered in infinite recursion scenarios. Before: $ nix-instantiate --eval --expr '(x: x x) (x: x x)' Segmentation fault: 11 After: $ nix-instantiate --eval --expr '(x: x x) (x: x x)' error: stack overflow at «string»:1:14: 1| (x: x x) (x: x x) | ^ $ nix-instantiate --eval --expr '(x: x x) (x: x x)' --show-trace error: … from call site at «string»:1:1: 1| (x: x x) (x: x x) | ^ … while calling anonymous lambda at «string»:1:2: 1| (x: x x) (x: x x) | ^ … from call site at «string»:1:5: 1| (x: x x) (x: x x) | ^ … while calling anonymous lambda at «string»:1:11: 1| (x: x x) (x: x x) | ^ … from call site at «string»:1:14: 1| (x: x x) (x: x x) | ^ (19997 duplicate traces omitted) error: stack overflow at «string»:1:14: 1| (x: x x) (x: x x) | ^ --- .../rl-next/stack-overflow-segfaults.md | 32 +++++ src/libexpr/eval-settings.hh | 3 + src/libexpr/eval.cc | 18 +++ src/libexpr/eval.hh | 5 + src/libutil/error.cc | 111 +++++++++++++++++- src/libutil/error.hh | 8 ++ .../lang/eval-fail-duplicate-traces.err.exp | 44 +++++++ .../lang/eval-fail-duplicate-traces.nix | 9 ++ ...val-fail-infinite-recursion-lambda.err.exp | 38 ++++++ .../eval-fail-infinite-recursion-lambda.nix | 1 + .../lang/eval-fail-mutual-recursion.err.exp | 57 +++++++++ .../lang/eval-fail-mutual-recursion.nix | 36 ++++++ 12 files changed, 358 insertions(+), 4 deletions(-) create mode 100644 doc/manual/rl-next/stack-overflow-segfaults.md create mode 100644 tests/functional/lang/eval-fail-duplicate-traces.err.exp create mode 100644 tests/functional/lang/eval-fail-duplicate-traces.nix create mode 100644 tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp create mode 100644 tests/functional/lang/eval-fail-infinite-recursion-lambda.nix create mode 100644 tests/functional/lang/eval-fail-mutual-recursion.err.exp create mode 100644 tests/functional/lang/eval-fail-mutual-recursion.nix diff --git a/doc/manual/rl-next/stack-overflow-segfaults.md b/doc/manual/rl-next/stack-overflow-segfaults.md new file mode 100644 index 000000000..3d9753248 --- /dev/null +++ b/doc/manual/rl-next/stack-overflow-segfaults.md @@ -0,0 +1,32 @@ +--- +synopsis: Some stack overflow segfaults are fixed +issues: 9616 +prs: 9617 +--- + +The number of nested function calls has been restricted, to detect and report +infinite function call recursions. The default maximum call depth is 10,000 and +can be set with [the `max-call-depth` +option](@docroot@/command-ref/conf-file.md#conf-max-call-depth). + +This fixes segfaults or the following unhelpful error message in many cases: + + error: stack overflow (possible infinite recursion) + +Before: + +``` +$ nix-instantiate --eval --expr '(x: x x) (x: x x)' +Segmentation fault: 11 +``` + +After: + +``` +$ nix-instantiate --eval --expr '(x: x x) (x: x x)' +error: stack overflow + + at «string»:1:14: + 1| (x: x x) (x: x x) + | ^ +``` diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index ad187ca01..2f6c12d45 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -124,6 +124,9 @@ struct EvalSettings : Config Setting traceVerbose{this, false, "trace-verbose", "Whether `builtins.traceVerbose` should trace its first argument when evaluated."}; + + Setting maxCallDepth{this, 10000, "max-call-depth", + "The maximum function call depth to allow before erroring."}; }; extern EvalSettings evalSettings; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 810843995..f73e22ba0 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1505,9 +1505,27 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v) v.mkLambda(&env, this); } +namespace { +/** Increments a count on construction and decrements on destruction. + */ +class CallDepth { + size_t & count; +public: + CallDepth(size_t & count) : count(count) { + ++count; + } + ~CallDepth() { + --count; + } +}; +}; void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos) { + if (callDepth > evalSettings.maxCallDepth) + error("stack overflow; max-call-depth exceeded").atPos(pos).template debugThrow(); + CallDepth _level(callDepth); + auto trace = evalSettings.traceFunctionCalls ? std::make_unique(positions[pos]) : nullptr; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index da2d256db..7dbffe38c 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -622,6 +622,11 @@ private: const SourcePath & basePath, std::shared_ptr & staticEnv); + /** + * Current Nix call stack depth, used with `max-call-depth` setting to throw stack overflow hopefully before we run out of system stack. + */ + size_t callDepth = 0; + public: /** diff --git a/src/libutil/error.cc b/src/libutil/error.cc index bc0194d59..e42925c2b 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -50,6 +50,32 @@ std::ostream & operator <<(std::ostream & str, const AbstractPos & pos) return str; } +/** + * An arbitrarily defined value comparison for the purpose of using traces in the key of a sorted container. + */ +inline bool operator<(const Trace& lhs, const Trace& rhs) +{ + // `std::shared_ptr` does not have value semantics for its comparison + // functions, so we need to check for nulls and compare the dereferenced + // values here. + if (lhs.pos != rhs.pos) { + if (!lhs.pos) + return true; + if (!rhs.pos) + return false; + if (*lhs.pos != *rhs.pos) + return *lhs.pos < *rhs.pos; + } + // This formats a freshly formatted hint string and then throws it away, which + // shouldn't be much of a problem because it only runs when pos is equal, and this function is + // used for trace printing, which is infrequent. + return std::forward_as_tuple(lhs.hint.str(), lhs.frame) + < std::forward_as_tuple(rhs.hint.str(), rhs.frame); +} +inline bool operator> (const Trace& lhs, const Trace& rhs) { return rhs < lhs; } +inline bool operator<=(const Trace& lhs, const Trace& rhs) { return !(lhs > rhs); } +inline bool operator>=(const Trace& lhs, const Trace& rhs) { return !(lhs < rhs); } + std::optional AbstractPos::getCodeLines() const { if (line == 0) @@ -185,6 +211,69 @@ static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std return hasPos; } +void printTrace( + std::ostream & output, + const std::string_view & indent, + size_t & count, + const Trace & trace) +{ + output << "\n" << "… " << trace.hint.str() << "\n"; + + if (printPosMaybe(output, indent, trace.pos)) + count++; +} + +void printSkippedTracesMaybe( + std::ostream & output, + const std::string_view & indent, + size_t & count, + std::vector & skippedTraces, + std::set tracesSeen) +{ + if (skippedTraces.size() > 0) { + // If we only skipped a few frames, print them out normally; + // messages like "1 duplicate frames omitted" aren't helpful. + if (skippedTraces.size() <= 5) { + for (auto & trace : skippedTraces) { + printTrace(output, indent, count, trace); + } + } else { + output << "\n" << ANSI_WARNING "(" << skippedTraces.size() << " duplicate frames omitted)" ANSI_NORMAL << "\n"; + // Clear the set of "seen" traces after printing a chunk of + // `duplicate frames omitted`. + // + // Consider a mutually recursive stack trace with: + // - 10 entries of A + // - 10 entries of B + // - 10 entries of A + // + // If we don't clear `tracesSeen` here, we would print output like this: + // - 1 entry of A + // - (9 duplicate frames omitted) + // - 1 entry of B + // - (19 duplicate frames omitted) + // + // This would obscure the control flow, which went from A, + // to B, and back to A again. + // + // In contrast, if we do clear `tracesSeen`, the output looks like this: + // - 1 entry of A + // - (9 duplicate frames omitted) + // - 1 entry of B + // - (9 duplicate frames omitted) + // - 1 entry of A + // - (9 duplicate frames omitted) + // + // See: `tests/functional/lang/eval-fail-mutual-recursion.nix` + tracesSeen.clear(); + } + } + // We've either printed each trace in `skippedTraces` normally, or + // printed a chunk of `duplicate frames omitted`. Either way, we've + // processed these traces and can clear them. + skippedTraces.clear(); +} + std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace) { std::string prefix; @@ -333,7 +422,13 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s bool frameOnly = false; if (!einfo.traces.empty()) { + // Stack traces seen since we last printed a chunk of `duplicate frames + // omitted`. + std::set tracesSeen; + // A consecutive sequence of stack traces that are all in `tracesSeen`. + std::vector skippedTraces; size_t count = 0; + for (const auto & trace : einfo.traces) { if (trace.hint.str().empty()) continue; if (frameOnly && !trace.frame) continue; @@ -343,14 +438,21 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s break; } + if (tracesSeen.count(trace)) { + skippedTraces.push_back(trace); + continue; + } + tracesSeen.insert(trace); + + printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen); + count++; frameOnly = trace.frame; - oss << "\n" << "… " << trace.hint.str() << "\n"; - - if (printPosMaybe(oss, ellipsisIndent, trace.pos)) - count++; + printTrace(oss, ellipsisIndent, count, trace); } + + printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen); oss << "\n" << prefix; } @@ -369,4 +471,5 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s return out; } + } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index c04dcbd77..baffca128 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -88,6 +89,8 @@ struct AbstractPos std::optional getCodeLines() const; virtual ~AbstractPos() = default; + + inline auto operator<=>(const AbstractPos& rhs) const = default; }; std::ostream & operator << (std::ostream & str, const AbstractPos & pos); @@ -103,6 +106,11 @@ struct Trace { bool frame; }; +inline bool operator<(const Trace& lhs, const Trace& rhs); +inline bool operator> (const Trace& lhs, const Trace& rhs); +inline bool operator<=(const Trace& lhs, const Trace& rhs); +inline bool operator>=(const Trace& lhs, const Trace& rhs); + struct ErrorInfo { Verbosity level; hintformat msg; diff --git a/tests/functional/lang/eval-fail-duplicate-traces.err.exp b/tests/functional/lang/eval-fail-duplicate-traces.err.exp new file mode 100644 index 000000000..32ad9b376 --- /dev/null +++ b/tests/functional/lang/eval-fail-duplicate-traces.err.exp @@ -0,0 +1,44 @@ +error: + … from call site + at /pwd/lang/eval-fail-duplicate-traces.nix:9:3: + 8| in + 9| throwAfter 2 + | ^ + 10| + + … while calling 'throwAfter' + at /pwd/lang/eval-fail-duplicate-traces.nix:4:16: + 3| let + 4| throwAfter = n: + | ^ + 5| if n > 0 + + … from call site + at /pwd/lang/eval-fail-duplicate-traces.nix:6:10: + 5| if n > 0 + 6| then throwAfter (n - 1) + | ^ + 7| else throw "Uh oh!"; + + … while calling 'throwAfter' + at /pwd/lang/eval-fail-duplicate-traces.nix:4:16: + 3| let + 4| throwAfter = n: + | ^ + 5| if n > 0 + + … from call site + at /pwd/lang/eval-fail-duplicate-traces.nix:6:10: + 5| if n > 0 + 6| then throwAfter (n - 1) + | ^ + 7| else throw "Uh oh!"; + + … while calling 'throwAfter' + at /pwd/lang/eval-fail-duplicate-traces.nix:4:16: + 3| let + 4| throwAfter = n: + | ^ + 5| if n > 0 + + error: Uh oh! diff --git a/tests/functional/lang/eval-fail-duplicate-traces.nix b/tests/functional/lang/eval-fail-duplicate-traces.nix new file mode 100644 index 000000000..17ce374ec --- /dev/null +++ b/tests/functional/lang/eval-fail-duplicate-traces.nix @@ -0,0 +1,9 @@ +# Check that we only omit duplicate stack traces when there's a bunch of them. +# Here, there's only a couple duplicate entries, so we output them all. +let + throwAfter = n: + if n > 0 + then throwAfter (n - 1) + else throw "Uh oh!"; +in + throwAfter 2 diff --git a/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp b/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp new file mode 100644 index 000000000..5d843d827 --- /dev/null +++ b/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp @@ -0,0 +1,38 @@ +error: + … from call site + at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:1: + 1| (x: x x) (x: x x) + | ^ + 2| + + … while calling anonymous lambda + at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:2: + 1| (x: x x) (x: x x) + | ^ + 2| + + … from call site + at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:5: + 1| (x: x x) (x: x x) + | ^ + 2| + + … while calling anonymous lambda + at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:11: + 1| (x: x x) (x: x x) + | ^ + 2| + + … from call site + at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:14: + 1| (x: x x) (x: x x) + | ^ + 2| + + (19997 duplicate frames omitted) + + error: stack overflow; max-call-depth exceeded + at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:14: + 1| (x: x x) (x: x x) + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-infinite-recursion-lambda.nix b/tests/functional/lang/eval-fail-infinite-recursion-lambda.nix new file mode 100644 index 000000000..dd0a8bf2e --- /dev/null +++ b/tests/functional/lang/eval-fail-infinite-recursion-lambda.nix @@ -0,0 +1 @@ +(x: x x) (x: x x) diff --git a/tests/functional/lang/eval-fail-mutual-recursion.err.exp b/tests/functional/lang/eval-fail-mutual-recursion.err.exp new file mode 100644 index 000000000..dc2e11766 --- /dev/null +++ b/tests/functional/lang/eval-fail-mutual-recursion.err.exp @@ -0,0 +1,57 @@ +error: + … from call site + at /pwd/lang/eval-fail-mutual-recursion.nix:36:3: + 35| in + 36| throwAfterA true 10 + | ^ + 37| + + … while calling 'throwAfterA' + at /pwd/lang/eval-fail-mutual-recursion.nix:29:26: + 28| + 29| throwAfterA = recurse: n: + | ^ + 30| if n > 0 + + … from call site + at /pwd/lang/eval-fail-mutual-recursion.nix:31:10: + 30| if n > 0 + 31| then throwAfterA recurse (n - 1) + | ^ + 32| else if recurse + + (19 duplicate frames omitted) + + … from call site + at /pwd/lang/eval-fail-mutual-recursion.nix:33:10: + 32| else if recurse + 33| then throwAfterB true 10 + | ^ + 34| else throw "Uh oh!"; + + … while calling 'throwAfterB' + at /pwd/lang/eval-fail-mutual-recursion.nix:22:26: + 21| let + 22| throwAfterB = recurse: n: + | ^ + 23| if n > 0 + + … from call site + at /pwd/lang/eval-fail-mutual-recursion.nix:24:10: + 23| if n > 0 + 24| then throwAfterB recurse (n - 1) + | ^ + 25| else if recurse + + (19 duplicate frames omitted) + + … from call site + at /pwd/lang/eval-fail-mutual-recursion.nix:26:10: + 25| else if recurse + 26| then throwAfterA false 10 + | ^ + 27| else throw "Uh oh!"; + + (21 duplicate frames omitted) + + error: Uh oh! diff --git a/tests/functional/lang/eval-fail-mutual-recursion.nix b/tests/functional/lang/eval-fail-mutual-recursion.nix new file mode 100644 index 000000000..d090d3158 --- /dev/null +++ b/tests/functional/lang/eval-fail-mutual-recursion.nix @@ -0,0 +1,36 @@ +# Check that stack frame deduplication only affects consecutive intervals, and +# that they are reported independently of any preceding sections, even if +# they're indistinguishable. +# +# In terms of the current implementation, we check that we clear the set of +# "seen frames" after eliding a group of frames. +# +# Suppose we have: +# - 10 frames in a function A +# - 10 frames in a function B +# - 10 frames in a function A +# +# We want to output: +# - a few frames of A (skip the rest) +# - a few frames of B (skip the rest) +# - a few frames of A (skip the rest) +# +# If we implemented this in the naive manner, we'd instead get: +# - a few frames of A (skip the rest) +# - a few frames of B (skip the rest, _and_ skip the remaining frames of A) +let + throwAfterB = recurse: n: + if n > 0 + then throwAfterB recurse (n - 1) + else if recurse + then throwAfterA false 10 + else throw "Uh oh!"; + + throwAfterA = recurse: n: + if n > 0 + then throwAfterA recurse (n - 1) + else if recurse + then throwAfterB true 10 + else throw "Uh oh!"; +in + throwAfterA true 10 From 79d3d412cacd210bc9a0e9ba5407eea67c8e3868 Mon Sep 17 00:00:00 2001 From: pennae Date: Tue, 26 Dec 2023 22:18:42 +0100 Subject: [PATCH 875/909] optimize derivation string parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit a bunch of derivation strings contain no escape sequences. we can optimize for this fact by first scanning for the end of a derivation string and simply returning the contents unmodified if no escape sequences were found. to make this even more efficient we can also use BackedStringViews to avoid copies, avoiding heap allocations for transient data. before: Benchmark 1: nix eval --raw --impure --expr 'with import {}; system' Time (mean ± σ): 6.952 s ± 0.015 s [User: 5.294 s, System: 1.452 s] Range (min … max): 6.926 s … 6.974 s 10 runs after: Benchmark 1: nix eval --raw --impure --expr 'with import {}; system' Time (mean ± σ): 6.907 s ± 0.012 s [User: 5.272 s, System: 1.429 s] Range (min … max): 6.893 s … 6.926 s 10 runs --- doc/manual/rl-next/drv-string-parse-hang.md | 6 ++ src/libstore/derivations.cc | 65 +++++++++++++-------- 2 files changed, 48 insertions(+), 23 deletions(-) create mode 100644 doc/manual/rl-next/drv-string-parse-hang.md diff --git a/doc/manual/rl-next/drv-string-parse-hang.md b/doc/manual/rl-next/drv-string-parse-hang.md new file mode 100644 index 000000000..1e041d3e9 --- /dev/null +++ b/doc/manual/rl-next/drv-string-parse-hang.md @@ -0,0 +1,6 @@ +--- +synopsis: Fix handling of truncated `.drv` files. +prs: 9673 +--- + +Previously a `.drv` that was truncated in the middle of a string would case nix to enter an infinite loop, eventually exhausting all memory and crashing. diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 973ce5211..89d902917 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -2,6 +2,7 @@ #include "downstream-placeholder.hh" #include "store-api.hh" #include "globals.hh" +#include "types.hh" #include "util.hh" #include "split.hh" #include "common-protocol.hh" @@ -186,20 +187,38 @@ static void expect(StringViewStream & str, std::string_view s) /* Read a C-style string from stream `str'. */ -static std::string parseString(StringViewStream & str) +static BackedStringView parseString(StringViewStream & str) { - std::string res; expect(str, "\""); - int c; - while ((c = str.get()) != '"') - if (c == '\\') { - c = str.get(); - if (c == 'n') res += '\n'; - else if (c == 'r') res += '\r'; - else if (c == 't') res += '\t'; - else res += c; + auto c = str.remaining.begin(), end = str.remaining.end(); + bool escaped = false; + for (; c != end && *c != '"'; c++) { + if (*c == '\\') { + c++; + if (c == end) + throw FormatError("unterminated string in derivation"); + escaped = true; } - else res += c; + } + + const auto contentLen = c - str.remaining.begin(); + const auto content = str.remaining.substr(0, contentLen); + str.remaining.remove_prefix(contentLen + 1); + + if (!escaped) + return content; + + std::string res; + res.reserve(content.size()); + for (c = content.begin(), end = content.end(); c != end; c++) + if (*c == '\\') { + c++; + if (*c == 'n') res += '\n'; + else if (*c == 'r') res += '\r'; + else if (*c == 't') res += '\t'; + else res += *c; + } + else res += *c; return res; } @@ -210,7 +229,7 @@ static void validatePath(std::string_view s) { static Path parsePath(StringViewStream & str) { - auto s = parseString(str); + auto s = parseString(str).toOwned(); validatePath(s); return s; } @@ -235,7 +254,7 @@ static StringSet parseStrings(StringViewStream & str, bool arePaths) StringSet res; expect(str, "["); while (!endOfList(str)) - res.insert(arePaths ? parsePath(str) : parseString(str)); + res.insert(arePaths ? parsePath(str) : parseString(str).toOwned()); return res; } @@ -296,7 +315,7 @@ static DerivationOutput parseDerivationOutput( expect(str, ","); const auto hash = parseString(str); expect(str, ")"); - return parseDerivationOutput(store, pathS, hashAlgo, hash, xpSettings); + return parseDerivationOutput(store, *pathS, *hashAlgo, *hash, xpSettings); } /** @@ -344,7 +363,7 @@ static DerivedPathMap::ChildNode parseDerivedPathMapNode( expect(str, ",["); while (!endOfList(str)) { expect(str, "("); - auto outputName = parseString(str); + auto outputName = parseString(str).toOwned(); expect(str, ","); node.childMap.insert_or_assign(outputName, parseDerivedPathMapNode(store, str, version)); expect(str, ")"); @@ -381,12 +400,12 @@ Derivation parseDerivation( case 'r': { expect(str, "rvWithVersion("); auto versionS = parseString(str); - if (versionS == "xp-dyn-drv") { + if (*versionS == "xp-dyn-drv") { // Only verison we have so far version = DerivationATermVersion::DynamicDerivations; xpSettings.require(Xp::DynamicDerivations); } else { - throw FormatError("Unknown derivation ATerm format version '%s'", versionS); + throw FormatError("Unknown derivation ATerm format version '%s'", *versionS); } expect(str, ","); break; @@ -398,7 +417,7 @@ Derivation parseDerivation( /* Parse the list of outputs. */ expect(str, "["); while (!endOfList(str)) { - expect(str, "("); std::string id = parseString(str); + expect(str, "("); std::string id = parseString(str).toOwned(); auto output = parseDerivationOutput(store, str, xpSettings); drv.outputs.emplace(std::move(id), std::move(output)); } @@ -414,19 +433,19 @@ Derivation parseDerivation( } expect(str, ","); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true)); - expect(str, ","); drv.platform = parseString(str); - expect(str, ","); drv.builder = parseString(str); + expect(str, ","); drv.platform = parseString(str).toOwned(); + expect(str, ","); drv.builder = parseString(str).toOwned(); /* Parse the builder arguments. */ expect(str, ",["); while (!endOfList(str)) - drv.args.push_back(parseString(str)); + drv.args.push_back(parseString(str).toOwned()); /* Parse the environment variables. */ expect(str, ",["); while (!endOfList(str)) { - expect(str, "("); auto name = parseString(str); - expect(str, ","); auto value = parseString(str); + expect(str, "("); auto name = parseString(str).toOwned(); + expect(str, ","); auto value = parseString(str).toOwned(); expect(str, ")"); drv.env[name] = value; } From 02c64abf1e892220cb62ce3b7e1598030fb6a61c Mon Sep 17 00:00:00 2001 From: pennae Date: Tue, 26 Dec 2023 05:44:52 +0100 Subject: [PATCH 876/909] use translation table for drv string parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the table is very small compared to cache sizes and a single indexed load is faster than three comparisons. before: Benchmark 1: nix eval --raw --impure --expr 'with import {}; system' Time (mean ± σ): 6.907 s ± 0.012 s [User: 5.272 s, System: 1.429 s] Range (min … max): 6.893 s … 6.926 s 10 runs after: Benchmark 1: nix eval --raw --impure --expr 'with import {}; system' Time (mean ± σ): 6.883 s ± 0.016 s [User: 5.250 s, System: 1.424 s] Range (min … max): 6.860 s … 6.905 s 10 runs --- src/libstore/derivations.cc | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 89d902917..89a345057 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -174,6 +174,17 @@ struct StringViewStream { return c; } }; + +constexpr struct Escapes { + char map[256]; + constexpr Escapes() { + for (int i = 0; i < 256; i++) map[i] = (char) (unsigned char) i; + map[(int) (unsigned char) 'n'] = '\n'; + map[(int) (unsigned char) 'r'] = '\r'; + map[(int) (unsigned char) 't'] = '\t'; + } + char operator[](char c) const { return map[(unsigned char) c]; } +} escapes; } @@ -213,10 +224,7 @@ static BackedStringView parseString(StringViewStream & str) for (c = content.begin(), end = content.end(); c != end; c++) if (*c == '\\') { c++; - if (*c == 'n') res += '\n'; - else if (*c == 'r') res += '\r'; - else if (*c == 't') res += '\t'; - else res += *c; + res += escapes[*c]; } else res += *c; return res; From c62686a95bd3ebbf3f5104c27222e751e84b84a3 Mon Sep 17 00:00:00 2001 From: pennae Date: Wed, 27 Dec 2023 04:26:50 +0100 Subject: [PATCH 877/909] reduce copies during drv parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit many paths need not be heap-allocated, and derivation env name/valye pairs can be moved into the map. before: Benchmark 1: nix eval --raw --impure --expr 'with import {}; system' Time (mean ± σ): 6.883 s ± 0.016 s [User: 5.250 s, System: 1.424 s] Range (min … max): 6.860 s … 6.905 s 10 runs after: Benchmark 1: nix eval --raw --impure --expr 'with import {}; system' Time (mean ± σ): 6.868 s ± 0.027 s [User: 5.194 s, System: 1.466 s] Range (min … max): 6.828 s … 6.913 s 10 runs --- src/libstore/derivations.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 89a345057..2fafcb8e7 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -235,10 +235,10 @@ static void validatePath(std::string_view s) { throw FormatError("bad path '%1%' in derivation", s); } -static Path parsePath(StringViewStream & str) +static BackedStringView parsePath(StringViewStream & str) { - auto s = parseString(str).toOwned(); - validatePath(s); + auto s = parseString(str); + validatePath(*s); return s; } @@ -262,7 +262,7 @@ static StringSet parseStrings(StringViewStream & str, bool arePaths) StringSet res; expect(str, "["); while (!endOfList(str)) - res.insert(arePaths ? parsePath(str) : parseString(str).toOwned()); + res.insert((arePaths ? parsePath(str) : parseString(str)).toOwned()); return res; } @@ -434,9 +434,9 @@ Derivation parseDerivation( expect(str, ",["); while (!endOfList(str)) { expect(str, "("); - Path drvPath = parsePath(str); + auto drvPath = parsePath(str); expect(str, ","); - drv.inputDrvs.map.insert_or_assign(store.parseStorePath(drvPath), parseDerivedPathMapNode(store, str, version)); + drv.inputDrvs.map.insert_or_assign(store.parseStorePath(*drvPath), parseDerivedPathMapNode(store, str, version)); expect(str, ")"); } @@ -455,7 +455,7 @@ Derivation parseDerivation( expect(str, "("); auto name = parseString(str).toOwned(); expect(str, ","); auto value = parseString(str).toOwned(); expect(str, ")"); - drv.env[name] = value; + drv.env.insert_or_assign(std::move(name), std::move(value)); } expect(str, ")"); From 1fe66852ff87e98615f35e8aac64675ff988fb5a Mon Sep 17 00:00:00 2001 From: pennae Date: Fri, 22 Dec 2023 18:19:53 +0100 Subject: [PATCH 878/909] reduce the size of Env by one pointer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit since `up` and `values` are both pointer-aligned the type field will also be pointer-aligned, wasting 48 bits of space on most machines. we can get away with removing the type field altogether by encoding some information into the `with` expr that created the env to begin with, reducing the GC load for the absolutely massive amount of single-entry envs we create for lambdas. this reduces memory usage of system eval by quite a bit (reducing heap size of our system eval from 8.4GB to 8.23GB) and gives similar savings in eval time. running `nix eval --raw --impure --expr 'with import {}; system'` before: Time (mean ± σ): 5.576 s ± 0.003 s [User: 5.197 s, System: 0.378 s] Range (min … max): 5.572 s … 5.581 s 10 runs after: Time (mean ± σ): 5.408 s ± 0.002 s [User: 5.019 s, System: 0.388 s] Range (min … max): 5.405 s … 5.411 s 10 runs --- doc/manual/rl-next/env-size-reduction.md | 7 +++++ doc/manual/rl-next/with-error-reporting.md | 31 ++++++++++++++++++++++ src/libcmd/repl.cc | 2 +- src/libexpr/eval-inline.hh | 2 -- src/libexpr/eval.cc | 29 ++++++++++---------- src/libexpr/eval.hh | 5 ---- src/libexpr/nixexpr.cc | 18 ++++++++----- src/libexpr/nixexpr.hh | 13 ++++++--- src/libexpr/primops.cc | 2 +- 9 files changed, 75 insertions(+), 34 deletions(-) create mode 100644 doc/manual/rl-next/env-size-reduction.md create mode 100644 doc/manual/rl-next/with-error-reporting.md diff --git a/doc/manual/rl-next/env-size-reduction.md b/doc/manual/rl-next/env-size-reduction.md new file mode 100644 index 000000000..40a58bc28 --- /dev/null +++ b/doc/manual/rl-next/env-size-reduction.md @@ -0,0 +1,7 @@ +--- +synopsis: Reduce eval memory usage and wall time +prs: 9658 +--- + +Reduce the size of the `Env` struct used in the evaluator by a pointer, or 8 bytes on most modern machines. +This reduces memory usage during eval by around 2% and wall time by around 3%. diff --git a/doc/manual/rl-next/with-error-reporting.md b/doc/manual/rl-next/with-error-reporting.md new file mode 100644 index 000000000..10b020956 --- /dev/null +++ b/doc/manual/rl-next/with-error-reporting.md @@ -0,0 +1,31 @@ +--- +synopsis: Better error reporting for `with` expressions +prs: 9658 +--- + +`with` expressions using non-attrset values to resolve variables are now reported with proper positions. + +Previously an incorrect `with` expression would report no position at all, making it hard to determine where the error originated: + +``` +nix-repl> with 1; a +error: + … + + at «none»:0: (source not available) + + error: value is an integer while a set was expected +``` + +Now position information is preserved and reported as with most other errors: + +``` +nix-repl> with 1; a +error: + … while evaluating the first subexpression of a with expression + at «string»:1:1: + 1| with 1; a + | ^ + + error: value is an integer while a set was expected +``` diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 97d709ff4..dea91ba63 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -112,7 +112,7 @@ NixRepl::NixRepl(const SearchPath & searchPath, nix::ref store, refstaticBaseEnv.get())) + , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get())) , historyFile(getDataDir() + "/nix/repl-history") { } diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 52aa75b5f..f7710f819 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -73,8 +73,6 @@ Env & EvalState::allocEnv(size_t size) #endif env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *)); - env->type = Env::Plain; - /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */ return *env; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 810843995..ee1a87d9a 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -543,7 +543,7 @@ EvalState::EvalState( , env1AllocCache(std::allocate_shared(traceable_allocator(), nullptr)) #endif , baseEnv(allocEnv(128)) - , staticBaseEnv{std::make_shared(false, nullptr)} + , staticBaseEnv{std::make_shared(nullptr, nullptr)} { corepkgsFS->setPathDisplay(""); internalFS->setPathDisplay("«nix-internal»", ""); @@ -781,7 +781,7 @@ void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se) // just for the current level of Env, not the whole chain. void printWithBindings(const SymbolTable & st, const Env & env) { - if (env.type == Env::HasWithAttrs) { + if (!env.values[0]->isThunk()) { std::cout << "with: "; std::cout << ANSI_MAGENTA; Bindings::iterator j = env.values[0]->attrs->begin(); @@ -835,7 +835,7 @@ void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const En if (env.up && se.up) { mapStaticEnvBindings(st, *se.up, *env.up, vm); - if (env.type == Env::HasWithAttrs) { + if (!env.values[0]->isThunk()) { // add 'with' bindings. Bindings::iterator j = env.values[0]->attrs->begin(); while (j != env.values[0]->attrs->end()) { @@ -973,22 +973,23 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) if (!var.fromWith) return env->values[var.displ]; + // This early exit defeats the `maybeThunk` optimization for variables from `with`, + // The added complexity of handling this appears to be similarly in cost, or + // the cases where applicable were insignificant in the first place. + if (noEval) return nullptr; + + auto * fromWith = var.fromWith; while (1) { - if (env->type == Env::HasWithExpr) { - if (noEval) return 0; - Value * v = allocValue(); - evalAttrs(*env->up, (Expr *) env->values[0], *v, noPos, ""); - env->values[0] = v; - env->type = Env::HasWithAttrs; - } + forceAttrs(*env->values[0], fromWith->pos, "while evaluating the first subexpression of a with expression"); Bindings::iterator j = env->values[0]->attrs->find(var.name); if (j != env->values[0]->attrs->end()) { if (countCalls) attrSelects[j->pos]++; return j->value; } - if (!env->prevWith) + if (!fromWith->parentWith) error("undefined variable '%1%'", symbols[var.name]).atPos(var.pos).withFrame(*env, var).debugThrow(); - for (size_t l = env->prevWith; l; --l, env = env->up) ; + for (size_t l = fromWith->prevWith; l; --l, env = env->up) ; + fromWith = fromWith->parentWith; } } @@ -1816,9 +1817,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v) { Env & env2(state.allocEnv(1)); env2.up = &env; - env2.prevWith = prevWith; - env2.type = Env::HasWithExpr; - env2.values[0] = (Value *) attrs; + env2.values[0] = attrs->maybeThunk(state, env); body->eval(state, env2, v); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index da2d256db..db606ebae 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -116,11 +116,6 @@ struct Constant struct Env { Env * up; - /** - * Number of of levels up to next `with` environment - */ - unsigned short prevWith:14; - enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2; Value * values[0]; }; diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 84860b30f..ede070cff 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -333,6 +333,8 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr & if (es.debugRepl) es.exprEnvs.insert(std::make_pair(this, env)); + fromWith = nullptr; + /* Check whether the variable appears in the environment. If so, set its level and displacement. */ const StaticEnv * curEnv; @@ -344,7 +346,6 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr & } else { auto i = curEnv->find(name); if (i != curEnv->vars.end()) { - fromWith = false; this->level = level; displ = i->second; return; @@ -360,7 +361,8 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr & .msg = hintfmt("undefined variable '%1%'", es.symbols[name]), .errPos = es.positions[pos] }); - fromWith = true; + for (auto * e = env.get(); e && !fromWith; e = e->up) + fromWith = e->isWith; this->level = withLevel; } @@ -393,7 +395,7 @@ void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr es.exprEnvs.insert(std::make_pair(this, env)); if (recursive) { - auto newEnv = std::make_shared(false, env.get(), recursive ? attrs.size() : 0); + auto newEnv = std::make_shared(nullptr, env.get(), recursive ? attrs.size() : 0); Displacement displ = 0; for (auto & i : attrs) @@ -435,7 +437,7 @@ void ExprLambda::bindVars(EvalState & es, const std::shared_ptr es.exprEnvs.insert(std::make_pair(this, env)); auto newEnv = std::make_shared( - false, env.get(), + nullptr, env.get(), (hasFormals() ? formals->formals.size() : 0) + (!arg ? 0 : 1)); @@ -471,7 +473,7 @@ void ExprLet::bindVars(EvalState & es, const std::shared_ptr & if (es.debugRepl) es.exprEnvs.insert(std::make_pair(this, env)); - auto newEnv = std::make_shared(false, env.get(), attrs->attrs.size()); + auto newEnv = std::make_shared(nullptr, env.get(), attrs->attrs.size()); Displacement displ = 0; for (auto & i : attrs->attrs) @@ -490,6 +492,10 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr & if (es.debugRepl) es.exprEnvs.insert(std::make_pair(this, env)); + parentWith = nullptr; + for (auto * e = env.get(); e && !parentWith; e = e->up) + parentWith = e->isWith; + /* Does this `with' have an enclosing `with'? If so, record its level so that `lookupVar' can look up variables in the previous `with' if this one doesn't contain the desired attribute. */ @@ -506,7 +512,7 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr & es.exprEnvs.insert(std::make_pair(this, env)); attrs->bindVars(es, env); - auto newEnv = std::make_shared(true, env.get()); + auto newEnv = std::make_shared(this, env.get()); body->bindVars(es, newEnv); } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 1e57fec7a..e50a157ee 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -138,6 +138,7 @@ std::ostream & operator << (std::ostream & str, const Pos & pos); struct Env; struct Value; class EvalState; +struct ExprWith; struct StaticEnv; @@ -226,8 +227,11 @@ struct ExprVar : Expr Symbol name; /* Whether the variable comes from an environment (e.g. a rec, let - or function argument) or from a "with". */ - bool fromWith; + or function argument) or from a "with". + + `nullptr`: Not from a `with`. + Valid pointer: the nearest, innermost `with` expression to query first. */ + ExprWith * fromWith; /* In the former case, the value is obtained by going `level` levels up from the current environment and getting the @@ -385,6 +389,7 @@ struct ExprWith : Expr PosIdx pos; Expr * attrs, * body; size_t prevWith; + ExprWith * parentWith; ExprWith(const PosIdx & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { }; PosIdx getPos() const override { return pos; } COMMON_METHODS @@ -478,14 +483,14 @@ extern ExprBlackHole eBlackHole; runtime. */ struct StaticEnv { - bool isWith; + ExprWith * isWith; const StaticEnv * up; // Note: these must be in sorted order. typedef std::vector> Vars; Vars vars; - StaticEnv(bool isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) { + StaticEnv(ExprWith * isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) { vars.reserve(expectedSize); }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a1502da45..924de3184 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -214,7 +214,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v Env * env = &state.allocEnv(vScope->attrs->size()); env->up = &state.baseEnv; - auto staticEnv = std::make_shared(false, state.staticBaseEnv.get(), vScope->attrs->size()); + auto staticEnv = std::make_shared(nullptr, state.staticBaseEnv.get(), vScope->attrs->size()); unsigned int displ = 0; for (auto & attr : *vScope->attrs) { From 3f796514b37a1e723a395fce8271428410e93f5f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Jan 2024 12:39:16 +0100 Subject: [PATCH 879/909] Optimize empty list constants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This avoids a Value allocation for empty list constants. During a `nix search nixpkgs`, about 82% of all thunked lists are empty, so this removes about 3 million Value allocations. Performance comparison on `nix search github:NixOS/nixpkgs/e1fa12d4f6c6fe19ccb59cac54b5b3f25e160870 --no-eval-cache`: maximum RSS: median = 3845432.0000 mean = 3845432.0000 stddev = 0.0000 min = 3845432.0000 max = 3845432.0000 [rejected?, p=0.00000, Δ=-70084.00000±0.00000] soft page faults: median = 965395.0000 mean = 965394.6667 stddev = 1.1181 min = 965392.0000 max = 965396.0000 [rejected?, p=0.00000, Δ=-17929.77778±38.59610] system CPU time: median = 1.8029 mean = 1.7702 stddev = 0.0621 min = 1.6749 max = 1.8417 [rejected, p=0.00064, Δ=-0.12873±0.09905] user CPU time: median = 14.1022 mean = 14.0633 stddev = 0.1869 min = 13.8118 max = 14.3190 [not rejected, p=0.03006, Δ=-0.18248±0.24928] elapsed time: median = 15.8205 mean = 15.8618 stddev = 0.2312 min = 15.5033 max = 16.1670 [not rejected, p=0.00558, Δ=-0.28963±0.29434] --- src/libexpr/eval.cc | 11 +++++++++++ src/libexpr/eval.hh | 3 +++ src/libexpr/nixexpr.hh | 1 + 3 files changed, 15 insertions(+) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 810843995..494b8338f 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -554,6 +554,8 @@ EvalState::EvalState( static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes"); + vEmptyList.mkList(0); + /* Initialise the Nix expression search path. */ if (!evalSettings.pureEval) { for (auto & i : _searchPath.elements) @@ -1384,6 +1386,15 @@ void ExprList::eval(EvalState & state, Env & env, Value & v) } +Value * ExprList::maybeThunk(EvalState & state, Env & env) +{ + if (elems.empty()) { + return &state.vEmptyList; + } + return Expr::maybeThunk(state, env); +} + + void ExprVar::eval(EvalState & state, Env & env, Value & v) { Value * v2 = state.lookupVar(&env, *this, false); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index da2d256db..bf85b50c8 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -305,6 +305,9 @@ public: return *errorBuilder; } + /* Empty list constant. */ + Value vEmptyList; + private: /* Cache for calls to addToStore(); maps source paths to the store diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 1e57fec7a..55e930758 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -299,6 +299,7 @@ struct ExprList : Expr std::vector elems; ExprList() { }; COMMON_METHODS + Value * maybeThunk(EvalState & state, Env & env) override; PosIdx getPos() const override { From 2b20f36f9515882589975d14a94ba1fd2b5c513a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 2 Jan 2024 12:33:51 -0500 Subject: [PATCH 880/909] Fix NetBSD build There was still a mistake after my earlier a7115a47ef0d83ea81b494f6bc5b11d8286e0672 and e13fc0bbdb1e1eefeb33ff4d18310958041b1ad5. This finally gets it right. --- configure.ac | 7 ++++++- src/libstore/globals.hh | 2 ++ src/libstore/posix-fs-canonicalise.cc | 4 ++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 1bc4f17b0..b97e25bbd 100644 --- a/configure.ac +++ b/configure.ac @@ -308,7 +308,12 @@ AC_SUBST(HAVE_SECCOMP, [$have_seccomp]) # Optional dependencies for better normalizing file system data AC_CHECK_HEADERS([sys/xattr.h]) -AC_CHECK_FUNCS([llistxattr lremovexattr]) +AS_IF([test "$ac_cv_header_sys_xattr_h" = "yes"],[ + AC_CHECK_FUNCS([llistxattr lremovexattr]) + AS_IF([test "$ac_cv_func_llistxattr" = "yes" && test "$ac_cv_func_lremovexattr" = "yes"],[ + AC_DEFINE([HAVE_ACL_SUPPORT], [1], [Define if we can manipulate file system Access Control Lists]) + ]) +]) # Look for aws-cpp-sdk-s3. AC_LANG_PUSH(C++) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index b35dc37a1..cf34ae354 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -946,7 +946,9 @@ public: may be useful in certain scenarios (e.g. to spin up containers or set up userspace network interfaces in tests). )"}; +#endif +#if HAVE_ACL_SUPPORT Setting ignoredAcls{ this, {"security.selinux", "system.nfs4_acl", "security.csm"}, "ignored-acls", R"( diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc index 5edda0157..8b29e90d4 100644 --- a/src/libstore/posix-fs-canonicalise.cc +++ b/src/libstore/posix-fs-canonicalise.cc @@ -1,4 +1,4 @@ -#if HAVE_SYS_XATTR_H +#if HAVE_ACL_SUPPORT # include #endif @@ -78,7 +78,7 @@ static void canonicalisePathMetaData_( if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) throw Error("file '%1%' has an unsupported type", path); -#if HAVE_SYS_XATTR_H && HAVE_LLISTXATTR && HAVE_LREMOVEXATTR +#if HAVE_ACL_SUPPORT /* Remove extended attributes / ACLs. */ ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); From 86e924443722a04f7d458594e3332ffaa73edb1d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 2 Jan 2024 12:41:53 -0500 Subject: [PATCH 881/909] Fix `buildNoTest` `checkInputs` is not right for this because we don't just need these deps when `doTest`, we also need them when `installUnitTests`. --- package.nix | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/package.nix b/package.nix index b5ff45083..56276ecc4 100644 --- a/package.nix +++ b/package.nix @@ -214,6 +214,9 @@ in { ] ++ lib.optionals (!stdenv.hostPlatform.isWindows) [ editline lowdown + ] ++ lib.optionals buildUnitTests [ + gtest + rapidcheck ] ++ lib.optional stdenv.isLinux libseccomp ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies @@ -232,11 +235,6 @@ in { dontBuild = !attrs.doBuild; doCheck = attrs.doCheck; - checkInputs = [ - gtest - rapidcheck - ]; - nativeCheckInputs = [ git mercurial From 7b8af5f916a73aa5927b103ff712280023cea840 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 2 Jan 2024 12:50:48 -0500 Subject: [PATCH 882/909] `buildNoTests`: Restore intent The thing we wanted to test was that building Nix without building or running tests, and without depending on libraries only needed by tests, works. But since 6c8f4ef3502aa214557541ec00538e41aeced6e3, we can also install unit tests, and during the conversion to using `package.nix` this started happening more often (they go to a separate output though, so this should be fine). This adds more `... = false` to restore the original intent: don't run unit test or functional tests, and don't install unit tests. --- flake.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index a8fc105e8..9217de9af 100644 --- a/flake.nix +++ b/flake.nix @@ -234,11 +234,11 @@ buildNoGc = forAllSystems (system: self.packages.${system}.nix.overrideAttrs (a: { configureFlags = (a.configureFlags or []) ++ ["--enable-gc=no"];})); buildNoTests = forAllSystems (system: - self.packages.${system}.nix.overrideAttrs (a: { - doCheck = - assert ! a?dontCheck; - false; - }) + self.packages.${system}.nix.override { + doCheck = false; + doInstallCheck = false; + installUnitTests = false; + } ); # Perl bindings for various platforms. From 484881f3021856cd0d0c0cb42d4473b3c7ea0051 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 3 Jan 2024 10:23:27 +0100 Subject: [PATCH 883/909] Move empty list constant --- src/libexpr/eval.hh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index bf85b50c8..e2180f00d 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -218,6 +218,11 @@ public: Bindings emptyBindings; + /** + * Empty list constant. + */ + Value vEmptyList; + /** * The accessor for the root filesystem. */ @@ -305,9 +310,6 @@ public: return *errorBuilder; } - /* Empty list constant. */ - Value vEmptyList; - private: /* Cache for calls to addToStore(); maps source paths to the store From 24e70489e59f9ab75310382dc59df09796ea8df4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 3 Jan 2024 19:14:50 +0100 Subject: [PATCH 884/909] withFramedSink(): Receive interrupts on the stderr thread Otherwise Nix deadlocks when Ctrl-C is received in withFramedSink(): the parent thread will wait forever for the stderr thread to shut down. Fixes the hang reported in https://github.com/NixOS/nix/issues/7245#issuecomment-1770560923. --- src/libstore/remote-store.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f0df646ca..078b9fe00 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -16,6 +16,8 @@ #include "logging.hh" #include "callback.hh" #include "filetransfer.hh" +#include "signals.hh" + #include namespace nix { @@ -1066,6 +1068,7 @@ void RemoteStore::ConnectionHandle::withFramedSink(std::function Date: Wed, 3 Jan 2024 19:30:02 +0100 Subject: [PATCH 885/909] Make some more threads receive interrupts Shouldn't hurt to do this. In particular, this should speed up shutting down the PathSubstitutionGoal thread if it's copying from a remote store. --- src/libstore/build/substitution-goal.cc | 3 +++ src/libutil/thread-pool.cc | 2 ++ 2 files changed, 5 insertions(+) diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 93867007d..c7e8e2825 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -2,6 +2,7 @@ #include "substitution-goal.hh" #include "nar-info.hh" #include "finally.hh" +#include "signals.hh" namespace nix { @@ -217,6 +218,8 @@ void PathSubstitutionGoal::tryToRun() thr = std::thread([this]() { try { + ReceiveInterrupts receiveInterrupts; + /* Wake up the worker loop when we're done. */ Finally updateStats([this]() { outPipe.writeSide.close(); }); diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index c5e735617..9a7dfee56 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -79,6 +79,8 @@ void ThreadPool::process() void ThreadPool::doWork(bool mainThread) { + ReceiveInterrupts receiveInterrupts; + if (!mainThread) interruptCheck = [&]() { return (bool) quit; }; From 12bb8cdd381156456a712e4a5a8af3b6bc852eab Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 3 Jan 2024 15:02:20 -0500 Subject: [PATCH 886/909] Signer infrastructure: Prep for #9076 This sets up infrastructure in libutil to allow for signing other than by a secret key in memory. #9076 uses this to implement remote signing. (Split from that PR to allow reviewing in smaller chunks.) Co-Authored-By: Raito Bezarius --- perl/lib/Nix/Store.xs | 1 - src/libstore/binary-cache-store.cc | 5 +- src/libstore/binary-cache-store.hh | 5 +- src/libstore/globals.cc | 5 -- src/libstore/keys.cc | 31 ++++++++++ src/libstore/keys.hh | 10 +++ src/libstore/local-store.cc | 7 ++- src/libstore/local.mk | 2 +- src/libstore/path-info.cc | 4 +- src/libstore/path-info.hh | 4 +- src/libstore/path.cc | 6 +- src/libstore/realisation.cc | 5 +- src/libstore/realisation.hh | 4 +- src/libstore/store-api.cc | 2 +- src/libutil/hash.cc | 9 +++ src/libutil/hash.hh | 6 +- src/libutil/local.mk | 7 ++- .../signature/local-keys.cc} | 54 +++++++--------- .../signature/local-keys.hh} | 42 +++++++++++-- src/libutil/signature/signer.cc | 23 +++++++ src/libutil/signature/signer.hh | 61 +++++++++++++++++++ src/libutil/util.cc | 4 ++ src/nix/sigs.cc | 5 +- src/nix/verify.cc | 1 + 24 files changed, 233 insertions(+), 70 deletions(-) create mode 100644 src/libstore/keys.cc create mode 100644 src/libstore/keys.hh rename src/{libstore/crypto.cc => libutil/signature/local-keys.cc} (64%) rename src/{libstore/crypto.hh => libutil/signature/local-keys.hh} (51%) create mode 100644 src/libutil/signature/signer.cc create mode 100644 src/libutil/signature/signer.hh diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 4964b8a34..423c01cf7 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -12,7 +12,6 @@ #include "realisation.hh" #include "globals.hh" #include "store-api.hh" -#include "crypto.hh" #include "posix-source-accessor.hh" #include diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 8a3052433..ea1279e2e 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -28,7 +28,8 @@ BinaryCacheStore::BinaryCacheStore(const Params & params) , Store(params) { if (secretKeyFile != "") - secretKey = std::unique_ptr(new SecretKey(readFile(secretKeyFile))); + signer = std::make_unique( + SecretKey { readFile(secretKeyFile) }); StringSink sink; sink << narVersionMagic1; @@ -274,7 +275,7 @@ ref BinaryCacheStore::addToStoreCommon( stats.narWriteCompressionTimeMs += duration; /* Atomically write the NAR info file.*/ - if (secretKey) narInfo->sign(*this, *secretKey); + if (signer) narInfo->sign(*this, *signer); writeNarInfo(narInfo); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 98e43ee6a..00ab73905 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "crypto.hh" +#include "signature/local-keys.hh" #include "store-api.hh" #include "log-store.hh" @@ -57,8 +57,7 @@ class BinaryCacheStore : public virtual BinaryCacheStoreConfig, { private: - - std::unique_ptr secretKey; + std::unique_ptr signer; protected: diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index f401d076d..50584e06c 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -15,8 +15,6 @@ #include -#include - #ifdef __GLIBC__ # include # include @@ -409,9 +407,6 @@ void initLibStore() { initLibUtil(); - if (sodium_init() == -1) - throw Error("could not initialise libsodium"); - loadConfFile(); preloadNSS(); diff --git a/src/libstore/keys.cc b/src/libstore/keys.cc new file mode 100644 index 000000000..2cc50970f --- /dev/null +++ b/src/libstore/keys.cc @@ -0,0 +1,31 @@ +#include "file-system.hh" +#include "globals.hh" +#include "keys.hh" + +namespace nix { + +PublicKeys getDefaultPublicKeys() +{ + PublicKeys publicKeys; + + // FIXME: filter duplicates + + for (auto s : settings.trustedPublicKeys.get()) { + PublicKey key(s); + publicKeys.emplace(key.name, key); + } + + for (auto secretKeyFile : settings.secretKeyFiles.get()) { + try { + SecretKey secretKey(readFile(secretKeyFile)); + publicKeys.emplace(secretKey.name, secretKey.toPublicKey()); + } catch (SysError & e) { + /* Ignore unreadable key files. That's normal in a + multi-user installation. */ + } + } + + return publicKeys; +} + +} diff --git a/src/libstore/keys.hh b/src/libstore/keys.hh new file mode 100644 index 000000000..3da19493f --- /dev/null +++ b/src/libstore/keys.hh @@ -0,0 +1,10 @@ +#pragma once +///@file + +#include "signature/local-keys.hh" + +namespace nix { + +PublicKeys getDefaultPublicKeys(); + +} diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 63e90ea1e..0f3c37c8a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -14,6 +14,7 @@ #include "signals.hh" #include "posix-fs-canonicalise.hh" #include "posix-source-accessor.hh" +#include "keys.hh" #include #include @@ -1578,7 +1579,8 @@ void LocalStore::signRealisation(Realisation & realisation) for (auto & secretKeyFile : secretKeyFiles.get()) { SecretKey secretKey(readFile(secretKeyFile)); - realisation.sign(secretKey); + LocalSigner signer(std::move(secretKey)); + realisation.sign(signer); } } @@ -1590,7 +1592,8 @@ void LocalStore::signPathInfo(ValidPathInfo & info) for (auto & secretKeyFile : secretKeyFiles.get()) { SecretKey secretKey(readFile(secretKeyFile)); - info.sign(*this, secretKey); + LocalSigner signer(std::move(secretKey)); + info.sign(*this, signer); } } diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 68ccdc409..675976314 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -8,7 +8,7 @@ libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc) libstore_LIBS = libutil -libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread +libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) -pthread ifdef HOST_LINUX libstore_LDFLAGS += -ldl endif diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index f58e31bfd..d82ccd0c9 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -38,9 +38,9 @@ std::string ValidPathInfo::fingerprint(const Store & store) const } -void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) +void ValidPathInfo::sign(const Store & store, const Signer & signer) { - sigs.insert(secretKey.signDetached(fingerprint(store))); + sigs.insert(signer.signDetached(fingerprint(store))); } std::optional ValidPathInfo::contentAddressWithReferences() const diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 077abc7e1..b6dc0855d 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "crypto.hh" +#include "signature/signer.hh" #include "path.hh" #include "hash.hh" #include "content-address.hh" @@ -107,7 +107,7 @@ struct ValidPathInfo : UnkeyedValidPathInfo { */ std::string fingerprint(const Store & store) const; - void sign(const Store & store, const SecretKey & secretKey); + void sign(const Store & store, const Signer & signer); /** * @return The `ContentAddressWithReferences` that determines the diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 1afd10af7..a15a78545 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -1,7 +1,5 @@ #include "store-dir-config.hh" -#include - namespace nix { static void checkName(std::string_view path, std::string_view name) @@ -49,9 +47,7 @@ StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x"); StorePath StorePath::random(std::string_view name) { - Hash hash(HashAlgorithm::SHA1); - randombytes_buf(hash.hash, hash.hashSize); - return StorePath(hash, name); + return StorePath(Hash::random(HashAlgorithm::SHA1), name); } StorePath StoreDirConfig::parseStorePath(std::string_view path) const diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 93ddb5b20..86bfdd1a8 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -1,6 +1,7 @@ #include "realisation.hh" #include "store-api.hh" #include "closure.hh" +#include "signature/local-keys.hh" #include namespace nix { @@ -113,9 +114,9 @@ std::string Realisation::fingerprint() const return serialized.dump(); } -void Realisation::sign(const SecretKey & secretKey) +void Realisation::sign(const Signer &signer) { - signatures.insert(secretKey.signDetached(fingerprint())); + signatures.insert(signer.signDetached(fingerprint())); } bool Realisation::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 4ba2123d8..ddb4af770 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -8,7 +8,7 @@ #include "derived-path.hh" #include #include "comparator.hh" -#include "crypto.hh" +#include "signature/signer.hh" namespace nix { @@ -64,7 +64,7 @@ struct Realisation { static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); std::string fingerprint() const; - void sign(const SecretKey &); + void sign(const Signer &); bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; size_t checkSignatures(const PublicKeys & publicKeys) const; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c2516afb5..c48bfc248 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,4 +1,4 @@ -#include "crypto.hh" +#include "signature/local-keys.hh" #include "source-accessor.hh" #include "globals.hh" #include "derived-path.hh" diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 502afbda2..d067da969 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -14,6 +14,8 @@ #include #include +#include + namespace nix { static size_t regularHashSize(HashAlgorithm type) { @@ -261,6 +263,13 @@ Hash::Hash(std::string_view rest, HashAlgorithm algo, bool isSRI) throw BadHash("hash '%s' has wrong length for hash algorithm '%s'", rest, printHashAlgo(this->algo)); } +Hash Hash::random(HashAlgorithm algo) +{ + Hash hash(algo); + randombytes_buf(hash.hash, hash.hashSize); + return hash; +} + Hash newHashAllowEmpty(std::string_view hashStr, std::optional ha) { if (hashStr.empty()) { diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 2fe9a53f5..f7e8eb265 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -5,7 +5,6 @@ #include "serialise.hh" #include "file-system.hh" - namespace nix { @@ -143,6 +142,11 @@ public: } static Hash dummy; + + /** + * @return a random hash with hash algorithm `algo` + */ + static Hash random(HashAlgorithm algo); }; /** diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 81efaafec..0fdebaf5c 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -4,14 +4,17 @@ libutil_NAME = libnixutil libutil_DIR := $(d) -libutil_SOURCES := $(wildcard $(d)/*.cc) +libutil_SOURCES := $(wildcard $(d)/*.cc $(d)/signature/*.cc) libutil_CXXFLAGS += -I src/libutil -libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context +libutil_LDFLAGS += -pthread $(LIBCURL_LIBS) $(SODIUM_LIBS) $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context $(foreach i, $(wildcard $(d)/args/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/args, 0644))) +$(foreach i, $(wildcard $(d)/signature/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/signature, 0644))) + ifeq ($(HAVE_LIBCPUID), 1) libutil_LDFLAGS += -lcpuid diff --git a/src/libstore/crypto.cc b/src/libutil/signature/local-keys.cc similarity index 64% rename from src/libstore/crypto.cc rename to src/libutil/signature/local-keys.cc index 1b705733c..858b036f5 100644 --- a/src/libstore/crypto.cc +++ b/src/libutil/signature/local-keys.cc @@ -1,13 +1,12 @@ -#include "crypto.hh" +#include "signature/local-keys.hh" + #include "file-system.hh" #include "util.hh" -#include "globals.hh" - #include namespace nix { -static std::pair split(std::string_view s) +BorrowedCryptoValue BorrowedCryptoValue::parse(std::string_view s) { size_t colon = s.find(':'); if (colon == std::string::npos || colon == 0) @@ -17,10 +16,10 @@ static std::pair split(std::string_view s) Key::Key(std::string_view s) { - auto ss = split(s); + auto ss = BorrowedCryptoValue::parse(s); - name = ss.first; - key = ss.second; + name = ss.name; + key = ss.payload; if (name == "" || key == "") throw Error("secret key is corrupt"); @@ -73,45 +72,34 @@ PublicKey::PublicKey(std::string_view s) throw Error("public key is not valid"); } -bool verifyDetached(const std::string & data, const std::string & sig, - const PublicKeys & publicKeys) +bool PublicKey::verifyDetached(std::string_view data, std::string_view sig) const { - auto ss = split(sig); + auto ss = BorrowedCryptoValue::parse(sig); - auto key = publicKeys.find(std::string(ss.first)); - if (key == publicKeys.end()) return false; + if (ss.name != std::string_view { name }) return false; - auto sig2 = base64Decode(ss.second); + return verifyDetachedAnon(data, ss.payload); +} + +bool PublicKey::verifyDetachedAnon(std::string_view data, std::string_view sig) const +{ + auto sig2 = base64Decode(sig); if (sig2.size() != crypto_sign_BYTES) throw Error("signature is not valid"); return crypto_sign_verify_detached((unsigned char *) sig2.data(), (unsigned char *) data.data(), data.size(), - (unsigned char *) key->second.key.data()) == 0; + (unsigned char *) key.data()) == 0; } -PublicKeys getDefaultPublicKeys() +bool verifyDetached(std::string_view data, std::string_view sig, const PublicKeys & publicKeys) { - PublicKeys publicKeys; + auto ss = BorrowedCryptoValue::parse(sig); - // FIXME: filter duplicates + auto key = publicKeys.find(std::string(ss.name)); + if (key == publicKeys.end()) return false; - for (auto s : settings.trustedPublicKeys.get()) { - PublicKey key(s); - publicKeys.emplace(key.name, key); - } - - for (auto secretKeyFile : settings.secretKeyFiles.get()) { - try { - SecretKey secretKey(readFile(secretKeyFile)); - publicKeys.emplace(secretKey.name, secretKey.toPublicKey()); - } catch (SysError & e) { - /* Ignore unreadable key files. That's normal in a - multi-user installation. */ - } - } - - return publicKeys; + return key->second.verifyDetachedAnon(data, ss.payload); } } diff --git a/src/libstore/crypto.hh b/src/libutil/signature/local-keys.hh similarity index 51% rename from src/libstore/crypto.hh rename to src/libutil/signature/local-keys.hh index 35216d470..4aafc1239 100644 --- a/src/libstore/crypto.hh +++ b/src/libutil/signature/local-keys.hh @@ -7,6 +7,25 @@ namespace nix { +/** + * Except where otherwise noted, Nix serializes keys and signatures in + * the form: + * + * ``` + * : + * ``` + */ +struct BorrowedCryptoValue { + std::string_view name; + std::string_view payload; + + /** + * This splits on the colon, the user can then separated decode the + * Base64 payload separately. + */ + static BorrowedCryptoValue parse(std::string_view); +}; + struct Key { std::string name; @@ -49,21 +68,36 @@ struct PublicKey : Key { PublicKey(std::string_view data); + /** + * @return true iff `sig` and this key's names match, and `sig` is a + * correct signature over `data` using the given public key. + */ + bool verifyDetached(std::string_view data, std::string_view sigs) const; + + /** + * @return true iff `sig` is a correct signature over `data` using the + * given public key. + * + * @param just the Base64 signature itself, not a colon-separated pair of a + * public key name and signature. + */ + bool verifyDetachedAnon(std::string_view data, std::string_view sigs) const; + private: PublicKey(std::string_view name, std::string && key) : Key(name, std::move(key)) { } friend struct SecretKey; }; +/** + * Map from key names to public keys + */ typedef std::map PublicKeys; /** * @return true iff ‘sig’ is a correct signature over ‘data’ using one * of the given public keys. */ -bool verifyDetached(const std::string & data, const std::string & sig, - const PublicKeys & publicKeys); - -PublicKeys getDefaultPublicKeys(); +bool verifyDetached(std::string_view data, std::string_view sig, const PublicKeys & publicKeys); } diff --git a/src/libutil/signature/signer.cc b/src/libutil/signature/signer.cc new file mode 100644 index 000000000..0d26867b5 --- /dev/null +++ b/src/libutil/signature/signer.cc @@ -0,0 +1,23 @@ +#include "signature/signer.hh" +#include "error.hh" + +#include + +namespace nix { + +LocalSigner::LocalSigner(SecretKey && privateKey) + : privateKey(privateKey) + , publicKey(privateKey.toPublicKey()) +{ } + +std::string LocalSigner::signDetached(std::string_view s) const +{ + return privateKey.signDetached(s); +} + +const PublicKey & LocalSigner::getPublicKey() +{ + return publicKey; +} + +} diff --git a/src/libutil/signature/signer.hh b/src/libutil/signature/signer.hh new file mode 100644 index 000000000..e50170fe2 --- /dev/null +++ b/src/libutil/signature/signer.hh @@ -0,0 +1,61 @@ +#pragma once + +#include "types.hh" +#include "signature/local-keys.hh" + +#include +#include + +namespace nix { + +/** + * An abstract signer + * + * Derive from this class to implement a custom signature scheme. + * + * It is only necessary to implement signature of bytes and provide a + * public key. + */ +struct Signer +{ + virtual ~Signer() = default; + + /** + * Sign the given data, creating a (detached) signature. + * + * @param data data to be signed. + * + * @return the [detached + * signature](https://en.wikipedia.org/wiki/Detached_signature), + * i.e. just the signature itself without a copy of the signed data. + */ + virtual std::string signDetached(std::string_view data) const = 0; + + /** + * View the public key associated with this `Signer`. + */ + virtual const PublicKey & getPublicKey() = 0; +}; + +using Signers = std::map; + +/** + * Local signer + * + * The private key is held in this machine's RAM + */ +struct LocalSigner : Signer +{ + LocalSigner(SecretKey && privateKey); + + std::string signDetached(std::string_view s) const override; + + const PublicKey & getPublicKey() override; + +private: + + SecretKey privateKey; + PublicKey publicKey; +}; + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 5bb3f374b..7b4b1d031 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -7,6 +7,7 @@ #include #include +#include namespace nix { @@ -28,6 +29,9 @@ void initLibUtil() { } // This is not actually the main point of this check, but let's make sure anyway: assert(caught); + + if (sodium_init() == -1) + throw Error("could not initialise libsodium"); } ////////////////////////////////////////////////////////////////////// diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index a57a407e6..dfef44869 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -112,7 +112,7 @@ struct CmdSign : StorePathsCommand std::string description() override { - return "sign store paths"; + return "sign store paths with a local key"; } void run(ref store, StorePaths && storePaths) override @@ -121,6 +121,7 @@ struct CmdSign : StorePathsCommand throw UsageError("you must specify a secret key file using '-k'"); SecretKey secretKey(readFile(secretKeyFile)); + LocalSigner signer(std::move(secretKey)); size_t added{0}; @@ -129,7 +130,7 @@ struct CmdSign : StorePathsCommand auto info2(*info); info2.sigs.clear(); - info2.sign(*store, secretKey); + info2.sign(*store, signer); assert(!info2.sigs.empty()); if (!info->sigs.count(*info2.sigs.begin())) { diff --git a/src/nix/verify.cc b/src/nix/verify.cc index f0234f7be..2a0cbd19f 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -5,6 +5,7 @@ #include "thread-pool.hh" #include "references.hh" #include "signals.hh" +#include "keys.hh" #include From 37ea1612c78b88884f7baecbb1bf81e65e571592 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 3 Jan 2024 19:38:22 -0500 Subject: [PATCH 887/909] flake: Go back to regular `nixos-23.05-small` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finally get off the ad-hoc staging commit! Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/36c4ac09e9bebcec1fa7b7539cddb0c9e837409c' (2023-11-30) → 'github:NixOS/nixpkgs/2c9c58e98243930f8cb70387934daa4bc8b00373' (2023-12-31) --- flake.lock | 8 ++++---- flake.nix | 12 +----------- tests/nixos/default.nix | 1 - 3 files changed, 5 insertions(+), 16 deletions(-) diff --git a/flake.lock b/flake.lock index db1a72c14..ae98d789a 100644 --- a/flake.lock +++ b/flake.lock @@ -34,16 +34,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1701355166, - "narHash": "sha256-4V7XMI0Gd+y0zsi++cEHd99u3GNL0xSTGRmiWKzGnUQ=", + "lastModified": 1704018918, + "narHash": "sha256-erjg/HrpC9liEfm7oLqb8GXCqsxaFwIIPqCsknW5aFY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "36c4ac09e9bebcec1fa7b7539cddb0c9e837409c", + "rev": "2c9c58e98243930f8cb70387934daa4bc8b00373", "type": "github" }, "original": { "owner": "NixOS", - "ref": "staging-23.05", + "ref": "nixos-23.05-small", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 9217de9af..e6a88af9f 100644 --- a/flake.nix +++ b/flake.nix @@ -1,17 +1,7 @@ { description = "The purely functional package manager"; - # TODO Go back to nixos-23.05-small once - # https://github.com/NixOS/nixpkgs/pull/271202 is merged. - # - # Also, do not grab arbitrary further staging commits. This PR was - # carefully made to be based on release-23.05 and just contain - # rebuild-causing changes to packages that Nix actually uses. - # - # Once this is updated to something containing - # https://github.com/NixOS/nixpkgs/pull/271423, don't forget - # to remove the `nix.checkAllErrors = false;` line in the tests. - inputs.nixpkgs.url = "github:NixOS/nixpkgs/staging-23.05"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 2645cac8e..4459aa664 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -10,7 +10,6 @@ let hostPkgs = nixpkgsFor.${system}.native; defaults = { nixpkgs.pkgs = nixpkgsFor.${system}.native; - nix.checkAllErrors = false; }; _module.args.nixpkgs = nixpkgs; }; From d8a2b06e2068b5209264dfc6d74d5cadf88b8684 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Thu, 4 Jan 2024 11:31:09 -0800 Subject: [PATCH 888/909] Remove `clang11Stdenv` Clang 11 doesn't have support for three-way-comparisons (<=>, "spaceship operator", "consistent comparisons") and is older than `clangStdenv`. `clangStdenv` is currently 12 on FreeBSD and Android and 16 on other platforms: https://github.com/NixOS/nixpkgs/blob/32e718f00c26c811be0062dd0777066f02406940/pkgs/top-level/all-packages.nix#L16629-L16644 Let's start by removing Clang 11 from our distribution. Next we can consider upgrading to Clang 17, which fully supports the spaceship operator: https://releases.llvm.org/17.0.1/tools/clang/docs/ReleaseNotes.html#what-s-new-in-clang-release --- doc/manual/src/contributing/hacking.md | 4 ++-- flake.nix | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index dce0422dc..9a03ac9b6 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -31,7 +31,7 @@ This shell also adds `./outputs/bin/nix` to your `$PATH` so you can run `nix` im To get a shell with one of the other [supported compilation environments](#compilation-environments): ```console -$ nix develop .#native-clang11StdenvPackages +$ nix develop .#native-clangStdenvPackages ``` > **Note** @@ -96,7 +96,7 @@ $ nix-shell To get a shell with one of the other [supported compilation environments](#compilation-environments): ```console -$ nix-shell --attr devShells.x86_64-linux.native-clang11StdenvPackages +$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages ``` > **Note** diff --git a/flake.nix b/flake.nix index e6a88af9f..32354a88f 100644 --- a/flake.nix +++ b/flake.nix @@ -52,7 +52,6 @@ stdenvs = [ "ccacheStdenv" - "clang11Stdenv" "clangStdenv" "gccStdenv" "libcxxStdenv" From 388c79d546db0a2e636aa56e4d4b9a5dfde50db5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Jan 2024 15:15:25 +0100 Subject: [PATCH 889/909] Don't pull in libboost_regex We're not using and we don't want to pull in libicu (37 MiB). --- package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.nix b/package.nix index 56276ecc4..dfebdb0e4 100644 --- a/package.nix +++ b/package.nix @@ -248,7 +248,7 @@ in { # Copy libboost_context so we don't get all of Boost in our closure. # https://github.com/NixOS/nixpkgs/issues/45462 mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib rm -f $out/lib/*.a '' + lib.optionalString stdenv.hostPlatform.isLinux '' chmod u+w $out/lib/*.so.* From a4d33e816ef6c5baaed4eb65e826cd5aa75c0343 Mon Sep 17 00:00:00 2001 From: wiki-me <68199012+wiki-me@users.noreply.github.com> Date: Sat, 6 Jan 2024 20:01:10 +0200 Subject: [PATCH 890/909] Improve documentation around upgrading nix (#9679) * Improve documentation around upgrading nix, add replacing nix channel with new one Co-authored-by: Valentin Gagarin --- doc/manual/src/installation/upgrading.md | 35 ++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/doc/manual/src/installation/upgrading.md b/doc/manual/src/installation/upgrading.md index 6d09f54d8..d1b64b80b 100644 --- a/doc/manual/src/installation/upgrading.md +++ b/doc/manual/src/installation/upgrading.md @@ -1,5 +1,40 @@ # Upgrading Nix +> **Note** +> +> These upgrade instructions apply for regular Linux distributions where Nix was installed following the [installation instructions in this manual](./index.md). + +First, find the name of the current [channel](@docroot@/command-ref/nix-channel.md) through which Nix is distributed: + +```console +$ nix-channel --list +``` + +By default this should return an entry for Nixpkgs: + +```console +nixpkgs https://nixos.org/channels/nixpkgs-23.05 +``` + +Check which Nix version will be installed: + +```console +$ nix-shell -p nix -I nixpkgs=channel:nixpkgs-23.11 --run "nix --version" +nix (Nix) 2.18.1 +``` + +> **Warning** +> +> Writing to the [local store](@docroot@/store/types/local-store.md) with a newer version of Nix, for example by building derivations with `nix-build` or `nix-store --realise`, may change the database schema! +> Reverting to an older version of Nix may therefore require purging the store database before it can be used. + +Update the channel entry: + +```console +$ nix-channel --remove nixpkgs +$ nix-channel --add https://nixos.org/channels/nixpkgs-23.11 nixpkgs +``` + Multi-user Nix users on macOS can upgrade Nix by running: `sudo -i sh -c 'nix-channel --update && nix-env --install --attr nixpkgs.nix && From 8e865f3aba526394ca333efe7258bd8db0050fbb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 6 Jan 2024 22:45:25 +0100 Subject: [PATCH 891/909] deduplicate installation instructions (#9507) * deduplicate installation instructions - reorder sections to present pinned installation more prominently - remove outdated notes on the macOS installer rework - update instructions to handle the installer tarball Co-authored-by: Travis A. Everett --- .../src/installation/installing-binary.md | 162 +++++++++--------- doc/manual/src/quick-start.md | 1 - 2 files changed, 77 insertions(+), 86 deletions(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index ffabb250a..0dc989159 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -1,26 +1,60 @@ # Installing a Binary Distribution -The easiest way to install Nix is to run the following command: +To install the latest version Nix, run the following command: ```console $ curl -L https://nixos.org/nix/install | sh ``` -This will run the installer interactively (causing it to explain what -it is doing more explicitly), and perform the default "type" of install -for your platform: -- single-user on Linux -- multi-user on macOS +This performs the default type of installation for your platform: - > **Notes on read-only filesystem root in macOS 10.15 Catalina +** - > - > - It took some time to support this cleanly. You may see posts, - > examples, and tutorials using obsolete workarounds. - > - Supporting it cleanly made macOS installs too complex to qualify - > as single-user, so this type is no longer supported on macOS. +- [Multi-user](#multi-user-installation): + - Linux with systemd and without SELinux + - macOS +- [Single-user](#single-user-installation): + - Linux without systemd + - Linux with SELinux -We recommend the multi-user install if it supports your platform and -you can authenticate with `sudo`. +We recommend the multi-user installation if it supports your platform and you can authenticate with `sudo`. + +The installer can configured with various command line arguments and environment variables. +To show available command line flags: + +```console +$ curl -L https://nixos.org/nix/install | sh -s -- --help +``` + +To check what it does and how it can be customised further, [download and edit the second-stage installation script](#installing-from-a-binary-tarball). + +# Installing a pinned Nix version from a URL + +Version-specific installation URLs for all Nix versions since 1.11.16 can be found at [releases.nixos.org](https://releases.nixos.org/?prefix=nix/). +The directory for each version contains the corresponding SHA-256 hash. + +All installation scripts are invoked the same way: + +```console +$ export VERSION=2.19.2 +$ curl -L https://releases.nixos.org/nix/nix-$VERSION/install | sh +``` + +# Multi User Installation + +The multi-user Nix installation creates system users and a system service for the Nix daemon. + +Supported systems: + +- Linux running systemd, with SELinux disabled +- macOS + +To explicitly instruct the installer to perform a multi-user installation on your system: + +```console +$ curl -L https://nixos.org/nix/install | sh -s -- --daemon +``` + +You can run this under your usual user account or `root`. +The script will invoke `sudo` as needed. # Single User Installation @@ -30,60 +64,48 @@ To explicitly select a single-user installation on your system: $ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon ``` -This will perform a single-user installation of Nix, meaning that `/nix` -is owned by the invoking user. You can run this under your usual user -account or root. The script will invoke `sudo` to create `/nix` -if it doesn’t already exist. If you don’t have `sudo`, you should -manually create `/nix` first as root, e.g.: +In a single-user installation, `/nix` is owned by the invoking user. +The script will invoke `sudo` to create `/nix` if it doesn’t already exist. +If you don’t have `sudo`, manually create `/nix` as `root`: ```console -$ mkdir /nix -$ chown alice /nix +$ su root +# mkdir /nix +# chown alice /nix ``` -The install script will modify the first writable file from amongst -`.bash_profile`, `.bash_login` and `.profile` to source -`~/.nix-profile/etc/profile.d/nix.sh`. You can set the -`NIX_INSTALLER_NO_MODIFY_PROFILE` environment variable before executing -the install script to disable this behaviour. +# Installing from a binary tarball -# Multi User Installation +You can also download a binary tarball that contains Nix and all its dependencies: +- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../contributing/hacking.md#platforms) +- Download and unpack the tarball +- Run the installer -The multi-user Nix installation creates system users, and a system -service for the Nix daemon. - -**Supported Systems** -- Linux running systemd, with SELinux disabled -- macOS - -You can instruct the installer to perform a multi-user installation on -your system: - -```console -$ curl -L https://nixos.org/nix/install | sh -s -- --daemon -``` - -The multi-user installation of Nix will create build users between the -user IDs 30001 and 30032, and a group with the group ID 30000. You -can run this under your usual user account or root. The script -will invoke `sudo` as needed. - -> **Note** +> **Example** > -> If you need Nix to use a different group ID or user ID set, you will -> have to download the tarball manually and [edit the install -> script](#installing-from-a-binary-tarball). +> ```console +> $ pushd $(mktemp -d) +> $ export VERSION=2.19.2 +> $ export SYSTEM=x86_64-linux +> $ curl -LO https://releases.nixos.org/nix/nix-$VERSION/nix-$VERSION-$SYSTEM.tar.xz +> $ tar xfj nix-$VERSION-$SYSTEM.tar.xz +> $ cd nix-$VERSION-$SYSTEM +> $ ./install +> $ popd +> ``` -The installer will modify `/etc/bashrc`, and `/etc/zshrc` if they exist. -The installer will first back up these files with a `.backup-before-nix` -extension. The installer will also create `/etc/profile.d/nix.sh`. +The installer can be customised with the environment variables declared in the file named `install-multi-user`. + +## Native packages for Linux distributions + +The Nix community maintains installers for some Linux distributions in their native packaging format(https://nix-community.github.io/nix-installers/). # macOS Installation + []{#sect-macos-installation-change-store-prefix}[]{#sect-macos-installation-encrypted-volume}[]{#sect-macos-installation-symlink}[]{#sect-macos-installation-recommended-notes} - -We believe we have ironed out how to cleanly support the read-only root +We believe we have ironed out how to cleanly support the read-only root file system on modern macOS. New installs will do this automatically. This section previously detailed the situation, options, and trade-offs, @@ -126,33 +148,3 @@ this to run the installer, but it may help if you run into trouble: boot process to avoid problems loading or restoring any programs that need access to your Nix store -# Installing a pinned Nix version from a URL - -Version-specific installation URLs for all Nix versions -since 1.11.16 can be found at [releases.nixos.org](https://releases.nixos.org/?prefix=nix/). -The corresponding SHA-256 hash can be found in the directory for the given version. - -These install scripts can be used the same as usual: - -```console -$ curl -L https://releases.nixos.org/nix/nix-/install | sh -``` - -# Installing from a binary tarball - -You can also download a binary tarball that contains Nix and all its -dependencies. (This is what the install script at - does automatically.) You should unpack -it somewhere (e.g. in `/tmp`), and then run the script named `install` -inside the binary tarball: - -```console -$ cd /tmp -$ tar xfj nix-1.8-x86_64-darwin.tar.bz2 -$ cd nix-1.8-x86_64-darwin -$ ./install -``` - -If you need to edit the multi-user installation script to use different -group ID or a different user ID range, modify the variables set in the -file named `install-multi-user`. diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 04a0b7c96..75853ced7 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -10,7 +10,6 @@ For more in-depth information you are kindly referred to subsequent chapters. ``` The install script will use `sudo`, so make sure you have sufficient rights. - On Linux, `--daemon` can be omitted for a single-user install. For other installation methods, see the detailed [installation instructions](installation/index.md). From eeb2f083c5646bd3a66344cff69be586fd89a450 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Sun, 24 Dec 2023 06:44:56 -0500 Subject: [PATCH 892/909] Improve error message for fixed-outputs with references. This codepath is possible, e.g. with a dockerTools.pullImage of an image with a Nix store. --- src/libstore/store-api.cc | 5 ++++- tests/functional/fixed.nix | 9 +++++++++ tests/functional/fixed.sh | 3 +++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c2516afb5..ad6e1cc0f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -194,7 +194,10 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { - assert(info.references.size() == 0); + if (!info.references.empty()) { + throw Error("fixed output derivation '%s' is not allowed to refer to other store paths.\nYou may need to use the 'unsafeDiscardReferences' derivation attribute, see the manual for more details.", + name); + } return makeStorePath("output:out", hashString(HashAlgorithm::SHA256, "fixed:out:" diff --git a/tests/functional/fixed.nix b/tests/functional/fixed.nix index babe71504..5bdf79333 100644 --- a/tests/functional/fixed.nix +++ b/tests/functional/fixed.nix @@ -48,6 +48,15 @@ rec { (f ./fixed.builder1.sh "flat" "md5" "ddd8be4b179a529afa5f2ffae4b9858") ]; + badReferences = mkDerivation rec { + name = "bad-hash"; + builder = script; + script = builtins.toFile "installer.sh" "echo $script >$out"; + outputHash = "1ixr6yd3297ciyp9im522dfxpqbkhcw0pylkb2aab915278fqaik"; + outputHashAlgo = "sha256"; + outputHashMode = "flat"; + }; + # Test for building two derivations in parallel that produce the # same output path because they're fixed-output derivations. parallelSame = [ diff --git a/tests/functional/fixed.sh b/tests/functional/fixed.sh index f1e1ce420..2405d059c 100644 --- a/tests/functional/fixed.sh +++ b/tests/functional/fixed.sh @@ -26,6 +26,9 @@ nix-build fixed.nix -A good2 --no-out-link echo 'testing reallyBad...' nix-instantiate fixed.nix -A reallyBad && fail "should fail" +echo 'testing fixed with references...' +expectStderr 1 nix-build fixed.nix -A badReferences | grepQuiet "not allowed to refer to other store paths" + # While we're at it, check attribute selection a bit more. echo 'testing attribute selection...' test $(nix-instantiate fixed.nix -A good.1 | wc -l) = 1 From c4c636284e4b7b057788383068967910c5a31856 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 Jan 2024 10:17:28 -0500 Subject: [PATCH 893/909] Only test bug fix with new enough deamon --- tests/functional/fixed.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/functional/fixed.sh b/tests/functional/fixed.sh index 2405d059c..d98d4cd15 100644 --- a/tests/functional/fixed.sh +++ b/tests/functional/fixed.sh @@ -26,8 +26,10 @@ nix-build fixed.nix -A good2 --no-out-link echo 'testing reallyBad...' nix-instantiate fixed.nix -A reallyBad && fail "should fail" -echo 'testing fixed with references...' -expectStderr 1 nix-build fixed.nix -A badReferences | grepQuiet "not allowed to refer to other store paths" +if isDaemonNewer "2.20pre20240108"; then + echo 'testing fixed with references...' + expectStderr 1 nix-build fixed.nix -A badReferences | grepQuiet "not allowed to refer to other store paths" +fi # While we're at it, check attribute selection a bit more. echo 'testing attribute selection...' From 605eba3829946eb04f1aaf1160cf11a55183c677 Mon Sep 17 00:00:00 2001 From: Weijia Wang <9713184+wegank@users.noreply.github.com> Date: Mon, 8 Jan 2024 17:31:27 +0100 Subject: [PATCH 894/909] Fix typo in configure.ac --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index b97e25bbd..369d62552 100644 --- a/configure.ac +++ b/configure.ac @@ -160,7 +160,7 @@ AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation AC_SUBST(ENABLE_DOC_GEN) AS_IF( - [test "$ENABLE_BUILD" == "no" && test "$ENABLE_GENERATED_DOCS" == "yes"], + [test "$ENABLE_BUILD" == "no" && test "$ENABLE_DOC_GEN" == "yes"], [AC_MSG_ERROR([Cannot enable generated docs when building overall is disabled. Please do not pass '--enable-doc-gen' or do not pass '--disable-build'.])]) # Building without API docs is the default as Nix' C++ interfaces are internal and unstable. From 6a243e5ed281344135285d9093ef36969a867d73 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 8 Jan 2024 19:38:36 +0100 Subject: [PATCH 895/909] fix an old lost direct (#9458) this part must have been moved quite a while ago, but apparently so far no one noticed --- doc/manual/redirects.js | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 3b507adf3..d04f32b49 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -21,6 +21,7 @@ const redirects = { "chap-distributed-builds": "advanced-topics/distributed-builds.html", "chap-post-build-hook": "advanced-topics/post-build-hook.html", "chap-post-build-hook-caveats": "advanced-topics/post-build-hook.html#implementation-caveats", + "chap-writing-nix-expressions": "language/index.html", "part-command-ref": "command-ref/command-ref.html", "conf-allow-import-from-derivation": "command-ref/conf-file.html#conf-allow-import-from-derivation", "conf-allow-new-privileges": "command-ref/conf-file.html#conf-allow-new-privileges", From 53fdcbca509b6c5dacaea3d3c465d86e49b0dd74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Luis=20Lafuente?= Date: Mon, 8 Jan 2024 19:46:38 +0100 Subject: [PATCH 896/909] Add clang format configuration --- .clang-format | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .clang-format diff --git a/.clang-format b/.clang-format new file mode 100644 index 000000000..9c0c0946a --- /dev/null +++ b/.clang-format @@ -0,0 +1,30 @@ +BasedOnStyle: LLVM +IndentWidth: 4 +BreakBeforeBraces: Custom +BraceWrapping: + AfterStruct: true + AfterClass: true + AfterFunction: true + AfterUnion: true + SplitEmptyRecord: false +PointerAlignment: Middle +FixNamespaceComments: false +SortIncludes: Never +#IndentPPDirectives: BeforeHash +SpaceAfterCStyleCast: true +SpaceAfterTemplateKeyword: false +AccessModifierOffset: -4 +AlignAfterOpenBracket: AlwaysBreak +AlignEscapedNewlines: DontAlign +ColumnLimit: 120 +BreakStringLiterals: false +BitFieldColonSpacing: None +AllowShortFunctionsOnASingleLine: Empty +AlwaysBreakTemplateDeclarations: Yes +BinPackParameters: false +BreakConstructorInitializers: BeforeComma +EmptyLineAfterAccessModifier: Leave # change to always/never later? +EmptyLineBeforeAccessModifier: Leave +#PackConstructorInitializers: BinPack +BreakBeforeBinaryOperators: NonAssignment +AlwaysBreakBeforeMultilineStrings: true From 4feb7d9f715021784952bea57b37a8628c9b6860 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Mon, 18 Dec 2023 13:14:42 -0800 Subject: [PATCH 897/909] Combine `AbstractPos`, `PosAdapter`, and `Pos` Also move `SourcePath` into `libutil`. These changes allow `error.hh` and `error.cc` to access source path and position information, which we can use to produce better error messages (for example, we could consider omitting filenames when two or more consecutive stack frames originate from the same file). --- src/libcmd/editor-for.cc | 1 + src/libcmd/editor-for.hh | 2 +- src/libcmd/installable-value.cc | 3 +- src/libcmd/repl.cc | 2 +- src/libexpr/eval.cc | 9 +- src/libexpr/eval.hh | 2 +- src/libexpr/nixexpr.cc | 63 ------- src/libexpr/nixexpr.hh | 26 +-- src/libexpr/primops.cc | 3 +- src/libexpr/value.hh | 1 + src/libfetchers/fetch-to-store.cc | 68 ++++++++ src/libfetchers/fetch-to-store.hh | 22 +++ src/libfetchers/fetchers.cc | 4 +- src/libfetchers/filtering-input-accessor.hh | 1 + src/libfetchers/fs-input-accessor.hh | 1 + src/libfetchers/input-accessor.cc | 129 --------------- src/libfetchers/input-accessor.hh | 174 -------------------- src/libfetchers/memory-input-accessor.cc | 1 + src/libfetchers/memory-input-accessor.hh | 1 + src/libstore/store-api.hh | 1 + src/libutil/error.cc | 55 +------ src/libutil/error.hh | 42 +---- src/libutil/input-accessor.hh | 27 +++ src/libutil/logging.cc | 6 +- src/libutil/position.cc | 112 +++++++++++++ src/libutil/position.hh | 74 +++++++++ src/libutil/ref.hh | 1 + src/{libstore => libutil}/repair-flag.hh | 0 src/libutil/source-path.cc | 105 ++++++++++++ src/libutil/source-path.hh | 114 +++++++++++++ 30 files changed, 561 insertions(+), 489 deletions(-) create mode 100644 src/libfetchers/fetch-to-store.cc create mode 100644 src/libfetchers/fetch-to-store.hh delete mode 100644 src/libfetchers/input-accessor.cc delete mode 100644 src/libfetchers/input-accessor.hh create mode 100644 src/libutil/input-accessor.hh create mode 100644 src/libutil/position.cc create mode 100644 src/libutil/position.hh rename src/{libstore => libutil}/repair-flag.hh (100%) create mode 100644 src/libutil/source-path.cc create mode 100644 src/libutil/source-path.hh diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc index 619d3673f..67653d9c9 100644 --- a/src/libcmd/editor-for.cc +++ b/src/libcmd/editor-for.cc @@ -1,5 +1,6 @@ #include "editor-for.hh" #include "environment-variables.hh" +#include "source-path.hh" namespace nix { diff --git a/src/libcmd/editor-for.hh b/src/libcmd/editor-for.hh index fbf4307c9..8acd7011e 100644 --- a/src/libcmd/editor-for.hh +++ b/src/libcmd/editor-for.hh @@ -2,7 +2,7 @@ ///@file #include "types.hh" -#include "input-accessor.hh" +#include "source-path.hh" namespace nix { diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index bdc34bbe3..c8a3e1b21 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -1,5 +1,6 @@ #include "installable-value.hh" #include "eval-cache.hh" +#include "fetch-to-store.hh" namespace nix { @@ -44,7 +45,7 @@ ref InstallableValue::require(ref installable) std::optional InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx) { if (v.type() == nPath) { - auto storePath = v.path().fetchToStore(*state->store); + auto storePath = fetchToStore(*state->store, v.path()); return {{ .path = DerivedPath::Opaque { .path = std::move(storePath), diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index dea91ba63..78c4538b2 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -221,7 +221,7 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi // prefer direct pos, but if noPos then try the expr. auto pos = dt.pos ? dt.pos - : static_cast>(positions[dt.expr.getPos() ? dt.expr.getPos() : noPos]); + : positions[dt.expr.getPos() ? dt.expr.getPos() : noPos]; if (pos) { out << pos; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 31f2d4952..d408f1adc 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -19,6 +19,7 @@ #include "signals.hh" #include "gc-small-vector.hh" #include "url.hh" +#include "fetch-to-store.hh" #include #include @@ -870,7 +871,7 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & ? std::make_unique( *this, DebugTrace { - .pos = error->info().errPos ? error->info().errPos : static_cast>(positions[expr.getPos()]), + .pos = error->info().errPos ? error->info().errPos : positions[expr.getPos()], .expr = expr, .env = env, .hint = error->info().msg, @@ -909,7 +910,7 @@ static std::unique_ptr makeDebugTraceStacker( EvalState & state, Expr & expr, Env & env, - std::shared_ptr && pos, + std::shared_ptr && pos, const char * s, const std::string & s2) { @@ -1187,7 +1188,7 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) *this, *e, this->baseEnv, - e->getPos() ? static_cast>(positions[e->getPos()]) : nullptr, + e->getPos() ? std::make_shared(positions[e->getPos()]) : nullptr, "while evaluating the file '%1%':", resolvedPath.to_string()) : nullptr; @@ -2368,7 +2369,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat auto dstPath = i != srcToStore.end() ? i->second : [&]() { - auto dstPath = path.fetchToStore(*store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair); + auto dstPath = fetchToStore(*store, path, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair); allowPath(dstPath); srcToStore.insert_or_assign(path, dstPath); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 6e3f08d55..5e0f1886d 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -142,7 +142,7 @@ struct RegexCache; std::shared_ptr makeRegexCache(); struct DebugTrace { - std::shared_ptr pos; + std::shared_ptr pos; const Expr & expr; const Env & env; hintformat hint; diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index ede070cff..964de6351 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -11,58 +11,6 @@ namespace nix { ExprBlackHole eBlackHole; -struct PosAdapter : AbstractPos -{ - Pos::Origin origin; - - PosAdapter(Pos::Origin origin) - : origin(std::move(origin)) - { - } - - std::optional getSource() const override - { - return std::visit(overloaded { - [](const Pos::none_tag &) -> std::optional { - return std::nullopt; - }, - [](const Pos::Stdin & s) -> std::optional { - // Get rid of the null terminators added by the parser. - return std::string(s.source->c_str()); - }, - [](const Pos::String & s) -> std::optional { - // Get rid of the null terminators added by the parser. - return std::string(s.source->c_str()); - }, - [](const SourcePath & path) -> std::optional { - try { - return path.readFile(); - } catch (Error &) { - return std::nullopt; - } - } - }, origin); - } - - void print(std::ostream & out) const override - { - std::visit(overloaded { - [&](const Pos::none_tag &) { out << "«none»"; }, - [&](const Pos::Stdin &) { out << "«stdin»"; }, - [&](const Pos::String & s) { out << "«string»"; }, - [&](const SourcePath & path) { out << path; } - }, origin); - } -}; - -Pos::operator std::shared_ptr() const -{ - auto pos = std::make_shared(origin); - pos->line = line; - pos->column = column; - return pos; -} - // FIXME: remove, because *symbols* are abstract and do not have a single // textual representation; see printIdentifier() std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) @@ -268,17 +216,6 @@ void ExprPos::show(const SymbolTable & symbols, std::ostream & str) const } -std::ostream & operator << (std::ostream & str, const Pos & pos) -{ - if (auto pos2 = (std::shared_ptr) pos) { - str << *pos2; - } else - str << "undefined position"; - - return str; -} - - std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) { std::ostringstream out; diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 71ed9ef30..3cd46ca27 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -8,6 +8,7 @@ #include "symbol-table.hh" #include "error.hh" #include "chunked-vector.hh" +#include "position.hh" namespace nix { @@ -28,27 +29,6 @@ public: using EvalError::EvalError; }; -/** - * Position objects. - */ -struct Pos -{ - uint32_t line; - uint32_t column; - - struct none_tag { }; - struct Stdin { ref source; }; - struct String { ref source; }; - - typedef std::variant Origin; - - Origin origin; - - explicit operator bool() const { return line > 0; } - - operator std::shared_ptr() const; -}; - class PosIdx { friend class PosTable; @@ -81,7 +61,7 @@ public: mutable uint32_t idx = std::numeric_limits::max(); // Used for searching in PosTable::[]. - explicit Origin(uint32_t idx): idx(idx), origin{Pos::none_tag()} {} + explicit Origin(uint32_t idx): idx(idx), origin{std::monostate()} {} public: const Pos::Origin origin; @@ -132,8 +112,6 @@ public: inline PosIdx noPos = {}; -std::ostream & operator << (std::ostream & str, const Pos & pos); - struct Env; struct Value; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index b2ffcc051..ee07e5568 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -16,6 +16,7 @@ #include "value-to-xml.hh" #include "primops.hh" #include "fs-input-accessor.hh" +#include "fetch-to-store.hh" #include #include @@ -2240,7 +2241,7 @@ static void addPath( }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - auto dstPath = path.fetchToStore(*state.store, name, method, filter.get(), state.repair); + auto dstPath = fetchToStore(*state.store, path, name, method, filter.get(), state.repair); if (expectedHash && expectedStorePath != dstPath) state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); state.allowAndSetStorePathString(dstPath, v); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index d9860e921..c65b336b0 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -8,6 +8,7 @@ #include "symbol-table.hh" #include "value/context.hh" #include "input-accessor.hh" +#include "source-path.hh" #if HAVE_BOEHMGC #include diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc new file mode 100644 index 000000000..196489e05 --- /dev/null +++ b/src/libfetchers/fetch-to-store.cc @@ -0,0 +1,68 @@ +#include "fetch-to-store.hh" +#include "fetchers.hh" +#include "cache.hh" + +namespace nix { + +StorePath fetchToStore( + Store & store, + const SourcePath & path, + std::string_view name, + ContentAddressMethod method, + PathFilter * filter, + RepairFlag repair) +{ + // FIXME: add an optimisation for the case where the accessor is + // an FSInputAccessor pointing to a store path. + + std::optional cacheKey; + + if (!filter && path.accessor->fingerprint) { + cacheKey = fetchers::Attrs{ + {"_what", "fetchToStore"}, + {"store", store.storeDir}, + {"name", std::string(name)}, + {"fingerprint", *path.accessor->fingerprint}, + { + "method", + std::visit(overloaded { + [](const TextIngestionMethod &) { + return "text"; + }, + [](const FileIngestionMethod & fim) { + switch (fim) { + case FileIngestionMethod::Flat: return "flat"; + case FileIngestionMethod::Recursive: return "nar"; + default: assert(false); + } + }, + }, method.raw), + }, + {"path", path.path.abs()} + }; + if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) { + debug("store path cache hit for '%s'", path); + return res->second; + } + } else + debug("source path '%s' is uncacheable", path); + + Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", path)); + + auto filter2 = filter ? *filter : defaultPathFilter; + + auto storePath = + settings.readOnlyMode + ? store.computeStorePath( + name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2).first + : store.addToStore( + name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2, repair); + + if (cacheKey) + fetchers::getCache()->add(store, *cacheKey, {}, storePath, true); + + return storePath; +} + + +} diff --git a/src/libfetchers/fetch-to-store.hh b/src/libfetchers/fetch-to-store.hh new file mode 100644 index 000000000..e5e039340 --- /dev/null +++ b/src/libfetchers/fetch-to-store.hh @@ -0,0 +1,22 @@ +#pragma once + +#include "source-path.hh" +#include "store-api.hh" +#include "file-system.hh" +#include "repair-flag.hh" +#include "file-content-address.hh" + +namespace nix { + +/** + * Copy the `path` to the Nix store. + */ +StorePath fetchToStore( + Store & store, + const SourcePath & path, + std::string_view name = "source", + ContentAddressMethod method = FileIngestionMethod::Recursive, + PathFilter * filter = nullptr, + RepairFlag repair = NoRepair); + +} diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index f309e5993..7f282c972 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -1,6 +1,8 @@ #include "fetchers.hh" #include "store-api.hh" #include "input-accessor.hh" +#include "source-path.hh" +#include "fetch-to-store.hh" #include @@ -374,7 +376,7 @@ void InputScheme::clone(const Input & input, const Path & destDir) const std::pair InputScheme::fetch(ref store, const Input & input) { auto [accessor, input2] = getAccessor(store, input); - auto storePath = SourcePath(accessor).fetchToStore(*store, input2.getName()); + auto storePath = fetchToStore(*store, SourcePath(accessor), input2.getName()); return {storePath, input2}; } diff --git a/src/libfetchers/filtering-input-accessor.hh b/src/libfetchers/filtering-input-accessor.hh index e1b83c929..a352a33a6 100644 --- a/src/libfetchers/filtering-input-accessor.hh +++ b/src/libfetchers/filtering-input-accessor.hh @@ -1,6 +1,7 @@ #pragma once #include "input-accessor.hh" +#include "source-path.hh" namespace nix { diff --git a/src/libfetchers/fs-input-accessor.hh b/src/libfetchers/fs-input-accessor.hh index ba5af5887..a98e83511 100644 --- a/src/libfetchers/fs-input-accessor.hh +++ b/src/libfetchers/fs-input-accessor.hh @@ -1,6 +1,7 @@ #pragma once #include "input-accessor.hh" +#include "source-path.hh" namespace nix { diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc deleted file mode 100644 index a647f5915..000000000 --- a/src/libfetchers/input-accessor.cc +++ /dev/null @@ -1,129 +0,0 @@ -#include "input-accessor.hh" -#include "store-api.hh" -#include "cache.hh" - -namespace nix { - -StorePath InputAccessor::fetchToStore( - Store & store, - const CanonPath & path, - std::string_view name, - ContentAddressMethod method, - PathFilter * filter, - RepairFlag repair) -{ - // FIXME: add an optimisation for the case where the accessor is - // an FSInputAccessor pointing to a store path. - - std::optional cacheKey; - - if (!filter && fingerprint) { - cacheKey = fetchers::Attrs{ - {"_what", "fetchToStore"}, - {"store", store.storeDir}, - {"name", std::string(name)}, - {"fingerprint", *fingerprint}, - { - "method", - std::visit(overloaded { - [](const TextIngestionMethod &) { - return "text"; - }, - [](const FileIngestionMethod & fim) { - switch (fim) { - case FileIngestionMethod::Flat: return "flat"; - case FileIngestionMethod::Recursive: return "nar"; - default: assert(false); - } - }, - }, method.raw), - }, - {"path", path.abs()} - }; - if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) { - debug("store path cache hit for '%s'", showPath(path)); - return res->second; - } - } else - debug("source path '%s' is uncacheable", showPath(path)); - - Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path))); - - auto filter2 = filter ? *filter : defaultPathFilter; - - auto storePath = - settings.readOnlyMode - ? store.computeStorePath( - name, *this, path, method, HashAlgorithm::SHA256, {}, filter2).first - : store.addToStore( - name, *this, path, method, HashAlgorithm::SHA256, {}, filter2, repair); - - if (cacheKey) - fetchers::getCache()->add(store, *cacheKey, {}, storePath, true); - - return storePath; -} - -std::ostream & operator << (std::ostream & str, const SourcePath & path) -{ - str << path.to_string(); - return str; -} - -StorePath SourcePath::fetchToStore( - Store & store, - std::string_view name, - ContentAddressMethod method, - PathFilter * filter, - RepairFlag repair) const -{ - return accessor->fetchToStore(store, path, name, method, filter, repair); -} - -std::string_view SourcePath::baseName() const -{ - return path.baseName().value_or("source"); -} - -SourcePath SourcePath::parent() const -{ - auto p = path.parent(); - assert(p); - return {accessor, std::move(*p)}; -} - -SourcePath SourcePath::resolveSymlinks() const -{ - auto res = SourcePath(accessor); - - int linksAllowed = 1024; - - std::list todo; - for (auto & c : path) - todo.push_back(std::string(c)); - - while (!todo.empty()) { - auto c = *todo.begin(); - todo.pop_front(); - if (c == "" || c == ".") - ; - else if (c == "..") - res.path.pop(); - else { - res.path.push(c); - if (auto st = res.maybeLstat(); st && st->type == InputAccessor::tSymlink) { - if (!linksAllowed--) - throw Error("infinite symlink recursion in path '%s'", path); - auto target = res.readLink(); - res.path.pop(); - if (hasPrefix(target, "/")) - res.path = CanonPath::root; - todo.splice(todo.begin(), tokenizeString>(target, "/")); - } - } - } - - return res; -} - -} diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh deleted file mode 100644 index d2a21cb4b..000000000 --- a/src/libfetchers/input-accessor.hh +++ /dev/null @@ -1,174 +0,0 @@ -#pragma once -///@file - -#include "source-accessor.hh" -#include "ref.hh" -#include "types.hh" -#include "file-system.hh" -#include "repair-flag.hh" -#include "content-address.hh" - -namespace nix { - -MakeError(RestrictedPathError, Error); - -struct SourcePath; -class StorePath; -class Store; - -struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this -{ - std::optional fingerprint; - - /** - * Return the maximum last-modified time of the files in this - * tree, if available. - */ - virtual std::optional getLastModified() - { - return std::nullopt; - } - - StorePath fetchToStore( - Store & store, - const CanonPath & path, - std::string_view name = "source", - ContentAddressMethod method = FileIngestionMethod::Recursive, - PathFilter * filter = nullptr, - RepairFlag repair = NoRepair); -}; - -/** - * An abstraction for accessing source files during - * evaluation. Currently, it's just a wrapper around `CanonPath` that - * accesses files in the regular filesystem, but in the future it will - * support fetching files in other ways. - */ -struct SourcePath -{ - ref accessor; - CanonPath path; - - SourcePath(ref accessor, CanonPath path = CanonPath::root) - : accessor(std::move(accessor)) - , path(std::move(path)) - { } - - std::string_view baseName() const; - - /** - * Construct the parent of this `SourcePath`. Aborts if `this` - * denotes the root. - */ - SourcePath parent() const; - - /** - * If this `SourcePath` denotes a regular file (not a symlink), - * return its contents; otherwise throw an error. - */ - std::string readFile() const - { return accessor->readFile(path); } - - /** - * Return whether this `SourcePath` denotes a file (of any type) - * that exists - */ - bool pathExists() const - { return accessor->pathExists(path); } - - /** - * Return stats about this `SourcePath`, or throw an exception if - * it doesn't exist. - */ - InputAccessor::Stat lstat() const - { return accessor->lstat(path); } - - /** - * Return stats about this `SourcePath`, or std::nullopt if it - * doesn't exist. - */ - std::optional maybeLstat() const - { return accessor->maybeLstat(path); } - - /** - * If this `SourcePath` denotes a directory (not a symlink), - * return its directory entries; otherwise throw an error. - */ - InputAccessor::DirEntries readDirectory() const - { return accessor->readDirectory(path); } - - /** - * If this `SourcePath` denotes a symlink, return its target; - * otherwise throw an error. - */ - std::string readLink() const - { return accessor->readLink(path); } - - /** - * Dump this `SourcePath` to `sink` as a NAR archive. - */ - void dumpPath( - Sink & sink, - PathFilter & filter = defaultPathFilter) const - { return accessor->dumpPath(path, sink, filter); } - - /** - * Copy this `SourcePath` to the Nix store. - */ - StorePath fetchToStore( - Store & store, - std::string_view name = "source", - ContentAddressMethod method = FileIngestionMethod::Recursive, - PathFilter * filter = nullptr, - RepairFlag repair = NoRepair) const; - - /** - * Return the location of this path in the "real" filesystem, if - * it has a physical location. - */ - std::optional getPhysicalPath() const - { return accessor->getPhysicalPath(path); } - - std::string to_string() const - { return accessor->showPath(path); } - - /** - * Append a `CanonPath` to this path. - */ - SourcePath operator + (const CanonPath & x) const - { return {accessor, path + x}; } - - /** - * Append a single component `c` to this path. `c` must not - * contain a slash. A slash is implicitly added between this path - * and `c`. - */ - SourcePath operator + (std::string_view c) const - { return {accessor, path + c}; } - - bool operator == (const SourcePath & x) const - { - return std::tie(accessor, path) == std::tie(x.accessor, x.path); - } - - bool operator != (const SourcePath & x) const - { - return std::tie(accessor, path) != std::tie(x.accessor, x.path); - } - - bool operator < (const SourcePath & x) const - { - return std::tie(accessor, path) < std::tie(x.accessor, x.path); - } - - /** - * Resolve any symlinks in this `SourcePath` (including its - * parents). The result is a `SourcePath` in which no element is a - * symlink. - */ - SourcePath resolveSymlinks() const; -}; - -std::ostream & operator << (std::ostream & str, const SourcePath & path); - -} diff --git a/src/libfetchers/memory-input-accessor.cc b/src/libfetchers/memory-input-accessor.cc index 057f3e37f..88a2e34e8 100644 --- a/src/libfetchers/memory-input-accessor.cc +++ b/src/libfetchers/memory-input-accessor.cc @@ -1,5 +1,6 @@ #include "memory-input-accessor.hh" #include "memory-source-accessor.hh" +#include "source-path.hh" namespace nix { diff --git a/src/libfetchers/memory-input-accessor.hh b/src/libfetchers/memory-input-accessor.hh index b75b02bfd..508b07722 100644 --- a/src/libfetchers/memory-input-accessor.hh +++ b/src/libfetchers/memory-input-accessor.hh @@ -1,4 +1,5 @@ #include "input-accessor.hh" +#include "source-path.hh" namespace nix { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 96a7ebd7b..9667b5e9e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -13,6 +13,7 @@ #include "path-info.hh" #include "repair-flag.hh" #include "store-dir-config.hh" +#include "source-path.hh" #include #include diff --git a/src/libutil/error.cc b/src/libutil/error.cc index e42925c2b..bd2f6b840 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -2,6 +2,7 @@ #include "environment-variables.hh" #include "signals.hh" #include "terminal.hh" +#include "position.hh" #include #include @@ -10,7 +11,7 @@ namespace nix { -void BaseError::addTrace(std::shared_ptr && e, hintformat hint, bool frame) +void BaseError::addTrace(std::shared_ptr && e, hintformat hint, bool frame) { err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame }); } @@ -41,15 +42,6 @@ std::ostream & operator <<(std::ostream & os, const hintformat & hf) return os << hf.str(); } -std::ostream & operator <<(std::ostream & str, const AbstractPos & pos) -{ - pos.print(str); - str << ":" << pos.line; - if (pos.column > 0) - str << ":" << pos.column; - return str; -} - /** * An arbitrarily defined value comparison for the purpose of using traces in the key of a sorted container. */ @@ -76,49 +68,10 @@ inline bool operator> (const Trace& lhs, const Trace& rhs) { return rhs < lhs; } inline bool operator<=(const Trace& lhs, const Trace& rhs) { return !(lhs > rhs); } inline bool operator>=(const Trace& lhs, const Trace& rhs) { return !(lhs < rhs); } -std::optional AbstractPos::getCodeLines() const -{ - if (line == 0) - return std::nullopt; - - if (auto source = getSource()) { - - std::istringstream iss(*source); - // count the newlines. - int count = 0; - std::string curLine; - int pl = line - 1; - - LinesOfCode loc; - - do { - std::getline(iss, curLine); - ++count; - if (count < pl) - ; - else if (count == pl) { - loc.prevLineOfCode = curLine; - } else if (count == pl + 1) { - loc.errLineOfCode = curLine; - } else if (count == pl + 2) { - loc.nextLineOfCode = curLine; - break; - } - - if (!iss.good()) - break; - } while (true); - - return loc; - } - - return std::nullopt; -} - // print lines of code to the ostream, indicating the error column. void printCodeLines(std::ostream & out, const std::string & prefix, - const AbstractPos & errPos, + const Pos & errPos, const LinesOfCode & loc) { // previous line of code. @@ -196,7 +149,7 @@ static bool printUnknownLocations = getEnv("_NIX_EVAL_SHOW_UNKNOWN_LOCATIONS").h * * @return true if a position was printed. */ -static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std::shared_ptr & pos) { +static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std::shared_ptr & pos) { bool hasPos = pos && *pos; if (hasPos) { oss << indent << ANSI_BLUE << "at " ANSI_WARNING << *pos << ANSI_NORMAL << ":"; diff --git a/src/libutil/error.hh b/src/libutil/error.hh index baffca128..234cbe1f6 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -63,45 +63,15 @@ struct LinesOfCode { std::optional nextLineOfCode; }; -/** - * An abstract type that represents a location in a source file. - */ -struct AbstractPos -{ - uint32_t line = 0; - uint32_t column = 0; - - /** - * An AbstractPos may be a "null object", representing an unknown position. - * - * Return true if this position is known. - */ - inline operator bool() const { return line != 0; }; - - /** - * Return the contents of the source file. - */ - virtual std::optional getSource() const - { return std::nullopt; }; - - virtual void print(std::ostream & out) const = 0; - - std::optional getCodeLines() const; - - virtual ~AbstractPos() = default; - - inline auto operator<=>(const AbstractPos& rhs) const = default; -}; - -std::ostream & operator << (std::ostream & str, const AbstractPos & pos); +struct Pos; void printCodeLines(std::ostream & out, const std::string & prefix, - const AbstractPos & errPos, + const Pos & errPos, const LinesOfCode & loc); struct Trace { - std::shared_ptr pos; + std::shared_ptr pos; hintformat hint; bool frame; }; @@ -114,7 +84,7 @@ inline bool operator>=(const Trace& lhs, const Trace& rhs); struct ErrorInfo { Verbosity level; hintformat msg; - std::shared_ptr errPos; + std::shared_ptr errPos; std::list traces; Suggestions suggestions; @@ -185,12 +155,12 @@ public: } template - void addTrace(std::shared_ptr && e, std::string_view fs, const Args & ... args) + void addTrace(std::shared_ptr && e, std::string_view fs, const Args & ... args) { addTrace(std::move(e), hintfmt(std::string(fs), args...)); } - void addTrace(std::shared_ptr && e, hintformat hint, bool frame = false); + void addTrace(std::shared_ptr && e, hintformat hint, bool frame = false); bool hasTrace() const { return !err.traces.empty(); } diff --git a/src/libutil/input-accessor.hh b/src/libutil/input-accessor.hh new file mode 100644 index 000000000..55b7c2f2f --- /dev/null +++ b/src/libutil/input-accessor.hh @@ -0,0 +1,27 @@ +#pragma once +///@file + +#include "source-accessor.hh" +#include "ref.hh" +#include "repair-flag.hh" + +namespace nix { + +MakeError(RestrictedPathError, Error); + +struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this +{ + std::optional fingerprint; + + /** + * Return the maximum last-modified time of the files in this + * tree, if available. + */ + virtual std::optional getLastModified() + { + return std::nullopt; + } + +}; + +} diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 60b0865bf..183aee2dc 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -4,6 +4,8 @@ #include "terminal.hh" #include "util.hh" #include "config.hh" +#include "source-path.hh" +#include "position.hh" #include #include @@ -136,13 +138,13 @@ Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type, logger.startActivity(id, lvl, type, s, fields, parent); } -void to_json(nlohmann::json & json, std::shared_ptr pos) +void to_json(nlohmann::json & json, std::shared_ptr pos) { if (pos) { json["line"] = pos->line; json["column"] = pos->column; std::ostringstream str; - pos->print(str); + pos->print(str, true); json["file"] = str.str(); } else { json["line"] = nullptr; diff --git a/src/libutil/position.cc b/src/libutil/position.cc new file mode 100644 index 000000000..b39a5a1d4 --- /dev/null +++ b/src/libutil/position.cc @@ -0,0 +1,112 @@ +#include "position.hh" + +namespace nix { + +Pos::Pos(const Pos * other) +{ + if (!other) { + return; + } + line = other->line; + column = other->column; + origin = std::move(other->origin); +} + +Pos::operator std::shared_ptr() const +{ + return std::make_shared(&*this); +} + +bool Pos::operator<(const Pos &rhs) const +{ + return std::forward_as_tuple(line, column, origin) + < std::forward_as_tuple(rhs.line, rhs.column, rhs.origin); +} + +std::optional Pos::getCodeLines() const +{ + if (line == 0) + return std::nullopt; + + if (auto source = getSource()) { + + std::istringstream iss(*source); + // count the newlines. + int count = 0; + std::string curLine; + int pl = line - 1; + + LinesOfCode loc; + + do { + std::getline(iss, curLine); + ++count; + if (count < pl) + ; + else if (count == pl) { + loc.prevLineOfCode = curLine; + } else if (count == pl + 1) { + loc.errLineOfCode = curLine; + } else if (count == pl + 2) { + loc.nextLineOfCode = curLine; + break; + } + + if (!iss.good()) + break; + } while (true); + + return loc; + } + + return std::nullopt; +} + + +std::optional Pos::getSource() const +{ + return std::visit(overloaded { + [](const std::monostate &) -> std::optional { + return std::nullopt; + }, + [](const Pos::Stdin & s) -> std::optional { + // Get rid of the null terminators added by the parser. + return std::string(s.source->c_str()); + }, + [](const Pos::String & s) -> std::optional { + // Get rid of the null terminators added by the parser. + return std::string(s.source->c_str()); + }, + [](const SourcePath & path) -> std::optional { + try { + return path.readFile(); + } catch (Error &) { + return std::nullopt; + } + } + }, origin); +} + +void Pos::print(std::ostream & out, bool showOrigin) const +{ + if (showOrigin) { + std::visit(overloaded { + [&](const std::monostate &) { out << "«none»"; }, + [&](const Pos::Stdin &) { out << "«stdin»"; }, + [&](const Pos::String & s) { out << "«string»"; }, + [&](const SourcePath & path) { out << path; } + }, origin); + out << ":"; + } + out << line; + if (column > 0) + out << ":" << column; +} + +std::ostream & operator<<(std::ostream & str, const Pos & pos) +{ + pos.print(str, true); + return str; +} + +} diff --git a/src/libutil/position.hh b/src/libutil/position.hh new file mode 100644 index 000000000..a184997ed --- /dev/null +++ b/src/libutil/position.hh @@ -0,0 +1,74 @@ +#pragma once +/** + * @file + * + * @brief Pos and AbstractPos + */ + +#include +#include + +#include "source-path.hh" + +namespace nix { + +/** + * A position and an origin for that position (like a source file). + */ +struct Pos +{ + uint32_t line = 0; + uint32_t column = 0; + + struct Stdin { + ref source; + bool operator==(const Stdin & rhs) const + { return *source == *rhs.source; } + bool operator!=(const Stdin & rhs) const + { return *source != *rhs.source; } + bool operator<(const Stdin & rhs) const + { return *source < *rhs.source; } + }; + struct String { + ref source; + bool operator==(const String & rhs) const + { return *source == *rhs.source; } + bool operator!=(const String & rhs) const + { return *source != *rhs.source; } + bool operator<(const String & rhs) const + { return *source < *rhs.source; } + }; + + typedef std::variant Origin; + + Origin origin = std::monostate(); + + Pos() { } + Pos(uint32_t line, uint32_t column, Origin origin) + : line(line), column(column), origin(origin) { } + Pos(Pos & other) = default; + Pos(const Pos & other) = default; + Pos(Pos && other) = default; + Pos(const Pos * other); + + explicit operator bool() const { return line > 0; } + + operator std::shared_ptr() const; + + /** + * Return the contents of the source file. + */ + std::optional getSource() const; + + void print(std::ostream & out, bool showOrigin) const; + + std::optional getCodeLines() const; + + bool operator==(const Pos & rhs) const = default; + bool operator!=(const Pos & rhs) const = default; + bool operator<(const Pos & rhs) const; +}; + +std::ostream & operator<<(std::ostream & str, const Pos & pos); + +} diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index af5f8304c..5d0c3696d 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include #include #include #include diff --git a/src/libstore/repair-flag.hh b/src/libutil/repair-flag.hh similarity index 100% rename from src/libstore/repair-flag.hh rename to src/libutil/repair-flag.hh diff --git a/src/libutil/source-path.cc b/src/libutil/source-path.cc new file mode 100644 index 000000000..d85b0b7fe --- /dev/null +++ b/src/libutil/source-path.cc @@ -0,0 +1,105 @@ +#include "source-path.hh" + +namespace nix { + +std::string_view SourcePath::baseName() const +{ return path.baseName().value_or("source"); } + +SourcePath SourcePath::parent() const +{ + auto p = path.parent(); + assert(p); + return {accessor, std::move(*p)}; +} + +std::string SourcePath::readFile() const +{ return accessor->readFile(path); } + +bool SourcePath::pathExists() const +{ return accessor->pathExists(path); } + +InputAccessor::Stat SourcePath::lstat() const +{ return accessor->lstat(path); } + +std::optional SourcePath::maybeLstat() const +{ return accessor->maybeLstat(path); } + +InputAccessor::DirEntries SourcePath::readDirectory() const +{ return accessor->readDirectory(path); } + +std::string SourcePath::readLink() const +{ return accessor->readLink(path); } + +void SourcePath::dumpPath( + Sink & sink, + PathFilter & filter) const +{ return accessor->dumpPath(path, sink, filter); } + +std::optional SourcePath::getPhysicalPath() const +{ return accessor->getPhysicalPath(path); } + +std::string SourcePath::to_string() const +{ return accessor->showPath(path); } + +SourcePath SourcePath::operator+(const CanonPath & x) const +{ return {accessor, path + x}; } + +SourcePath SourcePath::operator+(std::string_view c) const +{ return {accessor, path + c}; } + +bool SourcePath::operator==(const SourcePath & x) const +{ + return std::tie(*accessor, path) == std::tie(*x.accessor, x.path); +} + +bool SourcePath::operator!=(const SourcePath & x) const +{ + return std::tie(*accessor, path) != std::tie(*x.accessor, x.path); +} + +bool SourcePath::operator<(const SourcePath & x) const +{ + return std::tie(*accessor, path) < std::tie(*x.accessor, x.path); +} + +SourcePath SourcePath::resolveSymlinks() const +{ + auto res = SourcePath(accessor); + + int linksAllowed = 1024; + + std::list todo; + for (auto & c : path) + todo.push_back(std::string(c)); + + while (!todo.empty()) { + auto c = *todo.begin(); + todo.pop_front(); + if (c == "" || c == ".") + ; + else if (c == "..") + res.path.pop(); + else { + res.path.push(c); + if (auto st = res.maybeLstat(); st && st->type == InputAccessor::tSymlink) { + if (!linksAllowed--) + throw Error("infinite symlink recursion in path '%s'", path); + auto target = res.readLink(); + res.path.pop(); + if (hasPrefix(target, "/")) + res.path = CanonPath::root; + todo.splice(todo.begin(), tokenizeString>(target, "/")); + } + } + } + + return res; +} + +std::ostream & operator<<(std::ostream & str, const SourcePath & path) +{ + str << path.to_string(); + return str; +} + +} diff --git a/src/libutil/source-path.hh b/src/libutil/source-path.hh new file mode 100644 index 000000000..bf5625ca5 --- /dev/null +++ b/src/libutil/source-path.hh @@ -0,0 +1,114 @@ +#pragma once +/** + * @file + * + * @brief SourcePath + */ + +#include "ref.hh" +#include "canon-path.hh" +#include "input-accessor.hh" + +namespace nix { + +/** + * An abstraction for accessing source files during + * evaluation. Currently, it's just a wrapper around `CanonPath` that + * accesses files in the regular filesystem, but in the future it will + * support fetching files in other ways. + */ +struct SourcePath +{ + ref accessor; + CanonPath path; + + SourcePath(ref accessor, CanonPath path = CanonPath::root) + : accessor(std::move(accessor)) + , path(std::move(path)) + { } + + std::string_view baseName() const; + + /** + * Construct the parent of this `SourcePath`. Aborts if `this` + * denotes the root. + */ + SourcePath parent() const; + + /** + * If this `SourcePath` denotes a regular file (not a symlink), + * return its contents; otherwise throw an error. + */ + std::string readFile() const; + + /** + * Return whether this `SourcePath` denotes a file (of any type) + * that exists + */ + bool pathExists() const; + + /** + * Return stats about this `SourcePath`, or throw an exception if + * it doesn't exist. + */ + InputAccessor::Stat lstat() const; + + /** + * Return stats about this `SourcePath`, or std::nullopt if it + * doesn't exist. + */ + std::optional maybeLstat() const; + + /** + * If this `SourcePath` denotes a directory (not a symlink), + * return its directory entries; otherwise throw an error. + */ + InputAccessor::DirEntries readDirectory() const; + + /** + * If this `SourcePath` denotes a symlink, return its target; + * otherwise throw an error. + */ + std::string readLink() const; + + /** + * Dump this `SourcePath` to `sink` as a NAR archive. + */ + void dumpPath( + Sink & sink, + PathFilter & filter = defaultPathFilter) const; + + /** + * Return the location of this path in the "real" filesystem, if + * it has a physical location. + */ + std::optional getPhysicalPath() const; + + std::string to_string() const; + + /** + * Append a `CanonPath` to this path. + */ + SourcePath operator + (const CanonPath & x) const; + + /** + * Append a single component `c` to this path. `c` must not + * contain a slash. A slash is implicitly added between this path + * and `c`. + */ + SourcePath operator+(std::string_view c) const; + bool operator==(const SourcePath & x) const; + bool operator!=(const SourcePath & x) const; + bool operator<(const SourcePath & x) const; + + /** + * Resolve any symlinks in this `SourcePath` (including its + * parents). The result is a `SourcePath` in which no element is a + * symlink. + */ + SourcePath resolveSymlinks() const; +}; + +std::ostream & operator << (std::ostream & str, const SourcePath & path); + +} From bbd0a959e17e988ef1ec2fadd1ab5bb66420fd6f Mon Sep 17 00:00:00 2001 From: Weijia Wang <9713184+wegank@users.noreply.github.com> Date: Mon, 8 Jan 2024 20:37:42 +0100 Subject: [PATCH 898/909] Make lowdown optional Co-authored-by: John Ericson --- configure.ac | 16 +++++++++++++++- package.nix | 5 +++++ src/libcmd/markdown.cc | 6 ++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index b97e25bbd..929750932 100644 --- a/configure.ac +++ b/configure.ac @@ -374,7 +374,21 @@ PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9]) # Look for lowdown library. -PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"]) +AC_ARG_ENABLE([markdown], AS_HELP_STRING([--enable-markdown], [Enable Markdown rendering in the Nix binary (requires lowdown) [default=auto]]), + enable_markdown=$enableval, enable_markdown=auto) +AS_CASE(["$enable_markdown"], + [yes | auto], [ + PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [ + CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS" + have_lowdown=1 + AC_DEFINE(HAVE_LOWDOWN, 1, [Whether lowdown is available and should be used for Markdown rendering.]) + ], [ + AS_IF([test "x$enable_markdown" == "xyes"], [AC_MSG_ERROR([--enable-markdown was specified, but lowdown was not found.])]) + ]) + ], + [no], [have_lowdown=], + [AC_MSG_ERROR([--enable-markdown must be one of: yes, no, auto])]) +AC_SUBST(HAVE_LOWDOWN, [$have_lowdown]) # Look for libgit2. diff --git a/package.nix b/package.nix index dfebdb0e4..dd37809d0 100644 --- a/package.nix +++ b/package.nix @@ -68,6 +68,9 @@ # Whether to build the regular manual , enableManual ? __forDefaults.canRunInstalled +# Whether to enable Markdown rendering in the Nix binary. +, enableMarkdown ? !stdenv.hostPlatform.isWindows + # Whether to compile `rl-next.md`, the release notes for the next # not-yet-released version of Nix in the manul, from the individual # change log entries in the directory. @@ -213,6 +216,7 @@ in { xz ] ++ lib.optionals (!stdenv.hostPlatform.isWindows) [ editline + ] ++ lib.optionals enableMarkdown [ lowdown ] ++ lib.optionals buildUnitTests [ gtest @@ -269,6 +273,7 @@ in { (lib.enableFeature doInstallCheck "functional-tests") (lib.enableFeature enableInternalAPIDocs "internal-api-docs") (lib.enableFeature enableManual "doc-gen") + (lib.enableFeature enableMarkdown "markdown") (lib.enableFeature installUnitTests "install-unit-tests") ] ++ lib.optionals (!forDevShell) [ "--sysconfdir=/etc" diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 8b3bbc1b5..a4e3c5a77 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -4,12 +4,15 @@ #include "terminal.hh" #include +#if HAVE_LOWDOWN #include +#endif namespace nix { std::string renderMarkdownToTerminal(std::string_view markdown) { +#if HAVE_LOWDOWN int windowWidth = getWindowSize().second; struct lowdown_opts opts { @@ -48,6 +51,9 @@ std::string renderMarkdownToTerminal(std::string_view markdown) throw Error("allocation error while rendering Markdown"); return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI()); +#else + return std::string(markdown); +#endif } } From 29eb5ed1dc54ec45ab23b50ef259d2b370e8b1e8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 Jan 2024 14:47:42 -0500 Subject: [PATCH 899/909] Fix Internal API docs Because of source filtering, they were empty. Fixes #9694 --- package.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/package.nix b/package.nix index dfebdb0e4..aad022b32 100644 --- a/package.nix +++ b/package.nix @@ -164,6 +164,10 @@ in { ./doc/manual ] ++ lib.optionals enableInternalAPIDocs [ ./doc/internal-api + # Source might not be compiled, but still must be available + # for Doxygen to gather comments. + ./src + ./tests/unit ] ++ lib.optionals buildUnitTests [ ./tests/unit ] ++ lib.optionals doInstallCheck [ From 3d9e0c60e4cf135943d2c72a990ff2c0e3e311a7 Mon Sep 17 00:00:00 2001 From: DavHau Date: Tue, 9 Jan 2024 18:36:09 +0700 Subject: [PATCH 900/909] gitignore: add result-* --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d9f9d949b..a47b195bb 100644 --- a/.gitignore +++ b/.gitignore @@ -141,6 +141,7 @@ compile_commands.json nix-rust/target result +result-* # IDE .vscode/ From 2cea88dbc8c277d7403e6dd65f482fd2eb931e52 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 2 Sep 2023 15:56:37 -0400 Subject: [PATCH 901/909] Improve build sytem support for readline instead of editline Changes: - CPP variable is now `USE_READLINE` not `READLINE` - `configure.ac` supports with new CLI flag - `package.nix` supports with new configuration option - `flake.nix` CIs this (along with no markdown) Remove old Ubuntu 16.04 stop-gap too, as that is now quite old. Motivation: - editline does not build for Windows, but readline *should*. (I am still working on this in Nixpkgs at this time, however. So there will be a follow-up Nix PR removing the windows-only skipping of the readline library once I am done.) - Per https://salsa.debian.org/debian/nix/-/blob/master/debian/rules?ref_type=heads#L27 and #2551, Debian builds Nix with readline. Now we better support and CI that build configuration. This is picking up where #2551 left off, ensuring we test a few more things not merely have CPP for them. Co-authored-by: Weijia Wang <9713184+wegank@users.noreply.github.com> --- configure.ac | 29 ++++++++++++++++++----------- flake.nix | 9 +++++++++ package.nix | 12 +++++++++++- src/libcmd/repl.cc | 6 +++--- 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/configure.ac b/configure.ac index fdbb2629e..2594544ab 100644 --- a/configure.ac +++ b/configure.ac @@ -251,17 +251,25 @@ PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CX # Look for libcurl, a required dependency. PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"]) -# Look for editline, a required dependency. +# Look for editline or readline, a required dependency. # The the libeditline.pc file was added only in libeditline >= 1.15.2, # see https://github.com/troglobit/editline/commit/0a8f2ef4203c3a4a4726b9dd1336869cd0da8607, -# but e.g. Ubuntu 16.04 has an older version, so we fall back to searching for -# editline.h when the pkg-config approach fails. -PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"], [ - AC_CHECK_HEADERS([editline.h], [true], - [AC_MSG_ERROR([Nix requires libeditline; it was found neither via pkg-config nor its normal header.])]) - AC_SEARCH_LIBS([readline read_history], [editline], [], - [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])]) -]) +# Older versions are no longer supported. +AC_ARG_WITH( + [readline-flavor], + AS_HELP_STRING([--with-readline-flavor],[Which library to use for nice line editting with the Nix language REPL" [default=editline]]), + [readline_flavor=$withval], + [readline_flavor=editline]) +AS_CASE(["$readline_flavor"], + [editline], [ + readline_flavor_pc=libeditline + ], + [readline], [ + readline_flavor_pc=readline + AC_DEFINE([USE_READLINE], [1], [Use readline instead of editline]) + ], + [AC_MSG_ERROR([bad value "$readline_flavor" for --with-readline-flavor, must be one of: editline, readline])]) +PKG_CHECK_MODULES([EDITLINE], [$readline_flavor_pc], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"]) # Look for libsodium. PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) @@ -387,8 +395,7 @@ AS_CASE(["$enable_markdown"], ]) ], [no], [have_lowdown=], - [AC_MSG_ERROR([--enable-markdown must be one of: yes, no, auto])]) -AC_SUBST(HAVE_LOWDOWN, [$have_lowdown]) + [AC_MSG_ERROR([bad value "$enable_markdown" for --enable-markdown, must be one of: yes, no, auto])]) # Look for libgit2. diff --git a/flake.nix b/flake.nix index 32354a88f..c7aee7541 100644 --- a/flake.nix +++ b/flake.nix @@ -230,6 +230,15 @@ } ); + # Toggles some settings for better coverage. Windows needs these + # library combinations, and Debian build Nix with GNU readline too. + buildReadlineNoMarkdown = forAllSystems (system: + self.packages.${system}.nix.override { + enableMarkdown = false; + readlineFlavor = "readline"; + } + ); + # Perl bindings for various platforms. perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings); diff --git a/package.nix b/package.nix index 727f5e646..d0b9fc3f3 100644 --- a/package.nix +++ b/package.nix @@ -13,6 +13,7 @@ , changelog-d , curl , editline +, readline , fileset , flex , git @@ -71,6 +72,14 @@ # Whether to enable Markdown rendering in the Nix binary. , enableMarkdown ? !stdenv.hostPlatform.isWindows +# Which interactive line editor library to use for Nix's repl. +# +# Currently supported choices are: +# +# - editline (default) +# - readline +, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" + # Whether to compile `rl-next.md`, the release notes for the next # not-yet-released version of Nix in the manul, from the individual # change log entries in the directory. @@ -219,7 +228,7 @@ in { sqlite xz ] ++ lib.optionals (!stdenv.hostPlatform.isWindows) [ - editline + ({ inherit readline editline; }.${readlineFlavor}) ] ++ lib.optionals enableMarkdown [ lowdown ] ++ lib.optionals buildUnitTests [ @@ -279,6 +288,7 @@ in { (lib.enableFeature enableManual "doc-gen") (lib.enableFeature enableMarkdown "markdown") (lib.enableFeature installUnitTests "install-unit-tests") + (lib.withFeatureAs true "readline-flavor" readlineFlavor) ] ++ lib.optionals (!forDevShell) [ "--sysconfdir=/etc" ] ++ lib.optionals installUnitTests [ diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index dea91ba63..9c92f2b6e 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -5,7 +5,7 @@ #include -#ifdef READLINE +#ifdef USE_READLINE #include #include #else @@ -249,14 +249,14 @@ void NixRepl::mainLoop() } catch (SysError & e) { logWarning(e.info()); } -#ifndef READLINE +#ifndef USE_READLINE el_hist_size = 1000; #endif read_history(historyFile.c_str()); auto oldRepl = curRepl; curRepl = this; Finally restoreRepl([&] { curRepl = oldRepl; }); -#ifndef READLINE +#ifndef USE_READLINE rl_set_complete_func(completionCallback); rl_set_list_possib_func(listPossibleCallback); #endif From 0c3ce237549d43de52e897f12e6d6c8ee59ac227 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 9 Jan 2024 17:31:40 -0500 Subject: [PATCH 902/909] Improve the build without GC We don't just want to pass `--enable-gc=no`; we also want to make sure boehmgc is not a dependency. Creating a nix-level configuration option to do both, and then using that for the CI job, is more robust. --- flake.nix | 4 +++- package.nix | 12 ++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index c7aee7541..49f214e72 100644 --- a/flake.nix +++ b/flake.nix @@ -220,7 +220,9 @@ buildCross = forAllCrossSystems (crossSystem: lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}")); - buildNoGc = forAllSystems (system: self.packages.${system}.nix.overrideAttrs (a: { configureFlags = (a.configureFlags or []) ++ ["--enable-gc=no"];})); + buildNoGc = forAllSystems (system: + self.packages.${system}.nix.override { enableGC = false; } + ); buildNoTests = forAllSystems (system: self.packages.${system}.nix.override { diff --git a/package.nix b/package.nix index d0b9fc3f3..71ee80e33 100644 --- a/package.nix +++ b/package.nix @@ -69,6 +69,14 @@ # Whether to build the regular manual , enableManual ? __forDefaults.canRunInstalled +# Whether to use garbage collection for the Nix language evaluator. +# +# If it is disabled, we just leak memory, but this is not as bad as it +# sounds so long as evaluation just takes places within short-lived +# processes. (When the process exits, the memory is reclaimed; it is +# only leaked *within* the process.) +, enableGC ? true + # Whether to enable Markdown rendering in the Nix binary. , enableMarkdown ? !stdenv.hostPlatform.isWindows @@ -245,9 +253,8 @@ in { ; propagatedBuildInputs = [ - boehmgc nlohmann_json - ]; + ] ++ lib.optional enableGC boehmgc; dontBuild = !attrs.doBuild; doCheck = attrs.doCheck; @@ -286,6 +293,7 @@ in { (lib.enableFeature doInstallCheck "functional-tests") (lib.enableFeature enableInternalAPIDocs "internal-api-docs") (lib.enableFeature enableManual "doc-gen") + (lib.enableFeature enableGC "gc") (lib.enableFeature enableMarkdown "markdown") (lib.enableFeature installUnitTests "install-unit-tests") (lib.withFeatureAs true "readline-flavor" readlineFlavor) From 57dc4fc878bc74dfb38cd9d435a85c560b43cebb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 2 Sep 2023 16:21:44 -0400 Subject: [PATCH 903/909] Make more expressive `HOST_*` macro system --- mk/lib.mk | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mk/lib.mk b/mk/lib.mk index 3d503364f..a5a067e48 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -14,20 +14,34 @@ install-tests-groups := ifdef HOST_OS HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS))) + ifeq ($(patsubst mingw%,,$(HOST_KERNEL)),) + HOST_MINGW = 1 + HOST_WINDOWS = 1 + endif ifeq ($(HOST_KERNEL), cygwin) HOST_CYGWIN = 1 + HOST_WINDOWS = 1 + HOST_UNIX = 1 endif ifeq ($(patsubst darwin%,,$(HOST_KERNEL)),) HOST_DARWIN = 1 + HOST_UNIX = 1 endif ifeq ($(patsubst freebsd%,,$(HOST_KERNEL)),) HOST_FREEBSD = 1 + HOST_UNIX = 1 + endif + ifeq ($(patsubst netbsd%,,$(HOST_KERNEL)),) + HOST_NETBSD = 1 + HOST_UNIX = 1 endif ifeq ($(HOST_KERNEL), linux) HOST_LINUX = 1 + HOST_UNIX = 1 endif ifeq ($(patsubst solaris%,,$(HOST_KERNEL)),) HOST_SOLARIS = 1 + HOST_UNIX = 1 endif endif From f9e5eb5f0a61555d24fe85b8edccf49f0b176152 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 10 Jan 2024 20:26:02 -0500 Subject: [PATCH 904/909] Make indentation in makesfiles consistent Tab (as required) for rules, two spaces for `if`...`endif`. --- src/libexpr/local.mk | 2 +- src/libstore/local.mk | 12 ++++++------ src/libutil/local.mk | 2 +- tests/functional/local.mk | 10 +++++----- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index ed6bc761a..b60936a0e 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -18,7 +18,7 @@ libexpr_LIBS = libutil libstore libfetchers libexpr_LDFLAGS += -lboost_context -pthread ifdef HOST_LINUX - libexpr_LDFLAGS += -ldl + libexpr_LDFLAGS += -ldl endif # The dependency on libgc must be propagated (i.e. meaning that diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 675976314..f447e190d 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -16,15 +16,15 @@ endif $(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox))) ifeq ($(ENABLE_S3), 1) - libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-crt-cpp + libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-crt-cpp endif ifdef HOST_SOLARIS - libstore_LDFLAGS += -lsocket + libstore_LDFLAGS += -lsocket endif ifeq ($(HAVE_SECCOMP), 1) - libstore_LDFLAGS += $(LIBSECCOMP_LIBS) + libstore_LDFLAGS += $(LIBSECCOMP_LIBS) endif libstore_CXXFLAGS += \ @@ -48,9 +48,9 @@ $(d)/embedded-sandbox-shell.gen.hh: $(sandbox_shell) $(trace-gen) hexdump -v -e '1/1 "0x%x," "\n"' < $< > $@.tmp @mv $@.tmp $@ else -ifneq ($(sandbox_shell),) -libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\"" -endif + ifneq ($(sandbox_shell),) + libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\"" + endif endif $(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 0fdebaf5c..6e3d6d052 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -17,5 +17,5 @@ $(foreach i, $(wildcard $(d)/signature/*.hh), \ ifeq ($(HAVE_LIBCPUID), 1) - libutil_LDFLAGS += -lcpuid + libutil_LDFLAGS += -lcpuid endif diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 192e275e3..25fcbcfe7 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -129,15 +129,15 @@ nix_tests = \ impure-env.sh ifeq ($(HAVE_LIBCPUID), 1) - nix_tests += compute-levels.sh + nix_tests += compute-levels.sh endif ifeq ($(ENABLE_BUILD), yes) - nix_tests += test-libstoreconsumer.sh + nix_tests += test-libstoreconsumer.sh - ifeq ($(BUILD_SHARED_LIBS), 1) - nix_tests += plugins.sh - endif + ifeq ($(BUILD_SHARED_LIBS), 1) + nix_tests += plugins.sh + endif endif $(d)/test-libstoreconsumer.sh.test $(d)/test-libstoreconsumer.sh.test-debug: \ From 423484ad26850046c101affc9ff6ac4c36ccda06 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 9 Jan 2024 12:29:37 -0500 Subject: [PATCH 905/909] Only link with `-pthread` on Unix We don't want this with MinGW. --- mk/libraries.mk | 6 ++++++ src/libcmd/local.mk | 2 +- src/libexpr/local.mk | 2 +- src/libfetchers/local.mk | 2 +- src/libstore/local.mk | 2 +- src/libutil/local.mk | 2 +- src/nix/local.mk | 2 +- tests/functional/test-libstoreconsumer/local.mk | 2 +- tests/unit/libexpr-support/local.mk | 2 +- tests/unit/libstore-support/local.mk | 2 +- tests/unit/libutil-support/local.mk | 2 +- 11 files changed, 16 insertions(+), 10 deletions(-) diff --git a/mk/libraries.mk b/mk/libraries.mk index 1bc73d7f7..515a481f6 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -10,6 +10,12 @@ else endif endif +ifdef HOST_UNIX + THREAD_LDFLAGS = -pthread +else + THREAD_LDFLAGS = +endif + # Build a library with symbolic name $(1). The library is defined by # various variables prefixed by ‘$(1)_’: # diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index afd35af08..abb7459a7 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -8,7 +8,7 @@ libcmd_SOURCES := $(wildcard $(d)/*.cc) libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers -libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread +libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS) libcmd_LIBS = libstore libutil libexpr libmain libfetchers diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index b60936a0e..0c3e36750 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib libexpr_LIBS = libutil libstore libfetchers -libexpr_LDFLAGS += -lboost_context -pthread +libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS) ifdef HOST_LINUX libexpr_LDFLAGS += -ldl endif diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk index 266e7a211..e54db4937 100644 --- a/src/libfetchers/local.mk +++ b/src/libfetchers/local.mk @@ -8,6 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc) libfetchers_CXXFLAGS += -I src/libutil -I src/libstore -libfetchers_LDFLAGS += -pthread $(LIBGIT2_LIBS) -larchive +libfetchers_LDFLAGS += $(THREAD_LDFLAGS) $(LIBGIT2_LIBS) -larchive libfetchers_LIBS = libutil libstore diff --git a/src/libstore/local.mk b/src/libstore/local.mk index f447e190d..f86643849 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -8,7 +8,7 @@ libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc) libstore_LIBS = libutil -libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) -pthread +libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) $(THREAD_LDFLAGS) ifdef HOST_LINUX libstore_LDFLAGS += -ldl endif diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 6e3d6d052..200026c1e 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -8,7 +8,7 @@ libutil_SOURCES := $(wildcard $(d)/*.cc $(d)/signature/*.cc) libutil_CXXFLAGS += -I src/libutil -libutil_LDFLAGS += -pthread $(LIBCURL_LIBS) $(SODIUM_LIBS) $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context +libutil_LDFLAGS += $(THREAD_LDFLAGS) $(LIBCURL_LIBS) $(SODIUM_LIBS) $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context $(foreach i, $(wildcard $(d)/args/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/args, 0644))) diff --git a/src/nix/local.mk b/src/nix/local.mk index a21aa705f..1d6f560d6 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -18,7 +18,7 @@ nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr nix_LIBS = libexpr libmain libfetchers libstore libutil libcmd -nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS) +nix_LDFLAGS = $(THREAD_LDFLAGS) $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS) $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ diff --git a/tests/functional/test-libstoreconsumer/local.mk b/tests/functional/test-libstoreconsumer/local.mk index edc140723..a1825c405 100644 --- a/tests/functional/test-libstoreconsumer/local.mk +++ b/tests/functional/test-libstoreconsumer/local.mk @@ -12,4 +12,4 @@ test-libstoreconsumer_CXXFLAGS += -I src/libutil -I src/libstore test-libstoreconsumer_LIBS = libstore libutil -test-libstoreconsumer_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS) +test-libstoreconsumer_LDFLAGS = $(THREAD_LDFLAGS) $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS) diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk index 12a76206a..0501de33c 100644 --- a/tests/unit/libexpr-support/local.mk +++ b/tests/unit/libexpr-support/local.mk @@ -20,4 +20,4 @@ libexpr-test-support_LIBS = \ libstore-test-support libutil-test-support \ libexpr libstore libutil -libexpr-test-support_LDFLAGS := -pthread -lrapidcheck +libexpr-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk index ff075c96a..56dedd825 100644 --- a/tests/unit/libstore-support/local.mk +++ b/tests/unit/libstore-support/local.mk @@ -18,4 +18,4 @@ libstore-test-support_LIBS = \ libutil-test-support \ libstore libutil -libstore-test-support_LDFLAGS := -pthread -lrapidcheck +libstore-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk index 2ee2cdb6c..5f7835c9f 100644 --- a/tests/unit/libutil-support/local.mk +++ b/tests/unit/libutil-support/local.mk @@ -16,4 +16,4 @@ libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) libutil-test-support_LIBS = libutil -libutil-test-support_LDFLAGS := -pthread -lrapidcheck +libutil-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck From a923444a9462cd2fabcd816fa2e9cb54c485f13f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 11 Jan 2024 13:55:28 -0500 Subject: [PATCH 906/909] packages.nix: Fix `installUnitTests` condition The intent was we install the tests when we can *not* run them. Instead, we were installing them when we can. --- package.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package.nix b/package.nix index 71ee80e33..37410dc2f 100644 --- a/package.nix +++ b/package.nix @@ -100,7 +100,7 @@ # Whether to install unit tests. This is useful when cross compiling # since we cannot run them natively during the build, but can do so # later. -, installUnitTests ? __forDefaults.canRunInstalled +, installUnitTests ? doBuild && !__forDefaults.canExecuteHost # For running the functional tests against a pre-built Nix. Probably # want to use in conjunction with `doBuild = false;`. @@ -113,7 +113,8 @@ # Not a real argument, just the only way to approximate let-binding some # stuff for argument defaults. , __forDefaults ? { - canRunInstalled = doBuild && stdenv.buildPlatform.canExecute stdenv.hostPlatform; + canExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + canRunInstalled = doBuild && __forDefaults.canExecuteHost; } }: From 34bb6dcab1334ebc6ac0afaf4fe6f9e6f13de4b5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 11 Jan 2024 11:14:13 -0500 Subject: [PATCH 907/909] makefiles: Support `.exe` executable prefix on Windows --- mk/programs.mk | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mk/programs.mk b/mk/programs.mk index 6235311e9..623caaf55 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -1,5 +1,11 @@ programs-list := +ifdef HOST_WINDOWS + EXE_EXT = .exe +else + EXE_EXT = +endif + # Build a program with symbolic name $(1). The program is defined by # various variables prefixed by ‘$(1)_’: # @@ -31,7 +37,7 @@ define build-program _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src))) $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs)))) _libs := $$(foreach lib, $$($(1)_LIBS), $$(foreach lib2, $$($$(lib)_LIB_CLOSURE), $$($$(lib2)_PATH))) - $(1)_PATH := $$(_d)/$$($(1)_NAME) + $(1)_PATH := $$(_d)/$$($(1)_NAME)$(EXE_EXT) $$(eval $$(call create-dir, $$(_d))) @@ -42,7 +48,7 @@ define build-program ifdef $(1)_INSTALL_DIR - $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME) + $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME)$(EXE_EXT) $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) From af0345df3688494d1e53a659eacb16fc4b9915b5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 11 Jan 2024 11:14:13 -0500 Subject: [PATCH 908/909] makefiles: Do some HOST_CYGWIN -> HOST_WINDOWS These bits are not Cygwin-specific. --- mk/libraries.mk | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mk/libraries.mk b/mk/libraries.mk index 515a481f6..b99ba2782 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -3,7 +3,7 @@ libs-list := ifdef HOST_DARWIN SO_EXT = dylib else - ifdef HOST_CYGWIN + ifdef HOST_WINDOWS SO_EXT = dll else SO_EXT = so @@ -65,7 +65,7 @@ define build-library $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs)))) _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH)) - ifdef HOST_CYGWIN + ifdef HOST_WINDOWS $(1)_INSTALL_DIR ?= $$(bindir) else $(1)_INSTALL_DIR ?= $$(libdir) @@ -85,7 +85,7 @@ define build-library endif else ifndef HOST_DARWIN - ifndef HOST_CYGWIN + ifndef HOST_WINDOWS $(1)_LDFLAGS += -Wl,-z,defs endif endif From 90fdbfc601a8d005f57c984284c5922dc38480eb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 9 Jan 2024 12:41:42 -0500 Subject: [PATCH 909/909] Build Windows DLLs with `-Wl,--export-all-symbols` This is not the most elegant, but will match the SOs in exporting everything for now. Later we can refine what is public/private to clean up the interface. --- Makefile | 37 ++++++++++++++++++++++++++++++++----- mk/lib.mk | 33 +-------------------------------- mk/platform.mk | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 37 deletions(-) create mode 100644 mk/platform.mk diff --git a/Makefile b/Makefile index 1fdb6e897..7bbfbddbe 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,12 @@ +# External build directory support + include mk/build-dir.mk -include $(buildprefix)Makefile.config clean-files += $(buildprefix)Makefile.config +# List makefiles + ifeq ($(ENABLE_BUILD), yes) makefiles = \ mk/precompiled-headers.mk \ @@ -43,6 +47,8 @@ makefiles += \ tests/functional/plugins/local.mk endif +# Miscellaneous global Flags + OPTIMIZE = 1 ifeq ($(OPTIMIZE), 1) @@ -52,9 +58,29 @@ else GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE endif +include mk/platform.mk + +ifdef HOST_WINDOWS + # Windows DLLs are stricter about symbol visibility than Unix shared + # objects --- see https://gcc.gnu.org/wiki/Visibility for details. + # This is a temporary sledgehammer to export everything like on Unix, + # and not detail with this yet. + # + # TODO do not do this, and instead do fine-grained export annotations. + GLOBAL_LDFLAGS += -Wl,--export-all-symbols +endif + +GLOBAL_CXXFLAGS += -g -Wall -include $(buildprefix)config.h -std=c++2a -I src + +# Include the main lib, causing rules to be defined + include mk/lib.mk -# Must be included after `mk/lib.mk` so isn't the default target. +# Fallback stub rules for better UX when things are disabled +# +# These must be defined after `mk/lib.mk`. Otherwise the first rule +# incorrectly becomes the default target. + ifneq ($(ENABLE_UNIT_TESTS), yes) .PHONY: check check: @@ -69,8 +95,11 @@ installcheck: @exit 1 endif -# Must be included after `mk/lib.mk` so rules refer to variables defined -# by the library. Rules are not "lazy" like variables, unfortunately. +# Documentation or else fallback stub rules. +# +# The documentation makefiles be included after `mk/lib.mk` so rules +# refer to variables defined by `mk/lib.mk`. Rules are not "lazy" like +# variables, unfortunately. ifeq ($(ENABLE_DOC_GEN), yes) $(eval $(call include-sub-makefile, doc/manual/local.mk)) @@ -89,5 +118,3 @@ internal-api-html: @echo "Internal API docs are disabled. Configure with '--enable-internal-api-docs', or avoid calling 'make internal-api-html'." @exit 1 endif - -GLOBAL_CXXFLAGS += -g -Wall -include $(buildprefix)config.h -std=c++2a -I src diff --git a/mk/lib.mk b/mk/lib.mk index a5a067e48..10ce8d436 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -12,38 +12,7 @@ man-pages := install-tests := install-tests-groups := -ifdef HOST_OS - HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS))) - ifeq ($(patsubst mingw%,,$(HOST_KERNEL)),) - HOST_MINGW = 1 - HOST_WINDOWS = 1 - endif - ifeq ($(HOST_KERNEL), cygwin) - HOST_CYGWIN = 1 - HOST_WINDOWS = 1 - HOST_UNIX = 1 - endif - ifeq ($(patsubst darwin%,,$(HOST_KERNEL)),) - HOST_DARWIN = 1 - HOST_UNIX = 1 - endif - ifeq ($(patsubst freebsd%,,$(HOST_KERNEL)),) - HOST_FREEBSD = 1 - HOST_UNIX = 1 - endif - ifeq ($(patsubst netbsd%,,$(HOST_KERNEL)),) - HOST_NETBSD = 1 - HOST_UNIX = 1 - endif - ifeq ($(HOST_KERNEL), linux) - HOST_LINUX = 1 - HOST_UNIX = 1 - endif - ifeq ($(patsubst solaris%,,$(HOST_KERNEL)),) - HOST_SOLARIS = 1 - HOST_UNIX = 1 - endif -endif +include mk/platform.mk # Hack to define a literal space. space := diff --git a/mk/platform.mk b/mk/platform.mk new file mode 100644 index 000000000..fe960dedf --- /dev/null +++ b/mk/platform.mk @@ -0,0 +1,32 @@ +ifdef HOST_OS + HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS))) + ifeq ($(patsubst mingw%,,$(HOST_KERNEL)),) + HOST_MINGW = 1 + HOST_WINDOWS = 1 + endif + ifeq ($(HOST_KERNEL), cygwin) + HOST_CYGWIN = 1 + HOST_WINDOWS = 1 + HOST_UNIX = 1 + endif + ifeq ($(patsubst darwin%,,$(HOST_KERNEL)),) + HOST_DARWIN = 1 + HOST_UNIX = 1 + endif + ifeq ($(patsubst freebsd%,,$(HOST_KERNEL)),) + HOST_FREEBSD = 1 + HOST_UNIX = 1 + endif + ifeq ($(patsubst netbsd%,,$(HOST_KERNEL)),) + HOST_NETBSD = 1 + HOST_UNIX = 1 + endif + ifeq ($(HOST_KERNEL), linux) + HOST_LINUX = 1 + HOST_UNIX = 1 + endif + ifeq ($(patsubst solaris%,,$(HOST_KERNEL)),) + HOST_SOLARIS = 1 + HOST_UNIX = 1 + endif +endif