diff --git a/doc/manual/advanced-topics/advanced-topics.xml b/doc/manual/advanced-topics/advanced-topics.xml index 12a82625..871b7eb1 100644 --- a/doc/manual/advanced-topics/advanced-topics.xml +++ b/doc/manual/advanced-topics/advanced-topics.xml @@ -9,5 +9,6 @@ + diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml new file mode 100644 index 00000000..3dc43ee7 --- /dev/null +++ b/doc/manual/advanced-topics/post-build-hook.xml @@ -0,0 +1,160 @@ + + +Using the <xref linkend="conf-post-build-hook" /> +Uploading to an S3-compatible binary cache after each build + + +
+ Implementation Caveats + Here we use the post-build hook to upload to a binary cache. + This is a simple and working example, but it is not suitable for all + use cases. + + The post build hook program runs after each executed build, + and blocks the build loop. The build loop exits if the hook program + fails. + + Concretely, this implementation will make Nix slow or unusable + when the internet is slow or unreliable. + + A more advanced implementation might pass the store paths to a + user-supplied daemon or queue for processing the store paths outside + of the build loop. +
+ +
+ Prerequisites + + + This tutorial assumes you have configured an S3-compatible binary cache + according to the instructions at + , and + that the root user's default AWS profile can + upload to the bucket. + +
+ +
+ Set up a Signing Key + Use nix-store --generate-binary-cache-key to + create our public and private signing keys. We will sign paths + with the private key, and distribute the public key for verifying + the authenticity of the paths. + + +# nix-store --generate-binary-cache-key example-nix-cache-1 /etc/nix/key.private /etc/nix/key.public +# cat /etc/nix/key.public +example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= + + +Then, add the public key and the cache URL to your +nix.conf's +and like: + + +substituters = https://cache.nixos.org/ s3://example-nix-cache +trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= + + +we will restart the Nix daemon a later step. +
+ +
+ Implementing the build hook + Write the following script to + /etc/nix/upload-to-cache.sh: + + + +#!/bin/sh + +set -eu +set -f # disable globbing +export IFS=' ' + +echo "Signing paths" $OUT_PATHS +nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS +echo "Uploading paths" $OUT_PATHS +exec nix copy --to 's3://example-nix-cache' $OUT_PATHS + + + + Should <literal>$OUT_PATHS</literal> be quoted? + + The $OUT_PATHS variable is a space-separated + list of Nix store paths. In this case, we expect and want the + shell to perform word splitting to make each output path its + own argument to nix sign-paths. Nix guarantees + the paths will not contain any spaces, however a store path + might contain glob characters. The set -f + disables globbing in the shell. + + + + Then make sure the hook program is executable by the root user: + +# chmod +x /etc/nix/upload-to-cache.sh + +
+ +
+ Updating Nix Configuration + + Edit /etc/nix/nix.conf to run our hook, + by adding the following configuration snippet at the end: + + +post-build-hook = /etc/nix/upload-to-cache.sh + + +Then, restart the nix-daemon. +
+ +
+ Testing + + Build any derivation, for example: + + +$ nix-build -E '(import <nixpkgs> {}).writeText "example" (builtins.toString builtins.currentTime)' +these derivations will be built: + /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv +building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'... +running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'... +post-build-hook: Signing paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +post-build-hook: Uploading paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +/nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example + + + Then delete the path from the store, and try substituting it from the binary cache: + +$ rm ./result +$ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example + + +Now, copy the path back from the cache: + +$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'... +warning: you did not specify '--add-root'; the result might be removed by the garbage collector +/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example + +
+
+ Conclusion + + We now have a Nix installation configured to automatically sign and + upload every local build to a remote binary cache. + + + + Before deploying this to production, be sure to consider the + implementation caveats in . + +
+ diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 0a97719e..f3c721c7 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -664,6 +664,62 @@ password my-password + + post-build-hook + + Optional. The path to a program to execute after each build. + + This option is only settable in the global + nix.conf, or on the command line by trusted + users. + + When using the nix-daemon, the daemon executes the hook as + root. If the nix-daemon is not involved, the + hook runs as the user executing the nix-build. + + + The hook executes after an evaluation-time build. + The hook does not execute on substituted paths. + The hook's output always goes to the user's terminal. + If the hook fails, the build succeeds but no further builds execute. + The hook executes synchronously, and blocks other builds from progressing while it runs. + + + The program executes with no arguments. The program's environment + contains the following environment variables: + + + + DRV_PATH + + The derivation for the built paths. + Example: + /nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv + + + + + + OUT_PATHS + + Output paths of the built derivation, separated by a space character. + Example: + /nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev + /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc + /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info + /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man + /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23. + + + + + + See for an example + implementation. + + + + repeat How many times to repeat builds to check whether diff --git a/src/libstore/build.cc b/src/libstore/build.cc index a28619ab..4bec37e0 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1629,6 +1629,61 @@ void DerivationGoal::buildDone() being valid. */ registerOutputs(); + if (settings.postBuildHook != "") { + Activity act(*logger, lvlInfo, actPostBuildHook, + fmt("running post-build-hook '%s'", settings.postBuildHook), + Logger::Fields{drvPath}); + PushActivity pact(act.id); + auto outputPaths = drv->outputPaths(); + std::map hookEnvironment = getEnv(); + + hookEnvironment.emplace("DRV_PATH", drvPath); + hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", outputPaths))); + + RunOptions opts(settings.postBuildHook, {}); + opts.environment = hookEnvironment; + + struct LogSink : Sink { + Activity & act; + std::string currentLine; + + LogSink(Activity & act) : act(act) { } + + void operator() (const unsigned char * data, size_t len) override { + for (size_t i = 0; i < len; i++) { + auto c = data[i]; + + if (c == '\n') { + flushLine(); + } else { + currentLine += c; + } + } + } + + void flushLine() { + if (settings.verboseBuild) { + printError("post-build-hook: " + currentLine); + } else { + act.result(resPostBuildLogLine, currentLine); + } + currentLine.clear(); + } + + ~LogSink() { + if (currentLine != "") { + currentLine += '\n'; + flushLine(); + } + } + }; + LogSink sink(act); + + opts.standardOut = &sink; + opts.mergeStderrToStdout = true; + runProgram2(opts); + } + if (buildMode == bmCheck) { done(BuildResult::Built); return; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 0c0a9ec5..13effb50 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -315,6 +315,9 @@ public: "pre-build-hook", "A program to run just before a build to set derivation-specific build settings."}; + Setting postBuildHook{this, "", "post-build-hook", + "A program to run just after each succesful build."}; + Setting netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", "Path to the netrc file used to obtain usernames/passwords for downloads."}; diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 5f221944..5df03da7 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -26,6 +26,7 @@ typedef enum { actVerifyPaths = 107, actSubstitute = 108, actQueryPathInfo = 109, + actPostBuildHook = 110, } ActivityType; typedef enum { @@ -36,6 +37,7 @@ typedef enum { resSetPhase = 104, resProgress = 105, resSetExpected = 106, + resPostBuildLogLine = 107, } ResultType; typedef uint64_t ActivityId; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 17aee2d5..44fa7248 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -84,6 +84,15 @@ void clearEnv() unsetenv(name.first.c_str()); } +void replaceEnv(std::map newEnv) +{ + clearEnv(); + for (auto newEnvVar : newEnv) + { + setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); + } +} + Path absPath(Path path, Path dir) { @@ -1019,10 +1028,22 @@ void runProgram2(const RunOptions & options) if (options.standardOut) out.create(); if (source) in.create(); + ProcessOptions processOptions; + // vfork implies that the environment of the main process and the fork will + // be shared (technically this is undefined, but in practice that's the + // case), so we can't use it if we alter the environment + if (options.environment) + processOptions.allowVfork = false; + /* Fork. */ Pid pid = startProcess([&]() { + if (options.environment) + replaceEnv(*options.environment); if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("dupping stdout"); + if (options.mergeStderrToStdout) + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) throw SysError("dupping stdin"); @@ -1047,7 +1068,7 @@ void runProgram2(const RunOptions & options) execv(options.program.c_str(), stringsToCharPtrs(args_).data()); throw SysError("executing '%1%'", options.program); - }); + }, processOptions); out.writeSide = -1; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index fce3cab8..b538a0b4 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -270,12 +270,14 @@ struct RunOptions std::optional uid; std::optional gid; std::optional chdir; + std::optional> environment; Path program; bool searchPath = true; Strings args; std::optional input; Source * standardIn = nullptr; Sink * standardOut = nullptr; + bool mergeStderrToStdout = false; bool _killStderr = false; RunOptions(const Path & program, const Strings & args) diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index b1c1d87d..c0bcfb0c 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -170,6 +170,14 @@ public: name, sub); } + if (type == actPostBuildHook) { + auto name = storePathToName(getS(fields, 0)); + if (hasSuffix(name, ".drv")) + name.resize(name.size() - 4); + i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name); + i->name = DrvName(name).name; + } + if (type == actQueryPathInfo) { auto name = storePathToName(getS(fields, 0)); i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1)); @@ -228,14 +236,18 @@ public: update(*state); } - else if (type == resBuildLogLine) { + else if (type == resBuildLogLine || type == resPostBuildLogLine) { auto lastLine = trim(getS(fields, 0)); if (!lastLine.empty()) { auto i = state->its.find(act); assert(i != state->its.end()); ActInfo info = *i->second; if (printBuildLogs) { - log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + "> " + ANSI_NORMAL + lastLine); + auto suffix = "> "; + if (type == resPostBuildLogLine) { + suffix = " (post)> "; + } + log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + suffix + ANSI_NORMAL + lastLine); } else { state->activities.erase(i->second); info.lastLine = lastLine; diff --git a/tests/dependencies.nix b/tests/dependencies.nix index 687237ad..eca4b296 100644 --- a/tests/dependencies.nix +++ b/tests/dependencies.nix @@ -17,6 +17,7 @@ let { builder = ./dependencies.builder0.sh + "/FOOBAR/../."; input1 = input1 + "/."; input2 = "${input2}/."; + input1_drv = input1; meta.description = "Random test package"; }; diff --git a/tests/local.mk b/tests/local.mk index 261e4db0..ef359c63 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -28,7 +28,8 @@ nix_tests = \ check.sh \ plugins.sh \ search.sh \ - nix-copy-ssh.sh + nix-copy-ssh.sh \ + post-hook.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) diff --git a/tests/post-hook.sh b/tests/post-hook.sh new file mode 100644 index 00000000..3f82d0c5 --- /dev/null +++ b/tests/post-hook.sh @@ -0,0 +1,15 @@ +source common.sh + +clearStore + +export REMOTE_STORE=$TEST_ROOT/remote_store + +# Build the dependencies and push them to the remote store +nix-build dependencies.nix --post-build-hook $PWD/push_to_store.sh + +clearStore + +# Ensure that we the remote store contains both the runtime and buildtime +# closure of what we've just built +nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix +nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix input1_drv diff --git a/tests/push_to_store.sh b/tests/push_to_store.sh new file mode 100755 index 00000000..d97eb095 --- /dev/null +++ b/tests/push_to_store.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +echo Pushing "$@" to "$REMOTE_STORE" +echo -n "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs