Merge pull request #2995 from tweag/post-build-hook
Add a post build hook
This commit is contained in:
Eelco Dolstra
GitHub
commit
56df30cd3f
|
|
@ -9,5 +9,6 @@
|
|||
<xi:include href="distributed-builds.xml" />
|
||||
<xi:include href="cores-vs-jobs.xml" />
|
||||
<xi:include href="diff-hook.xml" />
|
||||
<xi:include href="post-build-hook.xml" />
|
||||
|
||||
</part>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,160 @@
|
|||
<chapter xmlns="http://docbook.org/ns/docbook"
|
||||
xmlns:xlink="http://www.w3.org/1999/xlink"
|
||||
xmlns:xi="http://www.w3.org/2001/XInclude"
|
||||
xml:id="chap-post-build-hook"
|
||||
version="5.0"
|
||||
>
|
||||
|
||||
<title>Using the <xref linkend="conf-post-build-hook" /></title>
|
||||
<subtitle>Uploading to an S3-compatible binary cache after each build</subtitle>
|
||||
|
||||
|
||||
<section xml:id="chap-post-build-hook-caveats">
|
||||
<title>Implementation Caveats</title>
|
||||
<para>Here we use the post-build hook to upload to a binary cache.
|
||||
This is a simple and working example, but it is not suitable for all
|
||||
use cases.</para>
|
||||
|
||||
<para>The post build hook program runs after each executed build,
|
||||
and blocks the build loop. The build loop exits if the hook program
|
||||
fails.</para>
|
||||
|
||||
<para>Concretely, this implementation will make Nix slow or unusable
|
||||
when the internet is slow or unreliable.</para>
|
||||
|
||||
<para>A more advanced implementation might pass the store paths to a
|
||||
user-supplied daemon or queue for processing the store paths outside
|
||||
of the build loop.</para>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<title>Prerequisites</title>
|
||||
|
||||
<para>
|
||||
This tutorial assumes you have configured an S3-compatible binary cache
|
||||
according to the instructions at
|
||||
<xref linkend="ssec-s3-substituter-authenticated-writes" />, and
|
||||
that the <literal>root</literal> user's default AWS profile can
|
||||
upload to the bucket.
|
||||
</para>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<title>Set up a Signing Key</title>
|
||||
<para>Use <command>nix-store --generate-binary-cache-key</command> to
|
||||
create our public and private signing keys. We will sign paths
|
||||
with the private key, and distribute the public key for verifying
|
||||
the authenticity of the paths.</para>
|
||||
|
||||
<screen>
|
||||
# nix-store --generate-binary-cache-key example-nix-cache-1 /etc/nix/key.private /etc/nix/key.public
|
||||
# cat /etc/nix/key.public
|
||||
example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
|
||||
</screen>
|
||||
|
||||
<para>Then, add the public key and the cache URL to your
|
||||
<filename>nix.conf</filename>'s <xref linkend="conf-trusted-public-keys" />
|
||||
and <xref linkend="conf-substituters" /> like:</para>
|
||||
|
||||
<programlisting>
|
||||
substituters = https://cache.nixos.org/ s3://example-nix-cache
|
||||
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
|
||||
</programlisting>
|
||||
|
||||
<para>we will restart the Nix daemon a later step.</para>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<title>Implementing the build hook</title>
|
||||
<para>Write the following script to
|
||||
<filename>/etc/nix/upload-to-cache.sh</filename>:
|
||||
</para>
|
||||
|
||||
<programlisting>
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
set -f # disable globbing
|
||||
export IFS=' '
|
||||
|
||||
echo "Signing paths" $OUT_PATHS
|
||||
nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS
|
||||
echo "Uploading paths" $OUT_PATHS
|
||||
exec nix copy --to 's3://example-nix-cache' $OUT_PATHS
|
||||
</programlisting>
|
||||
|
||||
<note>
|
||||
<title>Should <literal>$OUT_PATHS</literal> be quoted?</title>
|
||||
<para>
|
||||
The <literal>$OUT_PATHS</literal> variable is a space-separated
|
||||
list of Nix store paths. In this case, we expect and want the
|
||||
shell to perform word splitting to make each output path its
|
||||
own argument to <command>nix sign-paths</command>. Nix guarantees
|
||||
the paths will not contain any spaces, however a store path
|
||||
might contain glob characters. The <command>set -f</command>
|
||||
disables globbing in the shell.
|
||||
</para>
|
||||
</note>
|
||||
<para>
|
||||
Then make sure the hook program is executable by the <literal>root</literal> user:
|
||||
<screen>
|
||||
# chmod +x /etc/nix/upload-to-cache.sh
|
||||
</screen></para>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<title>Updating Nix Configuration</title>
|
||||
|
||||
<para>Edit <filename>/etc/nix/nix.conf</filename> to run our hook,
|
||||
by adding the following configuration snippet at the end:</para>
|
||||
|
||||
<programlisting>
|
||||
post-build-hook = /etc/nix/upload-to-cache.sh
|
||||
</programlisting>
|
||||
|
||||
<para>Then, restart the <command>nix-daemon</command>.</para>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<title>Testing</title>
|
||||
|
||||
<para>Build any derivation, for example:</para>
|
||||
|
||||
<screen>
|
||||
$ nix-build -E '(import <nixpkgs> {}).writeText "example" (builtins.toString builtins.currentTime)'
|
||||
these derivations will be built:
|
||||
/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv
|
||||
building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'...
|
||||
running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'...
|
||||
post-build-hook: Signing paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
|
||||
post-build-hook: Uploading paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
|
||||
/nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
|
||||
</screen>
|
||||
|
||||
<para>Then delete the path from the store, and try substituting it from the binary cache:</para>
|
||||
<screen>
|
||||
$ rm ./result
|
||||
$ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
|
||||
</screen>
|
||||
|
||||
<para>Now, copy the path back from the cache:</para>
|
||||
<screen>
|
||||
$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
|
||||
copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'...
|
||||
warning: you did not specify '--add-root'; the result might be removed by the garbage collector
|
||||
/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example
|
||||
</screen>
|
||||
</section>
|
||||
<section>
|
||||
<title>Conclusion</title>
|
||||
<para>
|
||||
We now have a Nix installation configured to automatically sign and
|
||||
upload every local build to a remote binary cache.
|
||||
</para>
|
||||
|
||||
<para>
|
||||
Before deploying this to production, be sure to consider the
|
||||
implementation caveats in <xref linkend="chap-post-build-hook-caveats" />.
|
||||
</para>
|
||||
</section>
|
||||
</chapter>
|
||||
|
|
@ -664,6 +664,62 @@ password <replaceable>my-password</replaceable>
|
|||
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry xml:id="conf-post-build-hook">
|
||||
<term><literal>post-build-hook</literal></term>
|
||||
<listitem>
|
||||
<para>Optional. The path to a program to execute after each build.</para>
|
||||
|
||||
<para>This option is only settable in the global
|
||||
<filename>nix.conf</filename>, or on the command line by trusted
|
||||
users.</para>
|
||||
|
||||
<para>When using the nix-daemon, the daemon executes the hook as
|
||||
<literal>root</literal>. If the nix-daemon is not involved, the
|
||||
hook runs as the user executing the nix-build.</para>
|
||||
|
||||
<itemizedlist>
|
||||
<listitem><para>The hook executes after an evaluation-time build.</para></listitem>
|
||||
<listitem><para>The hook does not execute on substituted paths.</para></listitem>
|
||||
<listitem><para>The hook's output always goes to the user's terminal.</para></listitem>
|
||||
<listitem><para>If the hook fails, the build succeeds but no further builds execute.</para></listitem>
|
||||
<listitem><para>The hook executes synchronously, and blocks other builds from progressing while it runs.</para>
|
||||
</itemizedlist>
|
||||
|
||||
<para>The program executes with no arguments. The program's environment
|
||||
contains the following environment variables:</para>
|
||||
|
||||
<variablelist>
|
||||
<varlistentry>
|
||||
<term><envar>DRV_PATH</envar></term>
|
||||
<listitem>
|
||||
<para>The derivation for the built paths.</para>
|
||||
<para>Example:
|
||||
<literal>/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv</literal>
|
||||
</para>
|
||||
</listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry>
|
||||
<term><envar>OUT_PATHS</envar></term>
|
||||
<listitem>
|
||||
<para>Output paths of the built derivation, separated by a space character.</para>
|
||||
<para>Example:
|
||||
<literal>/nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev
|
||||
/nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc
|
||||
/nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info
|
||||
/nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man
|
||||
/nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23</literal>.
|
||||
</para>
|
||||
</listitem>
|
||||
</varlistentry>
|
||||
</variablelist>
|
||||
|
||||
<para>See <xref linkend="chap-post-build-hook" /> for an example
|
||||
implementation.</para>
|
||||
|
||||
</listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry xml:id="conf-repeat"><term><literal>repeat</literal></term>
|
||||
|
||||
<listitem><para>How many times to repeat builds to check whether
|
||||
|
|
|
|||
|
|
@ -1629,6 +1629,61 @@ void DerivationGoal::buildDone()
|
|||
being valid. */
|
||||
registerOutputs();
|
||||
|
||||
if (settings.postBuildHook != "") {
|
||||
Activity act(*logger, lvlInfo, actPostBuildHook,
|
||||
fmt("running post-build-hook '%s'", settings.postBuildHook),
|
||||
Logger::Fields{drvPath});
|
||||
PushActivity pact(act.id);
|
||||
auto outputPaths = drv->outputPaths();
|
||||
std::map<std::string, std::string> hookEnvironment = getEnv();
|
||||
|
||||
hookEnvironment.emplace("DRV_PATH", drvPath);
|
||||
hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", outputPaths)));
|
||||
|
||||
RunOptions opts(settings.postBuildHook, {});
|
||||
opts.environment = hookEnvironment;
|
||||
|
||||
struct LogSink : Sink {
|
||||
Activity & act;
|
||||
std::string currentLine;
|
||||
|
||||
LogSink(Activity & act) : act(act) { }
|
||||
|
||||
void operator() (const unsigned char * data, size_t len) override {
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
auto c = data[i];
|
||||
|
||||
if (c == '\n') {
|
||||
flushLine();
|
||||
} else {
|
||||
currentLine += c;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void flushLine() {
|
||||
if (settings.verboseBuild) {
|
||||
printError("post-build-hook: " + currentLine);
|
||||
} else {
|
||||
act.result(resPostBuildLogLine, currentLine);
|
||||
}
|
||||
currentLine.clear();
|
||||
}
|
||||
|
||||
~LogSink() {
|
||||
if (currentLine != "") {
|
||||
currentLine += '\n';
|
||||
flushLine();
|
||||
}
|
||||
}
|
||||
};
|
||||
LogSink sink(act);
|
||||
|
||||
opts.standardOut = &sink;
|
||||
opts.mergeStderrToStdout = true;
|
||||
runProgram2(opts);
|
||||
}
|
||||
|
||||
if (buildMode == bmCheck) {
|
||||
done(BuildResult::Built);
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -315,6 +315,9 @@ public:
|
|||
"pre-build-hook",
|
||||
"A program to run just before a build to set derivation-specific build settings."};
|
||||
|
||||
Setting<std::string> postBuildHook{this, "", "post-build-hook",
|
||||
"A program to run just after each succesful build."};
|
||||
|
||||
Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file",
|
||||
"Path to the netrc file used to obtain usernames/passwords for downloads."};
|
||||
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ typedef enum {
|
|||
actVerifyPaths = 107,
|
||||
actSubstitute = 108,
|
||||
actQueryPathInfo = 109,
|
||||
actPostBuildHook = 110,
|
||||
} ActivityType;
|
||||
|
||||
typedef enum {
|
||||
|
|
@ -36,6 +37,7 @@ typedef enum {
|
|||
resSetPhase = 104,
|
||||
resProgress = 105,
|
||||
resSetExpected = 106,
|
||||
resPostBuildLogLine = 107,
|
||||
} ResultType;
|
||||
|
||||
typedef uint64_t ActivityId;
|
||||
|
|
|
|||
|
|
@ -84,6 +84,15 @@ void clearEnv()
|
|||
unsetenv(name.first.c_str());
|
||||
}
|
||||
|
||||
void replaceEnv(std::map<std::string, std::string> newEnv)
|
||||
{
|
||||
clearEnv();
|
||||
for (auto newEnvVar : newEnv)
|
||||
{
|
||||
setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Path absPath(Path path, Path dir)
|
||||
{
|
||||
|
|
@ -1019,10 +1028,22 @@ void runProgram2(const RunOptions & options)
|
|||
if (options.standardOut) out.create();
|
||||
if (source) in.create();
|
||||
|
||||
ProcessOptions processOptions;
|
||||
// vfork implies that the environment of the main process and the fork will
|
||||
// be shared (technically this is undefined, but in practice that's the
|
||||
// case), so we can't use it if we alter the environment
|
||||
if (options.environment)
|
||||
processOptions.allowVfork = false;
|
||||
|
||||
/* Fork. */
|
||||
Pid pid = startProcess([&]() {
|
||||
if (options.environment)
|
||||
replaceEnv(*options.environment);
|
||||
if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
|
||||
throw SysError("dupping stdout");
|
||||
if (options.mergeStderrToStdout)
|
||||
if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1)
|
||||
throw SysError("cannot dup stdout into stderr");
|
||||
if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1)
|
||||
throw SysError("dupping stdin");
|
||||
|
||||
|
|
@ -1047,7 +1068,7 @@ void runProgram2(const RunOptions & options)
|
|||
execv(options.program.c_str(), stringsToCharPtrs(args_).data());
|
||||
|
||||
throw SysError("executing '%1%'", options.program);
|
||||
});
|
||||
}, processOptions);
|
||||
|
||||
out.writeSide = -1;
|
||||
|
||||
|
|
|
|||
|
|
@ -270,12 +270,14 @@ struct RunOptions
|
|||
std::optional<uid_t> uid;
|
||||
std::optional<uid_t> gid;
|
||||
std::optional<Path> chdir;
|
||||
std::optional<std::map<std::string, std::string>> environment;
|
||||
Path program;
|
||||
bool searchPath = true;
|
||||
Strings args;
|
||||
std::optional<std::string> input;
|
||||
Source * standardIn = nullptr;
|
||||
Sink * standardOut = nullptr;
|
||||
bool mergeStderrToStdout = false;
|
||||
bool _killStderr = false;
|
||||
|
||||
RunOptions(const Path & program, const Strings & args)
|
||||
|
|
|
|||
|
|
@ -170,6 +170,14 @@ public:
|
|||
name, sub);
|
||||
}
|
||||
|
||||
if (type == actPostBuildHook) {
|
||||
auto name = storePathToName(getS(fields, 0));
|
||||
if (hasSuffix(name, ".drv"))
|
||||
name.resize(name.size() - 4);
|
||||
i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name);
|
||||
i->name = DrvName(name).name;
|
||||
}
|
||||
|
||||
if (type == actQueryPathInfo) {
|
||||
auto name = storePathToName(getS(fields, 0));
|
||||
i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1));
|
||||
|
|
@ -228,14 +236,18 @@ public:
|
|||
update(*state);
|
||||
}
|
||||
|
||||
else if (type == resBuildLogLine) {
|
||||
else if (type == resBuildLogLine || type == resPostBuildLogLine) {
|
||||
auto lastLine = trim(getS(fields, 0));
|
||||
if (!lastLine.empty()) {
|
||||
auto i = state->its.find(act);
|
||||
assert(i != state->its.end());
|
||||
ActInfo info = *i->second;
|
||||
if (printBuildLogs) {
|
||||
log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + "> " + ANSI_NORMAL + lastLine);
|
||||
auto suffix = "> ";
|
||||
if (type == resPostBuildLogLine) {
|
||||
suffix = " (post)> ";
|
||||
}
|
||||
log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + suffix + ANSI_NORMAL + lastLine);
|
||||
} else {
|
||||
state->activities.erase(i->second);
|
||||
info.lastLine = lastLine;
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ let {
|
|||
builder = ./dependencies.builder0.sh + "/FOOBAR/../.";
|
||||
input1 = input1 + "/.";
|
||||
input2 = "${input2}/.";
|
||||
input1_drv = input1;
|
||||
meta.description = "Random test package";
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,8 @@ nix_tests = \
|
|||
check.sh \
|
||||
plugins.sh \
|
||||
search.sh \
|
||||
nix-copy-ssh.sh
|
||||
nix-copy-ssh.sh \
|
||||
post-hook.sh
|
||||
# parallel.sh
|
||||
|
||||
install-tests += $(foreach x, $(nix_tests), tests/$(x))
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
source common.sh
|
||||
|
||||
clearStore
|
||||
|
||||
export REMOTE_STORE=$TEST_ROOT/remote_store
|
||||
|
||||
# Build the dependencies and push them to the remote store
|
||||
nix-build dependencies.nix --post-build-hook $PWD/push_to_store.sh
|
||||
|
||||
clearStore
|
||||
|
||||
# Ensure that we the remote store contains both the runtime and buildtime
|
||||
# closure of what we've just built
|
||||
nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix
|
||||
nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix input1_drv
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
#!/bin/sh
|
||||
|
||||
echo Pushing "$@" to "$REMOTE_STORE"
|
||||
echo -n "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs
|
||||
Loading…
Reference in New Issue