nix-copy-closure -s: Do substitutions via ‘nix-store --serve’

This means we no longer need an SSH master connection, since we only
execute a single command on the remote host.
This commit is contained in:
Eelco Dolstra 2014-07-10 20:43:04 +02:00
parent 7c3a5090bf
commit 7bb632b024
4 changed files with 43 additions and 11 deletions

View file

@ -29,16 +29,12 @@ sub copyTo {
my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor, my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor,
$includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes) = @_; $includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes) = @_;
$useSubstitutes = 0 if $dryRun;
# Get the closure of this path. # Get the closure of this path.
my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs, my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs,
map { followLinksToStorePath $_ } @{$storePaths}))); map { followLinksToStorePath $_ } @{$storePaths})));
# Optionally use substitutes on the remote host.
if (!$dryRun && $useSubstitutes) {
system "ssh $sshHost @{$sshOpts} nix-store -r --ignore-unknown @closure";
# Ignore exit status because this is just an optimisation.
}
# Start nix-store --serve on the remote host. # Start nix-store --serve on the remote host.
my ($from, $to); my ($from, $to);
my $pid = open2($from, $to, "ssh $sshHost @{$sshOpts} nix-store --serve --write"); my $pid = open2($from, $to, "ssh $sshHost @{$sshOpts} nix-store --serve --write");
@ -60,8 +56,9 @@ sub copyTo {
# Send the "query valid paths" command with the "lock" option # Send the "query valid paths" command with the "lock" option
# enabled. This prevents a race where the remote host # enabled. This prevents a race where the remote host
# garbage-collect paths that are already there. # garbage-collect paths that are already there. Optionally, ask
syswrite($to, pack("L<x4L<x4L<x4", 1, 1, scalar @closure)) or die; # the remote host to substitute missing paths.
syswrite($to, pack("L<x4L<x4L<x4L<x4", 1, 1, $useSubstitutes, scalar @closure)) or die;
writeString($_, $to) foreach @closure; writeString($_, $to) foreach @closure;
# Get back the set of paths that are already valid on the remote host. # Get back the set of paths that are already valid on the remote host.
@ -119,6 +116,12 @@ sub oldCopyTo {
my ($closure, $sshHost, $sshOpts, $storePaths, $compressor, $decompressor, my ($closure, $sshHost, $sshOpts, $storePaths, $compressor, $decompressor,
$includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes) = @_; $includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes) = @_;
# Optionally use substitutes on the remote host.
if (!$dryRun && $useSubstitutes) {
system "ssh $sshHost @{$sshOpts} nix-store -r --ignore-unknown @$closure";
# Ignore exit status because this is just an optimisation.
}
# Ask the remote host which paths are invalid. Because of limits # Ask the remote host which paths are invalid. Because of limits
# to the command line length, do this in chunks. Eventually, # to the command line length, do this in chunks. Eventually,
# we'll want to use --from-stdin, but we can't rely on the # we'll want to use --from-stdin, but we can't rely on the

View file

@ -81,9 +81,6 @@ while (@ARGV) {
die "$0: you did not specify a host name\n" unless defined $sshHost; die "$0: you did not specify a host name\n" unless defined $sshHost;
openSSHConnection $sshHost or die "$0: unable to start SSH\n";
if ($toMode) { # Copy TO the remote machine. if ($toMode) { # Copy TO the remote machine.
Nix::CopyClosure::copyTo( Nix::CopyClosure::copyTo(
$sshHost, [ @sshOpts ], [ @storePaths ], $compressor, $decompressor, $sshHost, [ @sshOpts ], [ @storePaths ], $compressor, $decompressor,
@ -92,6 +89,8 @@ if ($toMode) { # Copy TO the remote machine.
else { # Copy FROM the remote machine. else { # Copy FROM the remote machine.
openSSHConnection $sshHost or die "$0: unable to start SSH\n";
# Query the closure of the given store paths on the remote # Query the closure of the given store paths on the remote
# machine. Paths are assumed to be store paths; there is no # machine. Paths are assumed to be store paths; there is no
# resolution (following of symlinks). # resolution (following of symlinks).

View file

@ -59,6 +59,7 @@ static void query(std::pair<FdSink, FdSource> & pipes)
if (cmd == "have") { if (cmd == "have") {
writeInt(cmdQueryValidPaths, pipes.first); writeInt(cmdQueryValidPaths, pipes.first);
writeInt(0, pipes.first); // don't lock writeInt(0, pipes.first); // don't lock
writeInt(0, pipes.first); // don't substitute
writeStrings(tokenized, pipes.first); writeStrings(tokenized, pipes.first);
pipes.first.flush(); pipes.first.flush();
PathSet paths = readStrings<PathSet>(pipes.second); PathSet paths = readStrings<PathSet>(pipes.second);

View file

@ -896,16 +896,42 @@ static void opServe(Strings opFlags, Strings opArgs)
} }
switch (cmd) { switch (cmd) {
case cmdQueryValidPaths: { case cmdQueryValidPaths: {
bool lock = readInt(in); bool lock = readInt(in);
bool substitute = readInt(in);
PathSet paths = readStorePaths<PathSet>(in); PathSet paths = readStorePaths<PathSet>(in);
if (lock && writeAllowed) if (lock && writeAllowed)
for (auto & path : paths) for (auto & path : paths)
store->addTempRoot(path); store->addTempRoot(path);
/* If requested, substitute missing paths. This
implements nix-copy-closure's --use-substitutes
flag. */
if (substitute && writeAllowed) {
/* Filter out .drv files (we don't want to build anything). */
PathSet paths2;
for (auto & path : paths)
if (!isDerivation(path)) paths2.insert(path);
unsigned long long downloadSize, narSize;
PathSet willBuild, willSubstitute, unknown;
queryMissing(*store, PathSet(paths2.begin(), paths2.end()),
willBuild, willSubstitute, unknown, downloadSize, narSize);
/* FIXME: should use ensurePath(), but it only
does one path at a time. */
if (!willSubstitute.empty())
try {
store->buildPaths(willSubstitute);
} catch (Error & e) {
printMsg(lvlError, format("warning: %1%") % e.msg());
}
}
writeStrings(store->queryValidPaths(paths), out); writeStrings(store->queryValidPaths(paths), out);
out.flush(); out.flush();
break; break;
} }
case cmdQueryPathInfos: { case cmdQueryPathInfos: {
PathSet paths = readStorePaths<PathSet>(in); PathSet paths = readStorePaths<PathSet>(in);
// !!! Maybe we want a queryPathInfos? // !!! Maybe we want a queryPathInfos?
@ -924,10 +950,12 @@ static void opServe(Strings opFlags, Strings opArgs)
out.flush(); out.flush();
break; break;
} }
case cmdDumpStorePath: case cmdDumpStorePath:
dumpPath(readStorePath(in), out); dumpPath(readStorePath(in), out);
out.flush(); out.flush();
break; break;
case cmdImportPaths: { case cmdImportPaths: {
if (!writeAllowed) throw Error("importing paths not allowed"); if (!writeAllowed) throw Error("importing paths not allowed");
string compression = readString(in); string compression = readString(in);
@ -966,6 +994,7 @@ static void opServe(Strings opFlags, Strings opArgs)
break; break;
} }
default: default:
throw Error(format("unknown serve command %1%") % cmd); throw Error(format("unknown serve command %1%") % cmd);
} }