* Add an API function exportPaths() that provides the functionality of

‘nix-store --export’.
* Add a Perl module that provides the functionality of
  ‘nix-copy-closure --to’.  This is used by build-remote.pl so it no
  longer needs to start a separate nix-copy-closure process.  Also, it
  uses the Perl API to do the export, so it doesn't need to start a
  separate nix-store process either.  As a result, nix-copy-closure
  and build-remote.pl should no longer fail on very large closures due
  to an "Argument list too long" error.  (Note that having very many
  dependencies in a single derivation can still fail because the
  environment can become too large.  Can't be helped though.)
This commit is contained in:
Eelco Dolstra 2011-11-23 15:13:37 +00:00
parent ab20af3e6f
commit 5bbd693cae
10 changed files with 90 additions and 38 deletions

View file

@ -2,7 +2,7 @@ perlversion := $(shell perl -e 'use Config; print $$Config{version};')
perlarchname := $(shell perl -e 'use Config; print $$Config{archname};')
perllibdir = $(libdir)/perl5/site_perl/$(perlversion)/$(perlarchname)
PERL_MODULES = lib/Nix/Store.pm lib/Nix/Manifest.pm lib/Nix/GeneratePatches.pm lib/Nix/SSH.pm lib/Nix/Config.pm.in
PERL_MODULES = lib/Nix/Store.pm lib/Nix/Manifest.pm lib/Nix/GeneratePatches.pm lib/Nix/SSH.pm lib/Nix/CopyClosure.pm lib/Nix/Config.pm.in
all: $(PERL_MODULES:.in=)
ln -sfn $(abs_builddir)/.libs/libNixStore.so lib/Store.so

View file

@ -0,0 +1,46 @@
package Nix::CopyClosure;
use strict;
use Nix::Config;
use Nix::Store;
sub copyTo {
my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor, $includeOutputs, $dryRun, $sign) = @_;
$compressor = "$compressor |" if $compressor ne "";
$decompressor = "$decompressor |" if $decompressor ne "";
# Get the closure of this path.
my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs,
map { followLinksToStorePath $_ } @{$storePaths})));
# Ask the remote host which paths are invalid. Because of limits
# to the command line length, do this in chunks. Eventually,
# we'll want to use --from-stdin, but we can't rely on the
# target having this option yet.
my @missing = ();
while (scalar(@closure) > 0) {
my @ps = splice(@closure, 0, 1500);
open(READ, "set -f; ssh $sshHost @{$sshOpts} nix-store --check-validity --print-invalid @ps|");
while (<READ>) {
chomp;
push @missing, $_;
}
close READ or die;
}
# Export the store paths and import them on the remote machine.
if (scalar @missing > 0) {
print STDERR "copying ", scalar @missing, " missing paths to $sshHost...\n";
#print STDERR " $_\n" foreach @missing;
unless ($dryRun) {
open SSH, "| $compressor ssh $sshHost @{$sshOpts} '$decompressor nix-store --import'" or die;
exportPaths(fileno(SSH), $sign, @missing);
close SSH or die "copying store paths to remote machine `$sshHost' failed: $?";
}
}
}
1;

View file

@ -12,7 +12,7 @@ our %EXPORT_TAGS = ( 'all' => [ qw( ) ] );
our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } );
our @EXPORT = qw(isValidPath topoSortPaths computeFSClosure followLinksToStorePath);
our @EXPORT = qw(isValidPath topoSortPaths computeFSClosure followLinksToStorePath exportPaths);
our $VERSION = '0.15';

View file

@ -146,3 +146,16 @@ SV * followLinksToStorePath(char * path)
}
OUTPUT:
RETVAL
void exportPaths(int fd, int sign, ...)
PPCODE:
try {
doInit();
Paths paths;
for (int n = 2; n < items; ++n) paths.push_back(SvPV_nolen(ST(n)));
FdSink sink(fd);
exportPaths(*store, paths, sign, sink);
} catch (Error & e) {
croak(e.what());
}

View file

@ -5,6 +5,7 @@ use English '-no_match_vars';
use IO::Handle;
use Nix::Config;
use Nix::SSH qw/sshOpts openSSHConnection/;
use Nix::CopyClosure;
no warnings('once');
@ -225,8 +226,7 @@ sub removeRoots {
# Copy the derivation and its dependencies to the build machine.
system("NIX_SSHOPTS=\"@sshOpts\" @bindir@/nix-copy-closure $hostName $maybeSign $drvPath @inputs") == 0
or die "cannot copy inputs to $hostName: $?";
Nix::CopyClosure::copyTo($hostName, [ @sshOpts ], [ $drvPath, @inputs ], "", "", 0, 0, $maybeSign ne "");
# Perform the build.

View file

@ -3,6 +3,7 @@
use Nix::SSH;
use Nix::Config;
use Nix::Store;
use Nix::CopyClosure;
if (scalar @ARGV < 1) {
@ -39,8 +40,8 @@ while (@ARGV) {
$sign = 1;
}
elsif ($arg eq "--gzip") {
$compressor = "| gzip";
$decompressor = "gunzip |";
$compressor = "gzip";
$decompressor = "gunzip";
}
elsif ($arg eq "--from") {
$toMode = 0;
@ -67,30 +68,7 @@ openSSHConnection $sshHost or die "$0: unable to start SSH\n";
if ($toMode) { # Copy TO the remote machine.
# Get the closure of this path.
my @allStorePaths = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs, map { followLinksToStorePath $_ } @storePaths)));
# Ask the remote host which paths are invalid.
open(READ, "set -f; ssh $sshHost @sshOpts nix-store --check-validity --print-invalid @allStorePaths|");
my @missing = ();
while (<READ>) {
chomp;
push @missing, $_;
}
close READ or die;
# Export the store paths and import them on the remote machine.
if (scalar @missing > 0) {
print STDERR "copying these missing paths:\n";
print STDERR " $_\n" foreach @missing;
unless ($dryRun) {
my $extraOpts = $sign ? "--sign" : "";
system("set -f; nix-store --export $extraOpts @missing $compressor | ssh $sshHost @sshOpts '$decompressor nix-store --import'") == 0
or die "copying store paths to remote machine `$sshHost' failed: $?";
}
}
Nix::CopyClosure::copyTo($sshHost, [ @sshOpts ], [ @storePaths ], $compressor, $decompressor, $includeOutputs, $dryRun, $sign);
}
else { # Copy FROM the remote machine.
@ -112,8 +90,10 @@ else { # Copy FROM the remote machine.
# Export the store paths on the remote machine and import them on locally.
if (scalar @missing > 0) {
print STDERR "copying these missing paths:\n";
print STDERR " $_\n" foreach @missing;
print STDERR "copying ", scalar @missing, " missing paths from $sshHost...\n";
#print STDERR " $_\n" foreach @missing;
$compressor = "| $compressor" if $compressor ne "";
$decompressor = "$decompressor |" if $decompressor ne "";
unless ($dryRun) {
my $extraOpts = $sign ? "--sign" : "";
system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $decompressor $Nix::Config::binDir/nix-store --import") == 0

View file

@ -1255,7 +1255,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
bool haveSignature = readInt(hashAndReadSource) == 1;
if (requireSignature && !haveSignature)
throw Error("imported archive lacks a signature");
throw Error(format("imported archive of `%1%' lacks a signature") % dstPath);
if (haveSignature) {
string signature = readString(hashAndReadSource);

View file

@ -298,6 +298,17 @@ string showPaths(const PathSet & paths)
}
void exportPaths(StoreAPI & store, const Paths & paths,
bool sign, Sink & sink)
{
foreach (Paths::const_iterator, i, paths) {
writeInt(1, sink);
store.exportPath(*i, sign, sink);
}
writeInt(0, sink);
}
}

View file

@ -342,6 +342,12 @@ ValidPathInfo decodeValidPathInfo(std::istream & str,
bool hashGiven = false);
/* Export multiple paths in the format expected by nix-store
--import. */
void exportPaths(StoreAPI & store, const Paths & paths,
bool sign, Sink & sink);
}

View file

@ -594,11 +594,7 @@ static void opExport(Strings opFlags, Strings opArgs)
else throw UsageError(format("unknown flag `%1%'") % *i);
FdSink sink(STDOUT_FILENO);
for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); ++i) {
writeInt(1, sink);
store->exportPath(*i, sign, sink);
}
writeInt(0, sink);
exportPaths(*store, opArgs, sign, sink);
}