Add support for uncompressed NARs in binary caches

Issue NixOS/hydra#102.
This commit is contained in:
Eelco Dolstra 2013-07-01 21:02:36 +02:00
parent 7986711632
commit 5116214343
6 changed files with 31 additions and 17 deletions

View file

@ -6,25 +6,28 @@ let
''
export PATH=${nixBinDir}:${coreutils}
if [ $compressionType = "xz" ]; then
ext=xz
compressor="${xz} -9"
if [ $compressionType = xz ]; then
ext=.xz
compressor="| ${xz} -9"
elif [ $compressionType = bzip2 ]; then
ext=.bz2
compressor="| ${bzip2}"
else
ext=bz2
compressor="${bzip2}"
ext=
compressor=
fi
echo "packing $storePath..."
mkdir $out
dst=$out/tmp.nar.$ext
dst=$out/tmp.nar$ext
set -o pipefail
nix-store --dump "$storePath" | $compressor > $dst
eval "nix-store --dump \"$storePath\" $compressor > $dst"
hash=$(nix-hash --flat --type $hashAlgo --base32 $dst)
echo -n $hash > $out/nar-compressed-hash
mv $dst $out/$hash.nar.$ext
mv $dst $out/$hash.nar$ext
'';
in

View file

@ -20,6 +20,7 @@
<command>nix-push</command>
<arg choice='plain'><option>--dest</option> <replaceable>dest-dir</replaceable></arg>
<arg><option>--bzip2</option></arg>
<arg><option>--none</option></arg>
<arg><option>--force</option></arg>
<arg><option>--link</option></arg>
<arg><option>--manifest</option></arg>
@ -106,6 +107,12 @@ automatically.</para>
</varlistentry>
<varlistentry><term><option>--none</option></term>
<listitem><para>Do not compress NARs.</para></listitem>
</varlistentry>
<varlistentry><term><option>--force</option></term>
<listitem><para>Overwrite <filename>.narinfo</filename> files if

View file

@ -513,8 +513,9 @@ sub downloadBinary {
next unless defined $info;
my $decompressor;
if ($info->{compression} eq "bzip2") { $decompressor = "$Nix::Config::bzip2 -d"; }
elsif ($info->{compression} eq "xz") { $decompressor = "$Nix::Config::xz -d"; }
if ($info->{compression} eq "bzip2") { $decompressor = "| $Nix::Config::bzip2 -d"; }
elsif ($info->{compression} eq "xz") { $decompressor = "| $Nix::Config::xz -d"; }
elsif ($info->{compression} eq "none") { $decompressor = ""; }
else {
print STDERR "unknown compression method $info->{compression}\n";
next;
@ -522,7 +523,7 @@ sub downloadBinary {
my $url = "$cache->{url}/$info->{url}"; # FIXME: handle non-relative URLs
print STDERR "\n*** Downloading $url to $storePath...\n";
checkURL $url;
if (system("$Nix::Config::curl --fail --location --insecure '$url' | $decompressor | $Nix::Config::binDir/nix-store --restore $destPath") != 0) {
if (system("$Nix::Config::curl --fail --location --insecure '$url' $decompressor | $Nix::Config::binDir/nix-store --restore $destPath") != 0) {
warn "download of `$url' failed" . ($! ? ": $!" : "") . "\n";
next;
}

View file

@ -344,17 +344,18 @@ while (scalar @path > 0) {
checkURL $narFile->{url};
my $decompressor =
$narFile->{compressionType} eq "bzip2" ? "$Nix::Config::bzip2 -d" :
$narFile->{compressionType} eq "xz" ? "$Nix::Config::xz -d" :
$narFile->{compressionType} eq "bzip2" ? "| $Nix::Config::bzip2 -d" :
$narFile->{compressionType} eq "xz" ? "| $Nix::Config::xz -d" :
$narFile->{compressionType} eq "none" ? "" :
die "unknown compression type `$narFile->{compressionType}'";
if ($curStep < $maxStep) {
# The archive will be used a base to a patch.
system("$curl '$narFile->{url}' | $decompressor > $tmpNar") == 0
system("$curl '$narFile->{url}' $decompressor > $tmpNar") == 0
or die "cannot download and unpack `$narFile->{url}' to `$v'\n";
} else {
# Unpack the archive to the target path.
system("$curl '$narFile->{url}' | $decompressor | $Nix::Config::binDir/nix-store --restore '$destPath'") == 0
system("$curl '$narFile->{url}' $decompressor | $Nix::Config::binDir/nix-store --restore '$destPath'") == 0
or die "cannot download and unpack `$narFile->{url}' to `$v'\n";
}

View file

@ -34,6 +34,8 @@ for (my $n = 0; $n < scalar @ARGV; $n++) {
exec "man nix-push" or die;
} elsif ($arg eq "--bzip2") {
$compressionType = "bzip2";
} elsif ($arg eq "--none") {
$compressionType = "none";
} elsif ($arg eq "--force") {
$force = 1;
} elsif ($arg eq "--dest") {
@ -202,7 +204,7 @@ for (my $n = 0; $n < scalar @storePaths2; $n++) {
$compressedHash =~ /^[0-9a-z]+$/ or die "invalid hash";
close HASH;
my $narName = "$compressedHash.nar." . ($compressionType eq "xz" ? "xz" : "bz2");
my $narName = "$compressedHash.nar" . ($compressionType eq "xz" ? ".xz" : $compressionType eq "bzip2" ? ".bz2" : "");
my $narFile = "$narDir/$narName";
(-f $narFile) or die "NAR file for $storePath not found";

View file

@ -47,7 +47,7 @@ clearStore
nix-build --option binary-caches "file://$cacheDir" dependencies.nix --dry-run # get info
mkdir $cacheDir/tmp
mv $cacheDir/*.nar.xz $cacheDir/tmp/
mv $cacheDir/*.nar* $cacheDir/tmp/
NIX_DEBUG_SUBST=1 nix-build --option binary-caches "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result --fallback