Allow both bzip2 and xz compression

This commit is contained in:
Eelco Dolstra 2012-07-01 18:46:38 -04:00
parent 000132cbd1
commit f4190c38ba
3 changed files with 67 additions and 29 deletions

View file

@ -6,29 +6,37 @@ let
'' ''
export PATH=${nixBinDir}:${coreutils} export PATH=${nixBinDir}:${coreutils}
if [ $compressionType = "xz" ]; then
ext=xz
compressor=${xz} -9
else
ext=bz2
compressor=${bzip2}
fi
echo "packing $storePath..." echo "packing $storePath..."
mkdir $out mkdir $out
dst=$out/tmp.nar.xz dst=$out/tmp.nar.$ext
set -o pipefail set -o pipefail
nix-store --dump "$storePath" | ${xz} -9 > $dst nix-store --dump "$storePath" | $compressor -9 > $dst
hash=$(nix-hash --flat --type $hashAlgo --base32 $dst) hash=$(nix-hash --flat --type $hashAlgo --base32 $dst)
echo -n $hash > $out/nar-compressed-hash echo -n $hash > $out/nar-compressed-hash
mv $dst $out/$hash.nar.xz mv $dst $out/$hash.nar.$ext
''; '';
in in
{ storePath, hashAlgo }: { storePath, hashAlgo, compressionType }:
derivation { derivation {
name = "nar"; name = "nar";
system = builtins.currentSystem; system = builtins.currentSystem;
builder = shell; builder = shell;
args = [ "-e" builder ]; args = [ "-e" builder ];
inherit storePath hashAlgo; inherit storePath hashAlgo compressionType;
# Don't build in a chroot because Nix's dependencies may not be there. # Don't build in a chroot because Nix's dependencies may not be there.
__noChroot = true; __noChroot = true;

View file

@ -4,7 +4,7 @@ use strict;
use Nix::Config; use Nix::Config;
use Nix::Store; use Nix::Store;
my @binaryCacheUrls = ("file:///tmp/binary-cache"); my @binaryCacheUrls = ("file:///tmp/binary-cache2");
sub getInfoFrom { sub getInfoFrom {
my ($storePath, $pathHash, $binaryCacheUrl) = @_; my ($storePath, $pathHash, $binaryCacheUrl) = @_;
@ -15,14 +15,15 @@ sub getInfoFrom {
print STDERR "GOT CURL REPLY ", $? >> 8, "\n"; print STDERR "GOT CURL REPLY ", $? >> 8, "\n";
return undef; return undef;
} }
my ($storePath2, $url, $fileHash, $fileSize, $narHash, $narSize, $deriver); my ($storePath2, $url, $compression, $fileHash, $fileSize, $narHash, $narSize, $deriver);
my @refs; my @refs;
foreach my $line (split "\n", $s) { foreach my $line (split "\n", $s) {
$line =~ /^(.*): (.*)$/ or return undef; $line =~ /^(.*): (.*)$/ or return undef;
if ($1 eq "StorePath") { $storePath2 = $2; } if ($1 eq "StorePath") { $storePath2 = $2; }
elsif ($1 eq "URL") { $url = $2; } elsif ($1 eq "URL") { $url = $2; }
elsif ($1 eq "CompressedHash") { $fileHash = $2; } elsif ($1 eq "Compression") { $compression = $2; }
elsif ($1 eq "CompressedSize") { $fileSize = int($2); } elsif ($1 eq "FileHash") { $fileHash = $2; }
elsif ($1 eq "FileSize") { $fileSize = int($2); }
elsif ($1 eq "NarHash") { $narHash = $2; } elsif ($1 eq "NarHash") { $narHash = $2; }
elsif ($1 eq "NarSize") { $narSize = int($2); } elsif ($1 eq "NarSize") { $narSize = int($2); }
elsif ($1 eq "References") { @refs = split / /, $2; } elsif ($1 eq "References") { @refs = split / /, $2; }
@ -34,6 +35,7 @@ sub getInfoFrom {
} }
return return
{ url => $url { url => $url
, compression => ($compression || "bzip2")
, fileHash => $fileHash , fileHash => $fileHash
, fileSize => $fileSize , fileSize => $fileSize
, narHash => $narHash , narHash => $narHash
@ -64,7 +66,14 @@ sub downloadBinary {
cache: foreach my $binaryCacheUrl (@binaryCacheUrls) { cache: foreach my $binaryCacheUrl (@binaryCacheUrls) {
my $info = getInfoFrom($storePath, $pathHash, $binaryCacheUrl); my $info = getInfoFrom($storePath, $pathHash, $binaryCacheUrl);
if (defined $info) { if (defined $info) {
if (system("$Nix::Config::curl --fail --location $binaryCacheUrl/$info->{url} | $Nix::Config::xz -d | $Nix::Config::binDir/nix-store --restore $storePath") == 0) { my $decompressor;
if ($info->{compression} eq "bzip2") { $decompressor = "$Nix::Config::bzip2 -d"; }
elsif ($info->{compression} eq "xz") { $decompressor = "$Nix::Config::xz -d"; }
else {
print STDERR "unknown compression method $info->{compression}\n";
next;
}
if (system("$Nix::Config::curl --fail --location $binaryCacheUrl/$info->{url} | $decompressor | $Nix::Config::binDir/nix-store --restore $storePath") == 0) {
return 1; return 1;
} }
} }

View file

@ -22,16 +22,21 @@ $curl = "$curl $extraCurlFlags" if defined $extraCurlFlags;
# Parse the command line. # Parse the command line.
my $compressionType = "xz";
my $force = 0;
my $localCopy; my $localCopy;
my $localArchivesDir; my $localArchivesDir;
my $archivesPutURL; my $archivesPutURL;
my $archivesGetURL; my $archivesGetURL;
my @roots;
sub showSyntax { sub showSyntax {
print STDERR <<EOF print STDERR <<EOF
Usage: nix-push --copy ARCHIVES_DIR PATHS... Usage: nix-push --copy ARCHIVES_DIR PATHS...
or: nix-push ARCHIVES_PUT_URL ARCHIVES_GET_URL PATHS... or: nix-push --upload ARCHIVES_PUT_URL ARCHIVES_GET_URL PATHS...
`nix-push' copies or uploads the closure of PATHS to the given `nix-push' copies or uploads the closure of PATHS to the given
destination. destination.
@ -40,27 +45,42 @@ EOF
exit 1; exit 1;
} }
showSyntax if scalar @ARGV < 1; for (my $n = 0; $n < scalar @ARGV; $n++) {
my $arg = $ARGV[$n];
if ($ARGV[0] eq "--copy") { if ($arg eq "--help") {
showSyntax if scalar @ARGV < 2; showSyntax;
$localCopy = 1; } elsif ($arg eq "--bzip2") {
shift @ARGV; $compressionType = "bzip2";
$localArchivesDir = shift @ARGV; } elsif ($arg eq "--force") {
mkpath($localArchivesDir, 0, 0755); $force = 1;
} else { } elsif ($arg eq "--copy") {
showSyntax if scalar @ARGV < 2; $n++;
$localCopy = 0; die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
$archivesPutURL = shift @ARGV; $localCopy = 1;
$archivesGetURL = shift @ARGV; $localArchivesDir = $ARGV[$n];
mkpath($localArchivesDir, 0, 0755);
} elsif ($arg eq "--upload") {
die "$0: `$arg' requires two arguments\n" unless $n + 2 < scalar @ARGV;
$localCopy = 0;
$archivesPutURL = $ARGV[$n + 1];
$archivesGetURL = $ARGV[$n + 2];
$n++;
} elsif (substr($arg, 0, 1) eq "-") {
showSyntax;
} else {
push @roots, $arg;
}
} }
showSyntax if !defined $localCopy;
# From the given store paths, determine the set of requisite store # From the given store paths, determine the set of requisite store
# paths, i.e, the paths required to realise them. # paths, i.e, the paths required to realise them.
my %storePaths; my %storePaths;
foreach my $path (@ARGV) { foreach my $path (@roots) {
die unless $path =~ /^\//; die unless $path =~ /^\//;
# Get all paths referenced by the normalisation of the given # Get all paths referenced by the normalisation of the given
@ -92,7 +112,7 @@ foreach my $storePath (@storePaths) {
# Construct a Nix expression that creates a Nix archive. # Construct a Nix expression that creates a Nix archive.
my $nixexpr = my $nixexpr =
"(import <nix/nar.nix> " . "(import <nix/nar.nix> " .
"{ storePath = builtins.storePath \"$storePath\"; hashAlgo = \"$hashAlgo\"; }) "; "{ storePath = builtins.storePath \"$storePath\"; hashAlgo = \"$hashAlgo\"; compressionType = \"$compressionType\"; }) ";
print NIX $nixexpr; print NIX $nixexpr;
} }
@ -152,7 +172,7 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
$compressedHash =~ /^[0-9a-z]+$/ or die "invalid hash"; $compressedHash =~ /^[0-9a-z]+$/ or die "invalid hash";
close HASH; close HASH;
my $narName = "$compressedHash.nar.xz"; my $narName = "$compressedHash.nar." . ($compressionType eq "xz" ? "xz" : "bz2");
my $narFile = "$narDir/$narName"; my $narFile = "$narDir/$narName";
(-f $narFile) or die "NAR file for $storePath not found"; (-f $narFile) or die "NAR file for $storePath not found";
@ -184,8 +204,9 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
my $info; my $info;
$info .= "StorePath: $storePath\n"; $info .= "StorePath: $storePath\n";
$info .= "URL: $narName\n"; $info .= "URL: $narName\n";
$info .= "CompressedHash: sha256:$compressedHash\n"; $info .= "Compression: $compressionType\n";
$info .= "CompressedSize: $compressedSize\n"; $info .= "FileHash: sha256:$compressedHash\n";
$info .= "FileSize: $compressedSize\n";
$info .= "NarHash: $narHash\n"; $info .= "NarHash: $narHash\n";
$info .= "NarSize: $narSize\n"; $info .= "NarSize: $narSize\n";
$info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n"; $info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n";
@ -201,7 +222,7 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
if ($localCopy) { if ($localCopy) {
my $dst = "$localArchivesDir/$infoName.narinfo"; my $dst = "$localArchivesDir/$infoName.narinfo";
if (! -f $dst) { if ($force || ! -f $dst) {
my $tmp = "$localArchivesDir/.tmp.$$.$infoName"; my $tmp = "$localArchivesDir/.tmp.$$.$infoName";
open INFO, ">$tmp" or die; open INFO, ">$tmp" or die;
print INFO "$info" or die; print INFO "$info" or die;