* Prebuilt package sharing. We allow transparent binary deployment by

sharing package directories (i.e., the result of building a Nix
  descriptor).

  `nix-pull-prebuilts' obtains a list of all known prebuilts by
  consulting the paths and URLs specified in
  $prefix/etc/nix/prebuilts.conf.  The mappings ($pkghash,
  $prebuilthash) and ($prebuilthash, $location) are registered with
  Nix so that it can use the prebuilt with hash $prebuilthash when
  installing a package with hash $pkghash by downloading and unpacking
  $location.

  `nix-push-prebuilts' creates prebuilts for all packages for which no
  prebuilt is known to exist.  It can then optionally upload these
  to the network through rsync.

  `nix-[pull|push]-prebuilts' just provide a policy.  Nix provides the
  mechanism through the `nix [export|regprebuilt|regurl]' commands.
This commit is contained in:
Eelco Dolstra 2003-05-25 22:42:19 +00:00
parent 0ef4b6d0f8
commit 7dd91d3779
9 changed files with 230 additions and 79 deletions

View file

@ -1,5 +1,9 @@
bin_SCRIPTS = nix-generate-regscript nix-switch nix-collect-garbage bin_SCRIPTS = nix-generate-regscript nix-switch nix-collect-garbage \
nix-pull-prebuilts nix-push-prebuilts
install-exec-local: install-exec-local:
$(INSTALL) -d $(sysconfdir)/profile.d $(INSTALL) -d $(sysconfdir)/profile.d
$(INSTALL_PROGRAM) nix-profile.sh $(sysconfdir)/profile.d/nix.sh $(INSTALL_PROGRAM) nix-profile.sh $(sysconfdir)/profile.d/nix.sh
$(INSTALL) -d $(sysconfdir)/nix
# !!! don't overwrite local modifications
$(INSTALL_PROGRAM) prebuilts.conf $(sysconfdir)/nix/prebuilts.conf

View file

@ -1,20 +0,0 @@
#! /usr/bin/perl -w
my $dir = shift @ARGV;
$dir || die "missing directory";
my $url = shift @ARGV;
$url || die "missing base url";
chdir $dir || die "cannot chdir to $dir";
foreach my $prebuilt (glob("*.tar.bz2")) {
$prebuilt =~ /-([a-z0-9]+)-([a-z0-9]+).tar.bz2$/
|| die "invalid file name: $prebuilt";
my $pkgHash = $1;
my $prebuiltHash = $2;
print "regprebuilt $pkgHash $prebuiltHash\n";
print "regurl $prebuiltHash $url/$prebuilt\n";
}

69
scripts/nix-pull-prebuilts Executable file
View file

@ -0,0 +1,69 @@
#! /usr/bin/perl -w
my $prefix = $ENV{"NIX"} || "/nix"; # !!! use prefix
my $etcdir = "$prefix/etc/nix";
my $knowns = "$prefix/var/nix/known-prebuilts";
my $tmpfile = "$prefix/var/nix/prebuilts.tmp";
my $conffile = "$etcdir/prebuilts.conf";
sub register {
my $fn = shift;
return unless $fn =~ /([^\/]*)-([0-9a-z]{32})-([0-9a-z]{32})\.tar\.bz2/;
my $id = $1;
my $pkghash = $2;
my $prebuilthash = $3;
print "$pkghash => $prebuilthash ($id)\n";
system "nix regprebuilt $pkghash $prebuilthash";
if ($?) { die "`nix regprebuilt' failed"; }
print KNOWNS "$pkghash\n";
}
open KNOWNS, ">$knowns";
open CONFFILE, "<$conffile";
while (<CONFFILE>) {
chomp;
if (/^\s*(\S+)\s*(\#.*)?$/) {
my $url = $1;
print "obtaining prebuilt list from $url...\n";
if ($url =~ /^\//) {
# It's a local path.
foreach my $fn (glob "$url/*") {
register $fn;
}
} else {
# It's a URL.
system "wget '$url' -O '$tmpfile' 2> /dev/null"; # !!! escape
if ($?) { die "`wget' failed"; }
open INDEX, "<$tmpfile";
while (<INDEX>) {
# Get all links to prebuilts, that is, file names of the
# form foo-HASH-HASH.tar.bz2.
next unless (/HREF=\"([^\"]*)\"/);
my $fn = $1;
next if $fn =~ /\.\./;
next if $fn =~ /\//;
register $fn;
}
close INDEX;
unlink $tmpfile;
}
}
}
close CONFFILE;
close KNOWNS;

40
scripts/nix-push-prebuilts Executable file
View file

@ -0,0 +1,40 @@
#! /usr/bin/perl -w
my $prefix = $ENV{"NIX"} || "/nix"; # !!! use prefix
my $etcdir = "$prefix/etc/nix";
my $exportdir = "$prefix/var/nix/prebuilts/exports";
my $knowns = "$prefix/var/nix/known-prebuilts";
# For performance, put the known hashes in an associative array.
my %knowns = ();
open KNOWNS, "<$knowns";
while (<KNOWNS>) {
next unless /([0-9a-z]{32})/;
$knowns{$1} = 1;
}
close KNOWNS;
# For each installed package, check whether a prebuilt is known.
open PKGS, "nix listinst|";
open KNOWNS, ">>$knowns";
while (<PKGS>) {
chomp;
next unless /([0-9a-z]{32})/;
my $pkghash = $1;
if (!defined $knowns{$1}) {
# No known prebuilt exists for this package; so export it.
print "exporting $pkghash...\n";
system "nix export '$exportdir' $pkghash";
if ($?) { die "`nix export' failed"; }
print KNOWNS "$pkghash\n";
}
}
close KNOWNS;
close PKGS;
# Push the prebuilts to the server. !!! FIXME
system "rsync -av -e ssh '$exportdir' losser:/home/eelco/public_html/nix-prebuilts/";

4
scripts/prebuilts.conf Normal file
View file

@ -0,0 +1,4 @@
# A list of URLs or local paths from where we obtain prebuilts.
/nix/var/nix/prebuilts/imports
/nix/var/nix/prebuilts/exports
http://losser.st-lab.cs.uu.nl/~eelco/nix-prebuilts/

View file

@ -13,5 +13,8 @@ install-data-local:
$(INSTALL) -d $(localstatedir)/nix/descriptors $(INSTALL) -d $(localstatedir)/nix/descriptors
$(INSTALL) -d $(localstatedir)/nix/sources $(INSTALL) -d $(localstatedir)/nix/sources
$(INSTALL) -d $(localstatedir)/nix/links $(INSTALL) -d $(localstatedir)/nix/links
$(INSTALL) -d $(localstatedir)/nix/prebuilts
$(INSTALL) -d $(localstatedir)/nix/prebuilts/imports
$(INSTALL) -d $(localstatedir)/nix/prebuilts/exports
$(INSTALL) -d $(prefix)/pkg $(INSTALL) -d $(prefix)/pkg
$(bindir)/nix init $(bindir)/nix init

View file

@ -13,7 +13,6 @@ extern "C" {
static string nixDescriptorDir; static string nixDescriptorDir;
static string nixSourcesDir;
static bool verbose = false; static bool verbose = false;
@ -33,46 +32,6 @@ void registerFile(string filename)
throw Error("cannot register " + filename + " with Nix"); throw Error("cannot register " + filename + " with Nix");
} }
/* Return the directory part of the given path, i.e., everything
before the final `/'. */
string dirOf(string s)
{
unsigned int pos = s.rfind('/');
if (pos == string::npos) throw Error("invalid file name");
return string(s, 0, pos);
}
/* Return the base name of the given path, i.e., everything following
the final `/'. */
string baseNameOf(string s)
{
unsigned int pos = s.rfind('/');
if (pos == string::npos) throw Error("invalid file name");
return string(s, pos + 1);
}
/* Download object referenced by the given URL into the sources
directory. Return the file name it was downloaded to. */
string fetchURL(string url)
{
string filename = baseNameOf(url);
string fullname = nixSourcesDir + "/" + filename;
struct stat st;
if (stat(fullname.c_str(), &st)) {
/* !!! quoting */
string shellCmd =
"cd " + nixSourcesDir + " && wget --quiet -N \"" + url + "\"";
int res = system(shellCmd.c_str());
if (WEXITSTATUS(res) != 0)
throw Error("cannot fetch " + url);
}
return fullname;
}
Error badTerm(const string & msg, ATerm e) Error badTerm(const string & msg, ATerm e)
{ {
char * s = ATwriteToString(e); char * s = ATwriteToString(e);
@ -120,7 +79,7 @@ bool evaluateBool(ATerm e, EvalContext ctx)
ATerm evaluate(ATerm e, EvalContext ctx) ATerm evaluate(ATerm e, EvalContext ctx)
{ {
char * s; char * s;
ATerm e2; ATerm e2, e3;
ATerm eCond, eTrue, eFalse; ATerm eCond, eTrue, eFalse;
/* Check for normal forms first. */ /* Check for normal forms first. */
@ -166,6 +125,7 @@ ATerm evaluate(ATerm e, EvalContext ctx)
instantiateDescriptor(filename, ctx).c_str()); instantiateDescriptor(filename, ctx).c_str());
} }
#if 0
/* `Source' copies the specified file to nixSourcesDir, registers /* `Source' copies the specified file to nixSourcesDir, registers
it with Nix, and returns the hash of the file. */ it with Nix, and returns the hash of the file. */
else if (ATmatch(e, "Source(<term>)", &e2)) { else if (ATmatch(e, "Source(<term>)", &e2)) {
@ -185,15 +145,29 @@ ATerm evaluate(ATerm e, EvalContext ctx)
registerFile(target); registerFile(target);
return ATmake("File(<str>)", hashFile(target).c_str()); return ATmake("File(<str>)", hashFile(target).c_str());
} }
#endif
/* `Url' fetches a file from the network, caching it in /* `Local' registers a file with Nix, and returns the file's
nixSourcesDir and returning the file name. */ hash. */
else if (ATmatch(e, "Url(<term>)", &e2)) { else if (ATmatch(e, "Local(<term>)", &e2)) {
string url = evaluateStr(e2, ctx); string filename = absPath(evaluateStr(e2, ctx), ctx.dir); /* !!! */
string hash = hashFile(filename);
return ATmake("File(<str>)", hash.c_str());
}
/* `Url' registers a mapping from a hash to an url with Nix, and
returns the hash. */
else if (ATmatch(e, "Url(<term>, <term>)", &e2, &e3)) {
string hash = evaluateStr(e2, ctx);
checkHash(hash);
string url = evaluateStr(e3, ctx);
#if 0
if (verbose) if (verbose)
cerr << "fetching " << url << endl; cerr << "fetching " << url << endl;
string filename = fetchURL(url); string filename = fetchURL(url);
return ATmake("Str(<str>)", filename.c_str()); #endif
/* !!! register */
return ATmake("File(<str>)", hash.c_str());
} }
/* `If' provides conditional evaluation. */ /* `If' provides conditional evaluation. */
@ -329,7 +303,6 @@ void run(Strings::iterator argCur, Strings::iterator argEnd)
if (homeDir) nixHomeDir = homeDir; if (homeDir) nixHomeDir = homeDir;
nixDescriptorDir = nixHomeDir + "/var/nix/descriptors"; nixDescriptorDir = nixHomeDir + "/var/nix/descriptors";
nixSourcesDir = nixHomeDir + "/var/nix/sources";
for ( ; argCur != argEnd; argCur++) { for ( ; argCur != argEnd; argCur++) {
string arg(*argCur); string arg(*argCur);

View file

@ -28,6 +28,9 @@ static string dbInstPkgs = "pkginst";
static string dbPrebuilts = "prebuilts"; static string dbPrebuilts = "prebuilts";
static string nixSourcesDir;
/* Wrapper classes that ensures that the database is closed upon /* Wrapper classes that ensures that the database is closed upon
object destruction. */ object destruction. */
class Db2 : public Db class Db2 : public Db
@ -435,9 +438,9 @@ void exportPkgs(string outDir,
string pkgDir = getPkg(hash); string pkgDir = getPkg(hash);
string tmpFile = outDir + "/export_tmp"; string tmpFile = outDir + "/export_tmp";
string cmd = "cd " + pkgDir + " && tar cvfj " + tmpFile + " ."; string cmd = "cd " + pkgDir + " && tar cfj " + tmpFile + " .";
int res = system(cmd.c_str()); // !!! escaping int res = system(cmd.c_str()); // !!! escaping
if (WEXITSTATUS(res) != 0) if (!WIFEXITED(res) || WEXITSTATUS(res) != 0)
throw Error("cannot tar " + pkgDir); throw Error("cannot tar " + pkgDir);
string prebuiltHash = hashFile(tmpFile); string prebuiltHash = hashFile(tmpFile);
@ -458,10 +461,12 @@ void regPrebuilt(string pkgHash, string prebuiltHash)
} }
void registerFile(string filename) string registerFile(string filename)
{ {
filename = absPath(filename); filename = absPath(filename);
setDB(dbRefs, hashFile(filename), filename); string hash = hashFile(filename);
setDB(dbRefs, hash, filename);
return hash;
} }
@ -618,8 +623,46 @@ void printGraph(Strings::iterator first, Strings::iterator last)
} }
void run(Strings args) /* Download object referenced by the given URL into the sources
directory. Return the file name it was downloaded to. */
string fetchURL(string url)
{ {
string filename = baseNameOf(url);
string fullname = nixSourcesDir + "/" + filename;
struct stat st;
if (stat(fullname.c_str(), &st)) {
/* !!! quoting */
string shellCmd =
"cd " + nixSourcesDir + " && wget --quiet -N \"" + url + "\"";
int res = system(shellCmd.c_str());
if (WEXITSTATUS(res) != 0)
throw Error("cannot fetch " + url);
}
return fullname;
}
void fetch(string id)
{
string fn;
/* Fetch the object referenced by id. */
if (isHash(id)) {
throw Error("not implemented");
} else {
fn = fetchURL(id);
}
/* Register it by hash. */
string hash = registerFile(fn);
cout << hash << endl;
}
void fetch(Strings::iterator first, Strings::iterator last)
{
for (Strings::iterator it = first; it != last; it++)
fetch(*it);
} }
@ -675,6 +718,11 @@ Subcommands:
graph HASH... graph HASH...
Like closure, but print a dot graph specification. Like closure, but print a dot graph specification.
fetch ID...
Fetch the objects identified by ID and place them in the Nix
sources directory. ID can be a hash or URL. Print out the hash
of the object.
"; ";
} }
@ -686,6 +734,8 @@ void run(Strings::iterator argCur, Strings::iterator argEnd)
char * homeDir = getenv(nixHomeDirEnvVar.c_str()); char * homeDir = getenv(nixHomeDirEnvVar.c_str());
if (homeDir) nixHomeDir = homeDir; if (homeDir) nixHomeDir = homeDir;
nixSourcesDir = nixHomeDir + "/var/nix/sources";
/* Parse the global flags. */ /* Parse the global flags. */
for ( ; argCur != argEnd; argCur++) { for ( ; argCur != argEnd; argCur++) {
string arg(*argCur); string arg(*argCur);
@ -742,6 +792,8 @@ void run(Strings::iterator argCur, Strings::iterator argEnd)
printClosure(argCur, argEnd); printClosure(argCur, argEnd);
} else if (cmd == "graph") { } else if (cmd == "graph") {
printGraph(argCur, argEnd); printGraph(argCur, argEnd);
} else if (cmd == "fetch") {
fetch(argCur, argEnd);
} else } else
throw UsageError("unknown command: " + string(cmd)); throw UsageError("unknown command: " + string(cmd));
} }

View file

@ -85,17 +85,22 @@ string printHash(unsigned char * buf)
/* Verify that a reference is valid (that is, is a MD5 hash code). */ /* Verify that a reference is valid (that is, is a MD5 hash code). */
void checkHash(const string & s) bool isHash(const string & s)
{ {
string err = "invalid reference: " + s; if (s.length() != 32) return false;
if (s.length() != 32)
throw BadRefError(err);
for (int i = 0; i < 32; i++) { for (int i = 0; i < 32; i++) {
char c = s[i]; char c = s[i];
if (!((c >= '0' && c <= '9') || if (!((c >= '0' && c <= '9') ||
(c >= 'a' && c <= 'f'))) (c >= 'a' && c <= 'f')))
throw BadRefError(err); return false;
} }
return true;
}
void checkHash(const string & s)
{
if (!isHash(s)) throw BadRefError("invalid reference: " + s);
} }
@ -113,4 +118,25 @@ string hashFile(string filename)
} }
/* Return the directory part of the given path, i.e., everything
before the final `/'. */
string dirOf(string s)
{
unsigned int pos = s.rfind('/');
if (pos == string::npos) throw Error("invalid file name");
return string(s, 0, pos);
}
/* Return the base name of the given path, i.e., everything following
the final `/'. */
string baseNameOf(string s)
{
unsigned int pos = s.rfind('/');
if (pos == string::npos) throw Error("invalid file name");
return string(s, pos + 1);
}
#endif /* !__UTIL_H */ #endif /* !__UTIL_H */