* Substitutes and nix-pull now work again.

* Fixed a segfault caused by the buffering of stderr.
* Fix now allows the specification of the full output path.  This
  should be used with great care, since it by-passes the normal hash
  generation.
* Incremented the version number to 0.4 (prerelease).
This commit is contained in:
Eelco Dolstra 2003-10-16 16:29:57 +00:00
parent ab5e8767fa
commit 0791282b2f
12 changed files with 169 additions and 89 deletions

View file

@ -1,4 +1,4 @@
AC_INIT(nix, "0.3") AC_INIT(nix, "0.4")
AC_CONFIG_SRCDIR(src/nix.cc) AC_CONFIG_SRCDIR(src/nix.cc)
AC_CONFIG_AUX_DIR(config) AC_CONFIG_AUX_DIR(config)
AM_INIT_AUTOMAKE AM_INIT_AUTOMAKE

View file

@ -1,9 +1,9 @@
Function(["nar", "name"], Function(["nar", "outPath"],
Package( Package(
[ ("name", Var("name")) [ ("name", "unnar")
, ("outPath", Var("outPath"))
, ("build", Relative("nar/unnar.sh")) , ("build", Relative("nar/unnar.sh"))
, ("nar", Var("nar")) , ("nar", Var("nar"))
, ("id", Var("id"))
] ]
) )
) )

View file

@ -2,11 +2,18 @@
use strict; use strict;
use IPC::Open2; use IPC::Open2;
use POSIX qw(tmpnam);
my $tmpfile = "@localstatedir@/nix/pull.tmp"; my $tmpdir;
do { $tmpdir = tmpnam(); }
until mkdir $tmpdir, 0777;
my $manifest = "$tmpdir/manifest";
my $conffile = "@sysconfdir@/nix/prebuilts.conf"; my $conffile = "@sysconfdir@/nix/prebuilts.conf";
my @ids; #END { unlink $manifest; rmdir $tmpdir; }
my @srcpaths;
my @subs; my @subs;
my @sucs; my @sucs;
@ -20,70 +27,89 @@ while (<CONFFILE>) {
chomp; chomp;
if (/^\s*(\S+)\s*(\#.*)?$/) { if (/^\s*(\S+)\s*(\#.*)?$/) {
my $url = $1; my $url = $1;
$url =~ s/\/$//;
print "obtaining list of Nix archives at $url...\n"; print "obtaining list of Nix archives at $url...\n";
system "wget '$url' -O '$tmpfile' 2> /dev/null"; # !!! escape system "wget '$url'/MANIFEST -O '$manifest' 2> /dev/null"; # !!! escape
if ($?) { die "`wget' failed"; } if ($?) { die "`wget' failed"; }
open INDEX, "<$tmpfile"; open MANIFEST, "<$manifest";
while (<INDEX>) { my $inside = 0;
# Get all links to prebuilts, that is, file names of the
# form foo-HASH-HASH.tar.bz2.
next unless (/HREF=\"([^\"]*)\"/);
my $fn = $1;
next if $fn =~ /\.\./;
next if $fn =~ /\//;
next unless $fn =~ /^([0-9a-z]{32})-([0-9a-z]{32})(.*)\.nar\.bz2$/;
my $hash = $1;
my $id = $2;
my $outname = $3;
my $fsid;
if ($outname =~ /^-/) {
next unless $outname =~ /^-((s-([0-9a-z]{32}))?.*)$/;
$outname = $1;
$fsid = $3;
} else {
$outname = "unnamed";
}
print STDERR "$id ($outname)\n"; my $storepath;
my $narname;
my $hash;
my @preds;
# Construct a Fix expression that fetches and unpacks a while (<MANIFEST>) {
# Nix archive from the network. chomp;
my $fetch = s/\#.*$//g;
"App(IncludeFix(\"fetchurl/fetchurl.fix\"), " . next if (/^$/);
"[(\"url\", \"$url/$fn\"), (\"md5\", \"$hash\")])";
my $fixexpr =
"App(IncludeFix(\"nar/unnar.fix\"), " .
"[ (\"nar\", $fetch)" .
", (\"name\", \"$outname\")" .
", (\"id\", \"$id\")" .
"])";
if (!$first) { $fullexpr .= "," };
$first = 0;
$fullexpr .= $fixexpr; # !!! O(n^2)?
push @ids, $id; if (!$inside) {
if (/^\{$/) {
$inside = 1;
undef $storepath;
undef $narname;
undef $hash;
@preds = ();
}
else { die "bad line: $_"; }
} else {
if (/^\}$/) {
$inside = 0;
my $fullurl = "$url/$narname";
print "$storepath\n";
# Does the name encode a successor relation? # Construct a Fix expression that fetches and unpacks a
if (defined $fsid) { # Nix archive from the network.
push @sucs, $fsid; my $fetch =
push @sucs, $id; "App(IncludeFix(\"fetchurl/fetchurl.fix\"), " .
} "[(\"url\", \"$fullurl\"), (\"md5\", \"$hash\")])";
my $fixexpr =
"App(IncludeFix(\"nar/unnar.fix\"), " .
"[ (\"nar\", $fetch)" .
", (\"outPath\", \"$storepath\")" .
"])";
if (!$first) { $fullexpr .= "," };
$first = 0;
$fullexpr .= $fixexpr; # !!! O(n^2)?
push @srcpaths, $storepath;
foreach my $p (@preds) {
push @sucs, $p;
push @sucs, $storepath;
}
}
elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) {
$storepath = $1;
}
elsif (/^\s*NarName:\s*(\S+)\s*$/) {
$narname = $1;
}
elsif (/^\s*MD5:\s*(\S+)\s*$/) {
$hash = $1;
}
elsif (/^\s*SuccOf:\s*(\/\S+)\s*$/) {
push @preds, $1;
}
else { die "bad line: $_"; }
}
} }
close INDEX; close MANIFEST;
unlink $tmpfile;
} }
} }
$fullexpr .= "]"; $fullexpr .= "]";
# Instantiate Nix expressions from the Fix expressions we created above. # Instantiate Nix expressions from the Fix expressions we created above.
print STDERR "running fix...\n"; print STDERR "running fix...\n";
my $pid = open2(\*READ, \*WRITE, "fix -") or die "cannot run fix"; my $pid = open2(\*READ, \*WRITE, "fix -") or die "cannot run fix";
@ -93,23 +119,28 @@ close WRITE;
my $i = 0; my $i = 0;
while (<READ>) { while (<READ>) {
chomp; chomp;
die unless /^([0-9a-z]{32})$/; die unless /^\//;
my $nid = $1; my $subpath = $_;
die unless ($i < scalar @ids); die unless ($i < scalar @srcpaths);
my $id = $ids[$i++]; my $srcpath = $srcpaths[$i++];
push @subs, $id; push @subs, $srcpath;
push @subs, $nid; push @subs, $subpath;
print "$srcpath $subpath\n";
} }
waitpid $pid, 0; waitpid $pid, 0;
$? == 0 or die "fix failed"; $? == 0 or die "fix failed";
# Register all substitutes. # Register all substitutes.
print STDERR "registering substitutes...\n"; print STDERR "registering substitutes...\n";
print "@subs\n";
system "nix --substitute @subs"; system "nix --substitute @subs";
if ($?) { die "`nix --substitute' failed"; } if ($?) { die "`nix --substitute' failed"; }
# Register all successors. # Register all successors.
print STDERR "registering successors...\n"; print STDERR "registering successors...\n";
system "nix --successor @sucs"; print "@sucs\n";
system "nix --successor -vvvv @sucs";
if ($?) { die "`nix --successor' failed"; } if ($?) { die "`nix --successor' failed"; }

View file

@ -1,4 +1,5 @@
#include "dotgraph.hh" #include "dotgraph.hh"
#include "normalise.hh"
static string dotQuote(const string & s) static string dotQuote(const string & s)
@ -98,8 +99,8 @@ void printDotGraph(const PathSet & roots)
if (doneSet.find(nePath) == doneSet.end()) { if (doneSet.find(nePath) == doneSet.end()) {
doneSet.insert(nePath); doneSet.insert(nePath);
NixExpr ne = parseNixExpr(termFromPath(nePath)); NixExpr ne = exprFromPath(nePath);
string label, colour; string label, colour;

View file

@ -22,14 +22,6 @@ Hash hashTerm(ATerm t)
} }
ATerm termFromPath(const Path & path)
{
ATerm t = ATreadFromNamedFile(path.c_str());
if (!t) throw Error(format("cannot read aterm from `%1%'") % path);
return t;
}
Path writeTerm(ATerm t, const string & suffix) Path writeTerm(ATerm t, const string & suffix)
{ {
/* The id of a term is its hash. */ /* The id of a term is its hash. */

View file

@ -53,9 +53,6 @@ Error badTerm(const format & f, ATerm t);
/* Hash an aterm. */ /* Hash an aterm. */
Hash hashTerm(ATerm t); Hash hashTerm(ATerm t);
/* Read an aterm from disk. */
ATerm termFromPath(const Path & path);
/* Write an aterm to the Nix store directory, and return its path. */ /* Write an aterm to the Nix store directory, and return its path. */
Path writeTerm(ATerm t, const string & suffix); Path writeTerm(ATerm t, const string & suffix);

View file

@ -299,6 +299,7 @@ static Expr evalExpr2(EvalState & state, Expr e)
ne.type = NixExpr::neDerivation; ne.type = NixExpr::neDerivation;
ne.derivation.platform = SYSTEM; ne.derivation.platform = SYSTEM;
string name; string name;
Path outPath;
Hash outHash; Hash outHash;
bool outHashGiven = false; bool outHashGiven = false;
bnds = ATempty; bnds = ATempty;
@ -327,6 +328,7 @@ static Expr evalExpr2(EvalState & state, Expr e)
if (key == "build") ne.derivation.builder = s; if (key == "build") ne.derivation.builder = s;
if (key == "name") name = s; if (key == "name") name = s;
if (key == "outPath") outPath = s;
if (key == "id") { if (key == "id") {
outHash = parseHash(s); outHash = parseHash(s);
outHashGiven = true; outHashGiven = true;
@ -343,11 +345,13 @@ static Expr evalExpr2(EvalState & state, Expr e)
if (name == "") if (name == "")
throw badTerm("no package name specified", e); throw badTerm("no package name specified", e);
/* Hash the Nix expression with no outputs to produce a /* Determine the output path. */
unique but deterministic path name for this package. */
if (!outHashGiven) outHash = hashPackage(state, ne); if (!outHashGiven) outHash = hashPackage(state, ne);
Path outPath = if (outPath == "")
canonPath(nixStore + "/" + ((string) outHash).c_str() + "-" + name); /* Hash the Nix expression with no outputs to produce a
unique but deterministic path name for this package. */
outPath =
canonPath(nixStore + "/" + ((string) outHash).c_str() + "-" + name);
ne.derivation.env["out"] = outPath; ne.derivation.env["out"] = outPath;
ne.derivation.outputs.insert(outPath); ne.derivation.outputs.insert(outPath);

View file

@ -122,7 +122,7 @@ Path normaliseNixExpr(const Path & _nePath, PathSet pending)
Path nePath = useSuccessor(_nePath); Path nePath = useSuccessor(_nePath);
/* Get the Nix expression. */ /* Get the Nix expression. */
NixExpr ne = parseNixExpr(termFromPath(nePath)); NixExpr ne = exprFromPath(nePath, pending);
/* If this is a normal form (i.e., a closure) we are done. */ /* If this is a normal form (i.e., a closure) we are done. */
if (ne.type == NixExpr::neClosure) return nePath; if (ne.type == NixExpr::neClosure) return nePath;
@ -172,7 +172,7 @@ Path normaliseNixExpr(const Path & _nePath, PathSet pending)
{ {
Path nePath2 = useSuccessor(nePath); Path nePath2 = useSuccessor(nePath);
if (nePath != nePath2) { if (nePath != nePath2) {
NixExpr ne = parseNixExpr(termFromPath(nePath2)); NixExpr ne = exprFromPath(nePath2, pending);
debug(format("skipping build of expression `%1%', someone beat us to it") debug(format("skipping build of expression `%1%', someone beat us to it")
% (string) nePath); % (string) nePath);
if (ne.type != NixExpr::neClosure) abort(); if (ne.type != NixExpr::neClosure) abort();
@ -193,7 +193,7 @@ Path normaliseNixExpr(const Path & _nePath, PathSet pending)
realiseClosure(nfPath, pending); realiseClosure(nfPath, pending);
/* !!! nfPath should be a root of the garbage collector while /* !!! nfPath should be a root of the garbage collector while
we are building */ we are building */
NixExpr ne = parseNixExpr(termFromPath(nfPath)); NixExpr ne = exprFromPath(nfPath, pending);
if (ne.type != NixExpr::neClosure) abort(); if (ne.type != NixExpr::neClosure) abort();
for (ClosureElems::iterator j = ne.closure.elems.begin(); for (ClosureElems::iterator j = ne.closure.elems.begin();
j != ne.closure.elems.end(); j++) j != ne.closure.elems.end(); j++)
@ -364,16 +364,49 @@ void realiseClosure(const Path & nePath, PathSet pending)
{ {
Nest nest(lvlDebug, format("realising closure `%1%'") % nePath); Nest nest(lvlDebug, format("realising closure `%1%'") % nePath);
NixExpr ne = parseNixExpr(termFromPath(nePath)); NixExpr ne = exprFromPath(nePath, pending);
if (ne.type != NixExpr::neClosure) if (ne.type != NixExpr::neClosure)
throw Error(format("expected closure in `%1%'") % nePath); throw Error(format("expected closure in `%1%'") % nePath);
for (ClosureElems::const_iterator i = ne.closure.elems.begin(); for (ClosureElems::const_iterator i = ne.closure.elems.begin();
i != ne.closure.elems.end(); i++) i != ne.closure.elems.end(); i++)
assert(isValidPath(i->first)); ensurePath(i->first, pending);
#if 0 }
expandId(i->second.id, i->first, "/", pending);
#endif
void ensurePath(const Path & path, PathSet pending)
{
/* If the path is already valid, we're done. */
if (isValidPath(path)) return;
/* Otherwise, try the substitutes. */
Paths subPaths = querySubstitutes(path);
for (Paths::iterator i = subPaths.begin();
i != subPaths.end(); i++)
{
try {
normaliseNixExpr(*i, pending);
if (isValidPath(path)) return;
throw Error(format("substitute failed to produce expected output path"));
} catch (Error & e) {
msg(lvlTalkative,
format("building of substitute `%1%' for `%2%' failed: %3%")
% *i % path % e.what());
}
}
throw Error(format("path `%1%' is required, "
"but there are no (successful) substitutes") % path);
}
NixExpr exprFromPath(const Path & path, PathSet pending)
{
ensurePath(path, pending);
ATerm t = ATreadFromNamedFile(path.c_str());
if (!t) throw Error(format("cannot read aterm from `%1%'") % path);
return parseNixExpr(t);
} }
@ -381,7 +414,7 @@ PathSet nixExprRoots(const Path & nePath)
{ {
PathSet paths; PathSet paths;
NixExpr ne = parseNixExpr(termFromPath(nePath)); NixExpr ne = exprFromPath(nePath);
if (ne.type == NixExpr::neClosure) if (ne.type == NixExpr::neClosure)
paths.insert(ne.closure.roots.begin(), ne.closure.roots.end()); paths.insert(ne.closure.roots.begin(), ne.closure.roots.end());
@ -401,7 +434,7 @@ static void requisitesWorker(const Path & nePath,
if (doneSet.find(nePath) != doneSet.end()) return; if (doneSet.find(nePath) != doneSet.end()) return;
doneSet.insert(nePath); doneSet.insert(nePath);
NixExpr ne = parseNixExpr(termFromPath(nePath)); NixExpr ne = exprFromPath(nePath);
if (ne.type == NixExpr::neClosure) if (ne.type == NixExpr::neClosure)
for (ClosureElems::iterator i = ne.closure.elems.begin(); for (ClosureElems::iterator i = ne.closure.elems.begin();

View file

@ -18,6 +18,14 @@ Path normaliseNixExpr(const Path & nePath, PathSet pending = PathSet());
its output paths through substitutes... kaboom!). */ its output paths through substitutes... kaboom!). */
void realiseClosure(const Path & nePath, PathSet pending = PathSet()); void realiseClosure(const Path & nePath, PathSet pending = PathSet());
/* Ensure that a path exists, possibly by instantiating it by
realising a substitute. */
void ensurePath(const Path & path, PathSet pending = PathSet());
/* Read a Nix expression, after ensuring its existence through
ensurePath(). */
NixExpr exprFromPath(const Path & path, PathSet pending = PathSet());
/* Get the list of root (output) paths of the given Nix expression. */ /* Get the list of root (output) paths of the given Nix expression. */
PathSet nixExprRoots(const Path & nePath); PathSet nixExprRoots(const Path & nePath);

View file

@ -47,6 +47,8 @@ static void initAndRun(int argc, char * * argv)
} }
static char buf[1024];
int main(int argc, char * * argv) int main(int argc, char * * argv)
{ {
/* ATerm setup. */ /* ATerm setup. */
@ -54,7 +56,6 @@ int main(int argc, char * * argv)
ATinit(argc, argv, &bottomOfStack); ATinit(argc, argv, &bottomOfStack);
/* Turn on buffering for cerr. */ /* Turn on buffering for cerr. */
char buf[1024];
cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); cerr.rdbuf()->pubsetbuf(buf, sizeof(buf));
try { try {

View file

@ -175,7 +175,8 @@ void registerSuccessor(const Transaction & txn,
Paths revs; Paths revs;
nixDB.queryStrings(txn, dbSuccessorsRev, sucPath, revs); nixDB.queryStrings(txn, dbSuccessorsRev, sucPath, revs);
revs.push_back(srcPath); if (find(revs.begin(), revs.end(), srcPath) == revs.end())
revs.push_back(srcPath);
nixDB.setString(txn, dbSuccessors, srcPath, sucPath); nixDB.setString(txn, dbSuccessors, srcPath, sucPath);
nixDB.setStrings(txn, dbSuccessorsRev, sucPath, revs); nixDB.setStrings(txn, dbSuccessorsRev, sucPath, revs);
@ -212,7 +213,8 @@ void registerSubstitute(const Path & srcPath, const Path & subPath)
Paths revs; Paths revs;
nixDB.queryStrings(txn, dbSubstitutesRev, subPath, revs); nixDB.queryStrings(txn, dbSubstitutesRev, subPath, revs);
revs.push_back(srcPath); if (find(revs.begin(), revs.end(), srcPath) == revs.end())
revs.push_back(srcPath);
nixDB.setStrings(txn, dbSubstitutes, srcPath, subs); nixDB.setStrings(txn, dbSubstitutes, srcPath, subs);
nixDB.setStrings(txn, dbSubstitutesRev, subPath, revs); nixDB.setStrings(txn, dbSubstitutesRev, subPath, revs);
@ -221,6 +223,14 @@ void registerSubstitute(const Path & srcPath, const Path & subPath)
} }
Paths querySubstitutes(const Path & srcPath)
{
Paths subPaths;
nixDB.queryStrings(noTxn, dbSubstitutes, srcPath, subPaths);
return subPaths;
}
void registerValidPath(const Transaction & txn, const Path & _path) void registerValidPath(const Transaction & txn, const Path & _path)
{ {
Path path(canonPath(_path)); Path path(canonPath(_path));

View file

@ -42,6 +42,9 @@ Paths queryPredecessors(const Path & sucPath);
/* Register a substitute. */ /* Register a substitute. */
void registerSubstitute(const Path & srcPath, const Path & subPath); void registerSubstitute(const Path & srcPath, const Path & subPath);
/* Return the substitutes expression for the given path. */
Paths querySubstitutes(const Path & srcPath);
/* Register the validity of a path. */ /* Register the validity of a path. */
void registerValidPath(const Transaction & txn, const Path & path); void registerValidPath(const Transaction & txn, const Path & path);