* Substitutes and nix-pull now work again.

* Fixed a segfault caused by the buffering of stderr.
* Fix now allows the specification of the full output path.  This
  should be used with great care, since it by-passes the normal hash
  generation.
* Incremented the version number to 0.4 (prerelease).
This commit is contained in:
Eelco Dolstra 2003-10-16 16:29:57 +00:00
parent ab5e8767fa
commit 0791282b2f
12 changed files with 169 additions and 89 deletions

View file

@ -1,4 +1,4 @@
AC_INIT(nix, "0.3")
AC_INIT(nix, "0.4")
AC_CONFIG_SRCDIR(src/nix.cc)
AC_CONFIG_AUX_DIR(config)
AM_INIT_AUTOMAKE

View file

@ -1,9 +1,9 @@
Function(["nar", "name"],
Function(["nar", "outPath"],
Package(
[ ("name", Var("name"))
[ ("name", "unnar")
, ("outPath", Var("outPath"))
, ("build", Relative("nar/unnar.sh"))
, ("nar", Var("nar"))
, ("id", Var("id"))
]
)
)

View file

@ -2,11 +2,18 @@
use strict;
use IPC::Open2;
use POSIX qw(tmpnam);
my $tmpfile = "@localstatedir@/nix/pull.tmp";
my $tmpdir;
do { $tmpdir = tmpnam(); }
until mkdir $tmpdir, 0777;
my $manifest = "$tmpdir/manifest";
my $conffile = "@sysconfdir@/nix/prebuilts.conf";
my @ids;
#END { unlink $manifest; rmdir $tmpdir; }
my @srcpaths;
my @subs;
my @sucs;
@ -20,70 +27,89 @@ while (<CONFFILE>) {
chomp;
if (/^\s*(\S+)\s*(\#.*)?$/) {
my $url = $1;
$url =~ s/\/$//;
print "obtaining list of Nix archives at $url...\n";
system "wget '$url' -O '$tmpfile' 2> /dev/null"; # !!! escape
system "wget '$url'/MANIFEST -O '$manifest' 2> /dev/null"; # !!! escape
if ($?) { die "`wget' failed"; }
open INDEX, "<$tmpfile";
open MANIFEST, "<$manifest";
while (<INDEX>) {
# Get all links to prebuilts, that is, file names of the
# form foo-HASH-HASH.tar.bz2.
next unless (/HREF=\"([^\"]*)\"/);
my $fn = $1;
next if $fn =~ /\.\./;
next if $fn =~ /\//;
next unless $fn =~ /^([0-9a-z]{32})-([0-9a-z]{32})(.*)\.nar\.bz2$/;
my $hash = $1;
my $id = $2;
my $outname = $3;
my $fsid;
if ($outname =~ /^-/) {
next unless $outname =~ /^-((s-([0-9a-z]{32}))?.*)$/;
$outname = $1;
$fsid = $3;
} else {
$outname = "unnamed";
}
my $inside = 0;
print STDERR "$id ($outname)\n";
my $storepath;
my $narname;
my $hash;
my @preds;
# Construct a Fix expression that fetches and unpacks a
# Nix archive from the network.
my $fetch =
"App(IncludeFix(\"fetchurl/fetchurl.fix\"), " .
"[(\"url\", \"$url/$fn\"), (\"md5\", \"$hash\")])";
my $fixexpr =
"App(IncludeFix(\"nar/unnar.fix\"), " .
"[ (\"nar\", $fetch)" .
", (\"name\", \"$outname\")" .
", (\"id\", \"$id\")" .
"])";
if (!$first) { $fullexpr .= "," };
$first = 0;
$fullexpr .= $fixexpr; # !!! O(n^2)?
while (<MANIFEST>) {
chomp;
s/\#.*$//g;
next if (/^$/);
push @ids, $id;
if (!$inside) {
if (/^\{$/) {
$inside = 1;
undef $storepath;
undef $narname;
undef $hash;
@preds = ();
}
else { die "bad line: $_"; }
} else {
if (/^\}$/) {
$inside = 0;
my $fullurl = "$url/$narname";
print "$storepath\n";
# Does the name encode a successor relation?
if (defined $fsid) {
push @sucs, $fsid;
push @sucs, $id;
}
# Construct a Fix expression that fetches and unpacks a
# Nix archive from the network.
my $fetch =
"App(IncludeFix(\"fetchurl/fetchurl.fix\"), " .
"[(\"url\", \"$fullurl\"), (\"md5\", \"$hash\")])";
my $fixexpr =
"App(IncludeFix(\"nar/unnar.fix\"), " .
"[ (\"nar\", $fetch)" .
", (\"outPath\", \"$storepath\")" .
"])";
if (!$first) { $fullexpr .= "," };
$first = 0;
$fullexpr .= $fixexpr; # !!! O(n^2)?
push @srcpaths, $storepath;
foreach my $p (@preds) {
push @sucs, $p;
push @sucs, $storepath;
}
}
elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) {
$storepath = $1;
}
elsif (/^\s*NarName:\s*(\S+)\s*$/) {
$narname = $1;
}
elsif (/^\s*MD5:\s*(\S+)\s*$/) {
$hash = $1;
}
elsif (/^\s*SuccOf:\s*(\/\S+)\s*$/) {
push @preds, $1;
}
else { die "bad line: $_"; }
}
}
close INDEX;
unlink $tmpfile;
close MANIFEST;
}
}
$fullexpr .= "]";
# Instantiate Nix expressions from the Fix expressions we created above.
print STDERR "running fix...\n";
my $pid = open2(\*READ, \*WRITE, "fix -") or die "cannot run fix";
@ -93,23 +119,28 @@ close WRITE;
my $i = 0;
while (<READ>) {
chomp;
die unless /^([0-9a-z]{32})$/;
my $nid = $1;
die unless ($i < scalar @ids);
my $id = $ids[$i++];
push @subs, $id;
push @subs, $nid;
die unless /^\//;
my $subpath = $_;
die unless ($i < scalar @srcpaths);
my $srcpath = $srcpaths[$i++];
push @subs, $srcpath;
push @subs, $subpath;
print "$srcpath $subpath\n";
}
waitpid $pid, 0;
$? == 0 or die "fix failed";
# Register all substitutes.
print STDERR "registering substitutes...\n";
print "@subs\n";
system "nix --substitute @subs";
if ($?) { die "`nix --substitute' failed"; }
# Register all successors.
print STDERR "registering successors...\n";
system "nix --successor @sucs";
print "@sucs\n";
system "nix --successor -vvvv @sucs";
if ($?) { die "`nix --successor' failed"; }

View file

@ -1,4 +1,5 @@
#include "dotgraph.hh"
#include "normalise.hh"
static string dotQuote(const string & s)
@ -98,8 +99,8 @@ void printDotGraph(const PathSet & roots)
if (doneSet.find(nePath) == doneSet.end()) {
doneSet.insert(nePath);
NixExpr ne = parseNixExpr(termFromPath(nePath));
NixExpr ne = exprFromPath(nePath);
string label, colour;

View file

@ -22,14 +22,6 @@ Hash hashTerm(ATerm t)
}
ATerm termFromPath(const Path & path)
{
ATerm t = ATreadFromNamedFile(path.c_str());
if (!t) throw Error(format("cannot read aterm from `%1%'") % path);
return t;
}
Path writeTerm(ATerm t, const string & suffix)
{
/* The id of a term is its hash. */

View file

@ -53,9 +53,6 @@ Error badTerm(const format & f, ATerm t);
/* Hash an aterm. */
Hash hashTerm(ATerm t);
/* Read an aterm from disk. */
ATerm termFromPath(const Path & path);
/* Write an aterm to the Nix store directory, and return its path. */
Path writeTerm(ATerm t, const string & suffix);

View file

@ -299,6 +299,7 @@ static Expr evalExpr2(EvalState & state, Expr e)
ne.type = NixExpr::neDerivation;
ne.derivation.platform = SYSTEM;
string name;
Path outPath;
Hash outHash;
bool outHashGiven = false;
bnds = ATempty;
@ -327,6 +328,7 @@ static Expr evalExpr2(EvalState & state, Expr e)
if (key == "build") ne.derivation.builder = s;
if (key == "name") name = s;
if (key == "outPath") outPath = s;
if (key == "id") {
outHash = parseHash(s);
outHashGiven = true;
@ -343,11 +345,13 @@ static Expr evalExpr2(EvalState & state, Expr e)
if (name == "")
throw badTerm("no package name specified", e);
/* Hash the Nix expression with no outputs to produce a
unique but deterministic path name for this package. */
/* Determine the output path. */
if (!outHashGiven) outHash = hashPackage(state, ne);
Path outPath =
canonPath(nixStore + "/" + ((string) outHash).c_str() + "-" + name);
if (outPath == "")
/* Hash the Nix expression with no outputs to produce a
unique but deterministic path name for this package. */
outPath =
canonPath(nixStore + "/" + ((string) outHash).c_str() + "-" + name);
ne.derivation.env["out"] = outPath;
ne.derivation.outputs.insert(outPath);

View file

@ -122,7 +122,7 @@ Path normaliseNixExpr(const Path & _nePath, PathSet pending)
Path nePath = useSuccessor(_nePath);
/* Get the Nix expression. */
NixExpr ne = parseNixExpr(termFromPath(nePath));
NixExpr ne = exprFromPath(nePath, pending);
/* If this is a normal form (i.e., a closure) we are done. */
if (ne.type == NixExpr::neClosure) return nePath;
@ -172,7 +172,7 @@ Path normaliseNixExpr(const Path & _nePath, PathSet pending)
{
Path nePath2 = useSuccessor(nePath);
if (nePath != nePath2) {
NixExpr ne = parseNixExpr(termFromPath(nePath2));
NixExpr ne = exprFromPath(nePath2, pending);
debug(format("skipping build of expression `%1%', someone beat us to it")
% (string) nePath);
if (ne.type != NixExpr::neClosure) abort();
@ -193,7 +193,7 @@ Path normaliseNixExpr(const Path & _nePath, PathSet pending)
realiseClosure(nfPath, pending);
/* !!! nfPath should be a root of the garbage collector while
we are building */
NixExpr ne = parseNixExpr(termFromPath(nfPath));
NixExpr ne = exprFromPath(nfPath, pending);
if (ne.type != NixExpr::neClosure) abort();
for (ClosureElems::iterator j = ne.closure.elems.begin();
j != ne.closure.elems.end(); j++)
@ -364,16 +364,49 @@ void realiseClosure(const Path & nePath, PathSet pending)
{
Nest nest(lvlDebug, format("realising closure `%1%'") % nePath);
NixExpr ne = parseNixExpr(termFromPath(nePath));
NixExpr ne = exprFromPath(nePath, pending);
if (ne.type != NixExpr::neClosure)
throw Error(format("expected closure in `%1%'") % nePath);
for (ClosureElems::const_iterator i = ne.closure.elems.begin();
i != ne.closure.elems.end(); i++)
assert(isValidPath(i->first));
#if 0
expandId(i->second.id, i->first, "/", pending);
#endif
ensurePath(i->first, pending);
}
void ensurePath(const Path & path, PathSet pending)
{
/* If the path is already valid, we're done. */
if (isValidPath(path)) return;
/* Otherwise, try the substitutes. */
Paths subPaths = querySubstitutes(path);
for (Paths::iterator i = subPaths.begin();
i != subPaths.end(); i++)
{
try {
normaliseNixExpr(*i, pending);
if (isValidPath(path)) return;
throw Error(format("substitute failed to produce expected output path"));
} catch (Error & e) {
msg(lvlTalkative,
format("building of substitute `%1%' for `%2%' failed: %3%")
% *i % path % e.what());
}
}
throw Error(format("path `%1%' is required, "
"but there are no (successful) substitutes") % path);
}
NixExpr exprFromPath(const Path & path, PathSet pending)
{
ensurePath(path, pending);
ATerm t = ATreadFromNamedFile(path.c_str());
if (!t) throw Error(format("cannot read aterm from `%1%'") % path);
return parseNixExpr(t);
}
@ -381,7 +414,7 @@ PathSet nixExprRoots(const Path & nePath)
{
PathSet paths;
NixExpr ne = parseNixExpr(termFromPath(nePath));
NixExpr ne = exprFromPath(nePath);
if (ne.type == NixExpr::neClosure)
paths.insert(ne.closure.roots.begin(), ne.closure.roots.end());
@ -401,7 +434,7 @@ static void requisitesWorker(const Path & nePath,
if (doneSet.find(nePath) != doneSet.end()) return;
doneSet.insert(nePath);
NixExpr ne = parseNixExpr(termFromPath(nePath));
NixExpr ne = exprFromPath(nePath);
if (ne.type == NixExpr::neClosure)
for (ClosureElems::iterator i = ne.closure.elems.begin();

View file

@ -18,6 +18,14 @@ Path normaliseNixExpr(const Path & nePath, PathSet pending = PathSet());
its output paths through substitutes... kaboom!). */
void realiseClosure(const Path & nePath, PathSet pending = PathSet());
/* Ensure that a path exists, possibly by instantiating it by
realising a substitute. */
void ensurePath(const Path & path, PathSet pending = PathSet());
/* Read a Nix expression, after ensuring its existence through
ensurePath(). */
NixExpr exprFromPath(const Path & path, PathSet pending = PathSet());
/* Get the list of root (output) paths of the given Nix expression. */
PathSet nixExprRoots(const Path & nePath);

View file

@ -47,6 +47,8 @@ static void initAndRun(int argc, char * * argv)
}
static char buf[1024];
int main(int argc, char * * argv)
{
/* ATerm setup. */
@ -54,7 +56,6 @@ int main(int argc, char * * argv)
ATinit(argc, argv, &bottomOfStack);
/* Turn on buffering for cerr. */
char buf[1024];
cerr.rdbuf()->pubsetbuf(buf, sizeof(buf));
try {

View file

@ -175,7 +175,8 @@ void registerSuccessor(const Transaction & txn,
Paths revs;
nixDB.queryStrings(txn, dbSuccessorsRev, sucPath, revs);
revs.push_back(srcPath);
if (find(revs.begin(), revs.end(), srcPath) == revs.end())
revs.push_back(srcPath);
nixDB.setString(txn, dbSuccessors, srcPath, sucPath);
nixDB.setStrings(txn, dbSuccessorsRev, sucPath, revs);
@ -212,7 +213,8 @@ void registerSubstitute(const Path & srcPath, const Path & subPath)
Paths revs;
nixDB.queryStrings(txn, dbSubstitutesRev, subPath, revs);
revs.push_back(srcPath);
if (find(revs.begin(), revs.end(), srcPath) == revs.end())
revs.push_back(srcPath);
nixDB.setStrings(txn, dbSubstitutes, srcPath, subs);
nixDB.setStrings(txn, dbSubstitutesRev, subPath, revs);
@ -221,6 +223,14 @@ void registerSubstitute(const Path & srcPath, const Path & subPath)
}
Paths querySubstitutes(const Path & srcPath)
{
Paths subPaths;
nixDB.queryStrings(noTxn, dbSubstitutes, srcPath, subPaths);
return subPaths;
}
void registerValidPath(const Transaction & txn, const Path & _path)
{
Path path(canonPath(_path));

View file

@ -42,6 +42,9 @@ Paths queryPredecessors(const Path & sucPath);
/* Register a substitute. */
void registerSubstitute(const Path & srcPath, const Path & subPath);
/* Return the substitutes expression for the given path. */
Paths querySubstitutes(const Path & srcPath);
/* Register the validity of a path. */
void registerValidPath(const Transaction & txn, const Path & path);