Add nix-pack utility

This transforms a set of Nix expressions into a single file. The
typical use case is to replace the top-level default.nix in Nixpkgs
with a single file that includes (almost) all Nix expressions
reachable from that file. This reduces I/O overhead, especially on
non-SSD systems.

Makefile

@@ -12,6 +12,7 @@ makefiles = \
   src/nix-daemon/local.mk \
   src/download-via-ssh/local.mk \
   src/nix-log2xml/local.mk \
+  src/nix-pack/local.mk \
   src/bsdiff-4.3/local.mk \
   perl/local.mk \
   scripts/local.mk \

src/libexpr/nixexpr.cc

@@ -396,6 +396,101 @@ void ExprPos::bindVars(const StaticEnv & env)
 }
 
 
+/* Rewriting. */
+
+Expr * Expr::rewrite(Visitor & v)
+{
+    return v(*this);
+}
+
+Expr * ExprSelect::rewrite(Visitor & v)
+{
+    e = e->rewrite(v);
+    if (def) def = def->rewrite(v);
+    return v(*this);
+}
+
+Expr * ExprOpHasAttr::rewrite(Visitor & v)
+{
+    e = e->rewrite(v);
+    return v(*this);
+}
+
+Expr * ExprAttrs::rewrite(Visitor & v)
+{
+    for (auto & a : attrs)
+        a.second.e = a.second.e->rewrite(v);
+    for (auto & a : dynamicAttrs) {
+        a.nameExpr = a.nameExpr->rewrite(v);
+        a.valueExpr = a.valueExpr->rewrite(v);
+    }
+    return v(*this);
+}
+
+Expr * ExprList::rewrite(Visitor & v)
+{
+    for (auto & e : elems)
+        e = e->rewrite(v);
+    return v(*this);
+}
+
+Expr * ExprLambda::rewrite(Visitor & v)
+{
+    if (matchAttrs)
+        for (auto & f : formals->formals)
+            if (f.def) f.def = f.def->rewrite(v);
+    body = body->rewrite(v);
+    return v(*this);
+}
+
+Expr * ExprLet::rewrite(Visitor & v)
+{
+    for (auto & a : attrs->attrs)
+        a.second.e = a.second.e->rewrite(v);
+    for (auto & a : attrs->dynamicAttrs) {
+        a.nameExpr = a.nameExpr->rewrite(v);
+        a.valueExpr = a.valueExpr->rewrite(v);
+    }
+    body = body->rewrite(v);
+    return v(*this);
+}
+
+Expr * ExprWith::rewrite(Visitor & v)
+{
+    attrs = attrs->rewrite(v);
+    body = body->rewrite(v);
+    return v(*this);
+}
+
+Expr * ExprIf::rewrite(Visitor & v)
+{
+    cond = cond->rewrite(v);
+    then = then->rewrite(v);
+    else_ = else_->rewrite(v);
+    return v(*this);
+}
+
+Expr * ExprAssert::rewrite(Visitor & v)
+{
+    cond = cond->rewrite(v);
+    body = body->rewrite(v);
+    return v(*this);
+}
+
+Expr * ExprOpNot::rewrite(Visitor & v)
+{
+    e = e->rewrite(v);
+    return v(*this);
+}
+
+Expr * ExprConcatStrings::rewrite(Visitor & v)
+{
+    for (auto & e : *es)
+        e = e->rewrite(v);
+    return v(*this);
+}
+
+
 /* Storing function names. */
 
 void Expr::setName(Symbol & name)

src/libexpr/nixexpr.hh

@@ -4,6 +4,7 @@
 #include "symbol-table.hh"
 
 #include <map>
+#include <functional>
 
 
 namespace nix {
@@ -79,6 +80,10 @@ struct Expr
     virtual void eval(EvalState & state, Env & env, Value & v);
     virtual Value * maybeThunk(EvalState & state, Env & env);
     virtual void setName(Symbol & name);
+
+    /* Perform a bottom-up rewrite of the AST. */
+    typedef std::function<Expr *(Expr & e)> Visitor;
+    virtual Expr * rewrite(Visitor & v);
 };
 
 std::ostream & operator << (std::ostream & str, Expr & e);
@@ -154,6 +159,7 @@ struct ExprSelect : Expr
     ExprSelect(const Pos & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { };
     ExprSelect(const Pos & pos, Expr * e, const Symbol & name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); };
     COMMON_METHODS
+    Expr * rewrite(Visitor & v);
 };
 
 struct ExprOpHasAttr : Expr
@@ -162,6 +168,7 @@ struct ExprOpHasAttr : Expr
     AttrPath attrPath;
     ExprOpHasAttr(Expr * e, const AttrPath & attrPath) : e(e), attrPath(attrPath) { };
     COMMON_METHODS
+    Expr * rewrite(Visitor & v);
 };
 
 struct ExprAttrs : Expr
@@ -188,6 +195,7 @@ struct ExprAttrs : Expr
     DynamicAttrDefs dynamicAttrs;
     ExprAttrs() : recursive(false) { };
     COMMON_METHODS
+    Expr * rewrite(Visitor & v);
 };
 
 struct ExprList : Expr
@@ -195,6 +203,7 @@ struct ExprList : Expr
     std::vector<Expr *> elems;
     ExprList() { };
     COMMON_METHODS
+    Expr * rewrite(Visitor & v);
 };
 
 struct Formal
@@ -230,6 +239,7 @@ struct ExprLambda : Expr
     void setName(Symbol & name);
     string showNamePos() const;
     COMMON_METHODS
+    Expr * rewrite(Visitor & v);
 };
 
 struct ExprLet : Expr
@@ -238,6 +248,7 @@ struct ExprLet : Expr
     Expr * body;
     ExprLet(ExprAttrs * attrs, Expr * body) : attrs(attrs), body(body) { };
     COMMON_METHODS
+    Expr * rewrite(Visitor & v);
 };
 
 struct ExprWith : Expr
@@ -247,6 +258,7 @@ struct ExprWith : Expr
     unsigned int prevWith;
     ExprWith(const Pos & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { };
     COMMON_METHODS
+    Expr * rewrite(Visitor & v);
 };
 
 struct ExprIf : Expr
@@ -254,6 +266,7 @@ struct ExprIf : Expr
     Expr * cond, * then, * else_;
     ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { };
     COMMON_METHODS
+    Expr * rewrite(Visitor & v);
 };
 
 struct ExprAssert : Expr
@@ -262,6 +275,7 @@ struct ExprAssert : Expr
     Expr * cond, * body;
     ExprAssert(const Pos & pos, Expr * cond, Expr * body) : pos(pos), cond(cond), body(body) { };
     COMMON_METHODS
+    Expr * rewrite(Visitor & v);
 };
 
 struct ExprOpNot : Expr
@@ -269,6 +283,7 @@ struct ExprOpNot : Expr
     Expr * e;
     ExprOpNot(Expr * e) : e(e) { };
     COMMON_METHODS
+    Expr * rewrite(Visitor & v);
 };
 
 #define MakeBinOp(name, s) \
@@ -287,6 +302,12 @@ struct ExprOpNot : Expr
             e1->bindVars(env); e2->bindVars(env); \
         } \
         void eval(EvalState & state, Env & env, Value & v); \
+        Expr * rewrite(Visitor & v) \
+        { \
+            e1 = e1->rewrite(v); \
+            e2 = e2->rewrite(v); \
+            return v(*this); \
+        } \
     };
 
 MakeBinOp(App, "")
@@ -306,6 +327,7 @@ struct ExprConcatStrings : Expr
     ExprConcatStrings(const Pos & pos, bool forceString, vector<Expr *> * es)
         : pos(pos), forceString(forceString), es(es) { };
     COMMON_METHODS
+    Expr * rewrite(Visitor & v);
 };
 
 struct ExprPos : Expr

src/nix-pack/local.mk

@@ -0,0 +1,7 @@
+programs += nix-pack
+
+nix-pack_DIR := $(d)
+
+nix-pack_SOURCES := $(d)/nix-pack.cc
+
+nix-pack_LIBS = libexpr libmain libstore libutil libformat

src/nix-pack/nix-pack.cc

@@ -0,0 +1,92 @@
+#include <iostream>
+#include <cstdlib>
+#include <queue>
+
+#include "shared.hh"
+#include "eval.hh"
+
+using namespace nix;
+
+string pathToAttrName(const Path & path)
+{
+    string res;
+    for (auto & c : path)
+        if (c == '/') res += '_';
+        else if (c == '.') res += '_';
+        else if (c == '+') res += '_';
+        else res += c;
+    return "_file_" + res;
+}
+
+void packFile(EvalState & state, Path root, const string & startFile)
+{
+    root = canonPath(root);
+
+    std::queue<string> queue({startFile});
+    std::set<string> done;
+
+    std::cout << "let\n\n";
+
+    while (!queue.empty()) {
+        string file = queue.front();
+        queue.pop();
+        if (done.find(file) != done.end()) continue;
+        done.insert(file);
+        Path path = resolveExprPath(root + "/" + file);
+        std::cerr << "processing " << path << "\n";
+
+        Expr & ast(*state.parseExprFromFile(path));
+
+        Expr::Visitor visitor = [&](Expr & e) -> Expr * {
+            ExprApp * app = dynamic_cast<ExprApp *>(&e);
+            if (app) {
+                if (!app) return &e;
+                ExprVar * var = dynamic_cast<ExprVar *>(app->e1);
+                if (!var) return &e;
+                string fnName = var->name;
+                if (fnName != "import" &&
+                    fnName != "callPackage" &&
+                    fnName != "callPackage_i686" &&
+                    fnName != "builderDefsPackage") return &e;
+                ExprPath * path = dynamic_cast<ExprPath *>(app->e2);
+                if (!path) return &e;
+                string file2 = path->s;
+                if (file2.empty() || file2[0] == '/') return &e;
+                //std::cerr << " found " << file2 << "\n";
+                queue.push(file2);
+                Expr * res = new ExprVar(state.symbols.create(pathToAttrName(file2)));
+                if ((string) var->name == "import") return res;
+                app->e2 = res;
+                return app;
+            }
+
+            ExprPath * path = dynamic_cast<ExprPath *>(&e);
+            if (path) {
+                string old = path->s;
+                if (path->s == root) path->s = "./.";
+                else if (string(path->s, 0, root.size()) == root && string(path->s, root.size(), 1) == "/") {
+                    string file2(path->s, root.size() + 1);
+                    path->s = file2.find('/') == string::npos ? "./" + file2 : file2;
+                }
+            }
+
+            return &e;
+        };
+
+        Expr * astNew = ast.rewrite(visitor);
+
+        std::cout << "# " << file << "\n";
+        std::cout << state.symbols.create(pathToAttrName(file)) << " = " << *astNew << ";\n\n";
+    }
+
+    std::cout << "in " << pathToAttrName(startFile) << "\n";
+}
+
+int main(int argc, char * * argv)
+{
+    return handleExceptions(argv[0], [&]() {
+        initNix();
+        EvalState state = EvalState(Strings());
+        packFile(state, "/home/eelco/Dev/nixpkgs-stable", "default.nix");
+    });
+}