* Sync with thesis: do not include store location in the hash

computation (in the intensional model).

src/libexpr/primops.cc

@@ -1,6 +1,7 @@
 #include "build.hh"
 #include "eval.hh"
 #include "globals.hh"
+#include "misc.hh"
 #include "nixexpr-ast.hh"
 
 
@@ -71,8 +72,8 @@ static Hash hashDerivationModulo(EvalState & state, Derivation drv)
         {
             return hashString(htSHA256, "fixed:out:"
                 + i->second.hashAlgo + ":"
-                + i->second.hash + ":"
-                + i->second.path);
+                + i->second.hash /* !!! + ":"
+                                    + i->second.path */);
         }
     }
 
@@ -319,8 +320,10 @@ static Expr primDerivationStrict(EvalState & state, const ATermVector & args)
         
     /* Use the masked derivation expression to compute the output
        path. */
-    Path outPath = makeStorePath("output:out",
-        hashDerivationModulo(state, drv), drvName);
+    /* XXX */
+    Path outPath;
+    PathHash outPathHash;
+    makeStorePath(hashDerivationModulo(state, drv), drvName, outPath, outPathHash);
 
     /* Construct the final derivation store expression. */
     drv.env["out"] = outPath;

src/libmain/shared.cc

@@ -139,6 +139,21 @@ static void initAndRun(int argc, char * * argv)
 
     /* ATerm stuff.  !!! find a better place to put this */
     initDerivationsHelpers();
+
+    /* Random number generator needed by makeRandomStorePath(); !!!
+       improve. */
+    srand(time(0));
+
+    /* Set the trust ID to the user name. */
+    currentTrustId = getEnv("NIX_USER_ID"); /* !!! dangerous? */
+    if (currentTrustId == "") {
+        SwitchToOriginalUser sw;
+        uid_t uid = geteuid();
+        struct passwd * pw = getpwuid(uid);
+        if (!pw) throw Error(format("unknown user ID %1%, go away") % uid);
+        currentTrustId = pw->pw_name;
+    }
+    printMsg(lvlError, format("trust ID is `%1%'") % currentTrustId);
     
     /* Put the arguments in a vector. */
     Strings args, remaining;

src/libstore/Makefile.am

@@ -2,7 +2,7 @@ noinst_LIBRARIES = libstore.a
 
 libstore_a_SOURCES = \
  store.cc store.hh derivations.cc derivations.hh \
- build.cc misc.cc build.hh \
+ build.cc build.hh misc.cc misc.hh \
  globals.cc globals.hh db.cc db.hh \
  references.cc references.hh pathlocks.cc pathlocks.hh \
  gc.cc gc.hh derivations-ast.hh
@@ -15,4 +15,4 @@ AM_CXXFLAGS = -Wall \
 derivations-ast.cc derivations-ast.hh: ../aterm-helper.pl derivations-ast.def
 	$(perl) ../aterm-helper.pl derivations-ast.hh derivations-ast.cc < derivations-ast.def
 
-derivations.cc store.cc: derivations-ast.hh
+derivations.cc store.cc: derivations-ast.hh

src/libstore/build.cc

@@ -14,6 +14,7 @@
 #include "pathlocks.hh"
 #include "globals.hh"
 #include "gc.hh"
+#include "misc.hh"
 
 
 /* !!! TODO derivationFromPath shouldn't be used here */
@@ -314,6 +315,16 @@ private:
     
     /* The remainder is state held during the build. */
 
+    /* The map of output equivalence classes to temporary output
+       paths. */
+    typedef map<OutputEqClass, Path> OutputMap;
+    OutputMap tmpOutputs;
+
+    /* The hash rewrite map that rewrites output equivalences occuring
+       in the command-line arguments and environment variables to the
+       actual paths to be used. */
+    HashRewrites rewrites;
+
     /* Locks on the output paths. */
     PathLocks outputLocks;
 
@@ -399,7 +410,8 @@ private:
     void writeLog(int fd, const unsigned char * buf, size_t count);
 
     /* Return the set of (in)valid paths. */
-    PathSet checkPathValidity(bool returnValid);
+    typedef set<OutputEqClass> OutputEqClasses;
+    OutputEqClasses checkOutputValidity(bool returnValid);
 };
 
 
@@ -463,12 +475,15 @@ void DerivationGoal::haveStoreExpr()
     /* Get the derivation. */
     drv = derivationFromPath(drvPath);
 
+#if 0    
     for (DerivationOutputs::iterator i = drv.outputs.begin();
          i != drv.outputs.end(); ++i)
         addTempRoot(i->second.path);
+#endif    
 
-    /* Check what outputs paths are not already valid. */
-    PathSet invalidOutputs = checkPathValidity(false);
+    /* Check for what output path equivalence classes we do not
+       already have valid, trusted output paths. */
+    OutputEqClasses invalidOutputs = checkOutputValidity(false);
 
     /* If they are all valid, then we're done. */
     if (invalidOutputs.size() == 0) {
@@ -476,6 +491,7 @@ void DerivationGoal::haveStoreExpr()
         return;
     }
 
+#if 0    
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
@@ -485,6 +501,7 @@ void DerivationGoal::haveStoreExpr()
            substitutes. */
         if (querySubstitutes(noTxn, *i).size() > 0)
             addWaitee(worker.makeSubstitutionGoal(*i));
+#endif    
 
     if (waitees.empty()) /* to prevent hang (no wake-up event) */
         outputsSubstituted();
@@ -502,7 +519,7 @@ void DerivationGoal::outputsSubstituted()
 
     nrFailed = 0;
 
-    if (checkPathValidity(false).size() == 0) {
+    if (checkOutputValidity(false).size() == 0) {
         amDone(true);
         return;
     }
@@ -647,6 +664,7 @@ void DerivationGoal::buildDone()
 }
 
 
+#if 0
 static string readLine(int fd)
 {
     string s;
@@ -686,33 +704,27 @@ static void drain(int fd)
         else writeFull(STDERR_FILENO, buffer, rd);
     }
 }
+#endif
 
 
+#if 0
 PathSet outputPaths(const DerivationOutputs & outputs)
 {
     PathSet paths;
+    /* XXX */
     for (DerivationOutputs::const_iterator i = outputs.begin();
          i != outputs.end(); ++i)
         paths.insert(i->second.path);
     return paths;
 }
-
-
-string showPaths(const PathSet & paths)
-{
-    string s;
-    for (PathSet::const_iterator i = paths.begin();
-         i != paths.end(); ++i)
-    {
-        if (s.size() != 0) s += ", ";
-        s += "`" + *i + "'";
-    }
-    return s;
-}
+#endif
 
 
 DerivationGoal::HookReply DerivationGoal::tryBuildHook()
 {
+    return rpDecline;
+    
+#if 0    
     Path buildHook = getEnv("NIX_BUILD_HOOK");
     if (buildHook == "") return rpDecline;
     buildHook = absPath(buildHook);
@@ -861,6 +873,7 @@ DerivationGoal::HookReply DerivationGoal::tryBuildHook()
     }
 
     else throw Error(format("bad hook reply `%1%'") % reply);
+#endif
 }
 
 
@@ -881,11 +894,42 @@ void DerivationGoal::terminateBuildHook()
 
 bool DerivationGoal::prepareBuild()
 {
+    /* We direct each output of the derivation to a temporary location
+       in the Nix store.  Afterwards, we move the outputs to their
+       final, content-addressed location. */
+    for (DerivationOutputs::iterator i = drv.outputs.begin();
+         i != drv.outputs.end(); ++i)
+    {
+        Path tmpPath = makeRandomStorePath(namePartOf(i->second.eqClass));
+        printMsg(lvlError, format("mapping output id `%1%', class `%2%' to `%3%'")
+            % i->first % i->second.eqClass % tmpPath);
+        assert(i->second.eqClass.size() == tmpPath.size());
+
+        debug(format("building path `%1%'") % tmpPath);
+        
+        tmpOutputs[i->second.eqClass] = tmpPath;
+
+        /* This is a referenceable path.  Make a note of that for when
+           we are scanning for references in the output. */
+        allPaths.insert(tmpPath);
+        
+        /* The environment variables and command-line arguments of the
+           builder refer to the output path equivalence class.  Cause
+           those references to be rewritten to the temporary
+           locations. */
+        rewrites[hashPartOf(i->second.eqClass)] = hashPartOf(tmpPath);
+    }
+
     /* Obtain locks on all output paths.  The locks are automatically
        released when we exit this function or Nix crashes. */
     /* !!! BUG: this could block, which is not allowed. */
+#if 0
+    /* !!! acquire lock on the derivation or something? or on a
+       pseudo-path representing the output equivalence class? */
     outputLocks.lockPaths(outputPaths(drv.outputs));
+#endif
 
+#if 0    
     /* Now check again whether the outputs are valid.  This is because
        another process may have started building in parallel.  After
        it has finished and released the locks, we can (and should)
@@ -907,17 +951,7 @@ bool DerivationGoal::prepareBuild()
             format("derivation `%1%' is blocked by its output paths")
             % drvPath);
     }
-
-    /* Gather information necessary for computing the closure and/or
-       running the build hook. */
-    
-    /* The outputs are referenceable paths. */
-    for (DerivationOutputs::iterator i = drv.outputs.begin();
-         i != drv.outputs.end(); ++i)
-    {
-        debug(format("building path `%1%'") % i->second.path);
-        allPaths.insert(i->second.path);
-    }
+#endif    
 
     /* Determine the full set of input paths. */
 
@@ -930,14 +964,22 @@ bool DerivationGoal::prepareBuild()
            that are specified as inputs. */
         assert(isValidPath(i->first));
         Derivation inDrv = derivationFromPath(i->first);
+
         for (StringSet::iterator j = i->second.begin();
              j != i->second.end(); ++j)
-            if (inDrv.outputs.find(*j) != inDrv.outputs.end())
-                computeFSClosure(inDrv.outputs[*j].path, inputPaths);
-            else
-                throw Error(
-                    format("derivation `%1%' requires non-existent output `%2%' from input derivation `%3%'")
-                    % drvPath % *j % i->first);
+        {
+            OutputEqClass eqClass = findOutputEqClass(inDrv, *j);
+            PathSet inputs = findTrustedEqClassMembers(eqClass, currentTrustId);
+            if (inputs.size() == 0)
+                /* !!! shouldn't happen, except for garbage
+                   collection? */
+                throw Error(format("output `%1%' of derivation `%2%' is missing!")
+                    % *j % i->first);
+            for (PathSet::iterator k = inputs.begin(); k != inputs.end(); ++k) {
+                rewrites[hashPartOf(eqClass)] = hashPartOf(*k);
+                computeFSClosure(*k, inputPaths);
+            }
+        }
     }
 
     /* Second, the input sources. */
@@ -945,7 +987,31 @@ bool DerivationGoal::prepareBuild()
          i != drv.inputSrcs.end(); ++i)
         computeFSClosure(*i, inputPaths);
 
-    debug(format("added input paths %1%") % showPaths(inputPaths));
+    /* There might be equivalence class collisions now.  That is,
+       different input closures might contain different paths from the
+       *same* output path equivalence class.  We should pick one from
+       each, and rewrite dependent paths. */
+    Replacements replacements;
+    inputPaths = consolidatePaths(inputPaths, false, replacements);
+
+    HashRewrites rewrites2;
+    for (Replacements::iterator i = replacements.begin();
+         i != replacements.end(); ++i)
+    {
+        printMsg(lvlError, format("HASH REWRITE %1% %2%")
+            % hashPartOf(i->first).toString() % hashPartOf(i->second).toString());
+        rewrites2[hashPartOf(i->first)] = hashPartOf(i->second);
+    }
+
+    for (HashRewrites::iterator i = rewrites.begin();
+         i != rewrites.end(); ++i)
+        rewrites[i->first] = PathHash(rewriteHashes(i->second.toString(), rewrites2));
+    
+    /* !!! remove, debug only */
+    Replacements dummy;
+    consolidatePaths(inputPaths, true, dummy);
+    
+    printMsg(lvlError, format("added input paths %1%") % showPaths(inputPaths)); /* !!! */
 
     allPaths.insert(inputPaths.begin(), inputPaths.end());
 
@@ -956,7 +1022,7 @@ bool DerivationGoal::prepareBuild()
 void DerivationGoal::startBuilder()
 {
     startNest(nest, lvlInfo,
-        format("building path(s) %1%") % showPaths(outputPaths(drv.outputs)))
+        format("building derivation `%1%'") % drvPath)
     
     /* Right platform? */
     if (drv.platform != thisSystem)
@@ -966,6 +1032,7 @@ void DerivationGoal::startBuilder()
 
     /* If any of the outputs already exist but are not registered,
        delete them. */
+#if 0    
     for (DerivationOutputs::iterator i = drv.outputs.begin(); 
          i != drv.outputs.end(); ++i)
     {
@@ -977,6 +1044,7 @@ void DerivationGoal::startBuilder()
             deletePath(path);
         }
     }
+#endif
 
     /* Construct the environment passed to the builder. */
     typedef map<string, string> Environment;
@@ -1004,7 +1072,7 @@ void DerivationGoal::startBuilder()
     /* Add all bindings specified in the derivation. */
     for (StringPairs::iterator i = drv.env.begin();
          i != drv.env.end(); ++i)
-        env[i->first] = i->second;
+        env[i->first] = rewriteHashes(i->second, rewrites);
 
     /* Create a temporary directory where the build will take
        place. */
@@ -1027,8 +1095,8 @@ void DerivationGoal::startBuilder()
             env["NIX_OUTPUT_CHECKED"] = "1";
 
     /* Run the builder. */
-    printMsg(lvlChatty, format("executing builder `%1%'") %
-        drv.builder);
+    string builder = rewriteHashes(drv.builder, rewrites);
+    printMsg(lvlChatty, format("executing builder `%1%'") % builder);
 
     /* Create the log file and pipe. */
     openLogFile();
@@ -1054,22 +1122,22 @@ void DerivationGoal::startBuilder()
 
             /* Fill in the arguments. */
             Strings args(drv.args);
-            args.push_front(baseNameOf(drv.builder));
+            args.push_front(baseNameOf(builder));
             const char * * argArr = strings2CharPtrs(args);
 
             /* Fill in the environment. */
             Strings envStrs;
             for (Environment::const_iterator i = env.begin();
                  i != env.end(); ++i)
-                envStrs.push_back(i->first + "=" + i->second);
+                envStrs.push_back(i->first + "=" +
+                    rewriteHashes(i->second, rewrites));
             const char * * envArr = strings2CharPtrs(envStrs);
 
             /* Execute the program.  This should not return. */
-            execve(drv.builder.c_str(),
+            execve(builder.c_str(),
                 (char * *) argArr, (char * *) envArr);
 
-            throw SysError(format("executing `%1%'")
-                % drv.builder);
+            throw SysError(format("executing `%1%'") % builder);
             
         } catch (exception & e) {
             cerr << format("build error: %1%\n") % e.what();
@@ -1089,23 +1157,17 @@ void DerivationGoal::computeClosure()
 {
     map<Path, PathSet> allReferences;
     map<Path, Hash> contentHashes;
-    
-    /* Check whether the output paths were created, and grep each
-       output path to determine what other paths it references.  Also make all
-       output paths read-only. */
+
     for (DerivationOutputs::iterator i = drv.outputs.begin(); 
          i != drv.outputs.end(); ++i)
     {
-        Path path = i->second.path;
+        Path path = tmpOutputs[i->second.eqClass];
         if (!pathExists(path)) {
             throw BuildError(
                 format("builder for `%1%' failed to produce output path `%2%'")
                 % drvPath % path);
         }
-
-        startNest(nest, lvlTalkative,
-            format("scanning for references inside `%1%'") % path);
-
+        
         /* Check that fixed-output derivations produced the right
            outputs (i.e., the content hash should match the specified
            hash). */ 
@@ -1143,62 +1205,51 @@ void DerivationGoal::computeClosure()
                     % path % algo % printHash(h) % printHash(h2));
         }
 
-	canonicalisePathMetaData(path);
-
-	/* For this output path, find the references to other paths contained
-	   in it. */
-        PathSet references;
+	/* For this output path, find the references to other paths
+	   contained in it. */
+        PathSet referenced;
         if (!pathExists(path + "/nix-support/no-scan")) {
-            Paths references2;
-            references2 = filterReferences(path, 
+            Paths referenced2 = filterReferences(path, 
                 Paths(allPaths.begin(), allPaths.end()));
-            references = PathSet(references2.begin(), references2.end());
+            referenced = PathSet(referenced2.begin(), referenced2.end());
 
             /* For debugging, print out the referenced and
                unreferenced paths. */
-            for (PathSet::iterator i = inputPaths.begin();
-                 i != inputPaths.end(); ++i)
-            {
-                PathSet::iterator j = references.find(*i);
-                if (j == references.end())
-                    debug(format("unreferenced input: `%1%'") % *i);
-                else
-                    debug(format("referenced input: `%1%'") % *i);
-            }
+            PathSet unreferenced;
+            insert_iterator<PathSet> ins(unreferenced, unreferenced.begin());
+            set_difference(
+                inputPaths.begin(), inputPaths.end(),
+                referenced.begin(), referenced.end(), ins);
+            printMsg(lvlError, format("unreferenced inputs: %1%") % showPaths(unreferenced));
+            printMsg(lvlError, format("referenced inputs: %1%") % showPaths(referenced));
         }
 
-        allReferences[path] = references;
+        /* Rewrite each output to a name matching its content hash.
+           I.e., enforce the hash invariant: the hash part of a store
+           path matches the contents at that path.
 
-        /* Hash the contents of the path.  The hash is stored in the
-           database so that we can verify later on whether nobody has
-           messed with the store.  !!! inefficient: it would be nice
-           if we could combine this with filterReferences(). */
-        contentHashes[path] = hashPath(htSHA256, path);
+           This also registers the final output path as valid, and
+           sets it references. */
+        Path finalPath = addToStore(path,
+            hashPartOf(path), namePartOf(path),
+            referenced);
+        printMsg(lvlError, format("produced final path `%1%'") % finalPath);
+
+        /* Register the fact that this output path is a member of some
+           output path equivalence class (for a certain user, at
+           least).  This is how subsequent derivations will be able to
+           find it. */
+        Transaction txn;
+        createStoreTransaction(txn);
+        addOutputEqMember(txn, i->second.eqClass, currentTrustId, finalPath);
+        txn.commit();
+
+        /* Get rid of the temporary output.  !!! optimise all this by
+           *moving* the temporary output to the new location and
+           applying rewrites in situ. */
+        deletePath(path);
     }
-
-    /* Register each output path as valid, and register the sets of
-       paths referenced by each of them.  This is wrapped in one
-       database transaction to ensure that if we crash, either
-       everything is registered or nothing is.  This is for
-       recoverability: unregistered paths in the store can be deleted
-       arbitrarily, while registered paths can only be deleted by
-       running the garbage collector.
-
-       The reason that we do the transaction here and not on the fly
-       while we are scanning (above) is so that we don't hold database
-       locks for too long. */
-    Transaction txn;
-    createStoreTransaction(txn);
-    for (DerivationOutputs::iterator i = drv.outputs.begin(); 
-         i != drv.outputs.end(); ++i)
-    {
-        registerValidPath(txn, i->second.path,
-            contentHashes[i->second.path],
-            allReferences[i->second.path],
-            drvPath);
-    }
-    txn.commit();
-
+    
     /* It is now safe to delete the lock files, since all future
        lockers will see that the output paths are valid; they will not
        create new lock files with the same names as the old (unlinked)
@@ -1278,16 +1329,19 @@ void DerivationGoal::writeLog(int fd,
 }
 
 
-PathSet DerivationGoal::checkPathValidity(bool returnValid)
+DerivationGoal::OutputEqClasses DerivationGoal::checkOutputValidity(bool returnValid)
 {
-    PathSet result;
+    OutputEqClasses result;
     for (DerivationOutputs::iterator i = drv.outputs.begin();
          i != drv.outputs.end(); ++i)
-        if (isValidPath(i->second.path)) {
-            if (returnValid) result.insert(i->second.path);
+    {
+        PathSet paths = findTrustedEqClassMembers(i->second.eqClass, currentTrustId);
+        if (paths.size() > 0) {
+            if (returnValid) result.insert(i->second.eqClass);
         } else {
-            if (!returnValid) result.insert(i->second.path);
+            if (!returnValid) result.insert(i->second.eqClass);
         }
+    }
     return result;
 }
 
@@ -1302,11 +1356,13 @@ private:
     /* The store path that should be realised through a substitute. */
     Path storePath;
 
+#if 0    
     /* The remaining substitutes for this path. */
     Substitutes subs;
 
     /* The current substitute. */
     Substitute sub;
+#endif    
 
     /* Outgoing references for this path. */
     PathSet references;
@@ -1375,10 +1431,12 @@ void SubstitutionGoal::init()
         return;
     }
 
+#if 0
     /* Read the substitutes. */
     subs = querySubstitutes(noTxn, storePath);
+#endif
 
-    /* To maintain the closure invairant, we first have to realise the
+    /* To maintain the closure invariant, we first have to realise the
        paths referenced by this one. */
     queryReferences(noTxn, storePath, references);
 
@@ -1414,7 +1472,7 @@ void SubstitutionGoal::tryNext()
 {
     trace("trying next substitute");
 
-    if (subs.size() == 0) {
+    if (true /* !!! subs.size() == 0 */) {
         /* None left.  Terminate this goal and let someone else deal
            with it. */
         printMsg(lvlError,
@@ -1423,8 +1481,10 @@ void SubstitutionGoal::tryNext()
         amDone(false);
         return;
     }
+#if 0    
     sub = subs.front();
     subs.pop_front();
+#endif
 
     /* Wait until we can run the substitute program. */
     state = &SubstitutionGoal::tryToRun;
@@ -1434,6 +1494,7 @@ void SubstitutionGoal::tryNext()
 
 void SubstitutionGoal::tryToRun()
 {
+#if 0
     trace("trying to run");
 
     /* Make sure that we are allowed to start a build. */
@@ -1505,11 +1566,13 @@ void SubstitutionGoal::tryToRun()
         pid, logPipe.readSide, true);
 
     state = &SubstitutionGoal::finished;
+#endif    
 }
 
 
 void SubstitutionGoal::finished()
 {
+#if 0    
     trace("substitute finished");
 
     /* Since we got an EOF on the logger pipe, the substitute is
@@ -1566,6 +1629,7 @@ void SubstitutionGoal::finished()
         format("substitution of path `%1%' succeeded") % storePath);
 
     amDone();
+#endif
 }
 
 

src/libstore/build.hh

@@ -3,6 +3,7 @@
 
 #include "derivations.hh"
 
+
 /* Ensure that the output paths of the derivation are valid.  If they
    are already valid, this is a no-op.  Otherwise, validity can
    be reached in two ways.  First, if the output paths have
@@ -11,26 +12,10 @@
    sub-derivations. */
 void buildDerivations(const PathSet & drvPaths);
 
+
 /* Ensure that a path is valid.  If it is not currently valid, it may
    be made valid by running a substitute (if defined for the path). */
 void ensurePath(const Path & storePath);
 
-/* Read a derivation, after ensuring its existence through
-   ensurePath(). */
-Derivation derivationFromPath(const Path & drvPath);
-
-/* Place in `paths' the set of all store paths in the file system
-   closure of `storePath'; that is, all paths than can be directly or
-   indirectly reached from it.  `paths' is not cleared.  If
-   `flipDirection' is true, the set of paths that can reach
-   `storePath' is returned; that is, the closures under the `referers'
-   relation instead of the `references' relation is returned. */
-void computeFSClosure(const Path & storePath,
-    PathSet & paths, bool flipDirection = false);
-
-/* Return the path corresponding to the output identifier `id' in the
-   given derivation. */
-Path findOutput(const Derivation & drv, string id);
-    
 
 #endif /* !__BUILD_H */

src/libstore/derivations.cc

@@ -62,12 +62,12 @@ Derivation parseDerivation(ATerm t)
         throwBadDrv(t);
 
     for (ATermIterator i(outs); i; ++i) {
-        ATerm id, path, hashAlgo, hash;
-        if (!matchDerivationOutput(*i, id, path, hashAlgo, hash))
+        ATerm id, eqClass, hashAlgo, hash;
+        if (!matchDerivationOutput(*i, id, eqClass, hashAlgo, hash))
             throwBadDrv(t);
         DerivationOutput out;
-        out.path = aterm2String(path);
-        checkPath(out.path);
+        out.eqClass = aterm2String(eqClass);
+        // !!!        checkPath(out.path);
         out.hashAlgo = aterm2String(hashAlgo);
         out.hash = aterm2String(hash);
         drv.outputs[aterm2String(id)] = out;
@@ -125,7 +125,7 @@ ATerm unparseDerivation(const Derivation & drv)
         outputs = ATinsert(outputs,
             makeDerivationOutput(
                 toATerm(i->first),
-                toATerm(i->second.path),
+                toATerm(i->second.eqClass),
                 toATerm(i->second.hashAlgo),
                 toATerm(i->second.hash)));
 

src/libstore/derivations.hh

@@ -13,15 +13,15 @@ const string drvExtension = ".drv";
 
 struct DerivationOutput
 {
-    Path path;
+    OutputEqClass eqClass;
     string hashAlgo; /* hash used for expected hash computation */
     string hash; /* expected hash, may be null */
     DerivationOutput()
     {
     }
-    DerivationOutput(Path path, string hashAlgo, string hash)
+    DerivationOutput(OutputEqClass eqClass, string hashAlgo, string hash)
     {
-        this->path = path;
+        this->eqClass = eqClass;
         this->hashAlgo = hashAlgo;
         this->hash = hash;
     }

src/libstore/gc.cc

@@ -2,6 +2,7 @@
 #include "gc.hh"
 #include "build.hh"
 #include "pathlocks.hh"
+#include "misc.hh"
 
 #include <boost/shared_ptr.hpp>
 
@@ -338,12 +339,14 @@ void collectGarbage(GCAction action, PathSet & result)
         for (PathSet::iterator i = livePaths.begin();
              i != livePaths.end(); ++i)
         {
+#if 0            
             /* Note that the deriver need not be valid (e.g., if we
                previously ran the collector with `gcKeepDerivations'
                turned off). */
             Path deriver = queryDeriver(noTxn, *i);
             if (deriver != "" && isValidPath(deriver))
                 computeFSClosure(deriver, livePaths);
+#endif            
         }
     }
 
@@ -353,10 +356,13 @@ void collectGarbage(GCAction action, PathSet & result)
              i != livePaths.end(); ++i)
             if (isDerivation(*i)) {
                 Derivation drv = derivationFromPath(*i);
+                assert(0);
+#if 0                
                 for (DerivationOutputs::iterator j = drv.outputs.begin();
                      j != drv.outputs.end(); ++j)
                     if (isValidPath(j->second.path))
                         computeFSClosure(j->second.path, livePaths);
+#endif
             }
     }
 

src/libstore/globals.cc

@@ -22,6 +22,8 @@ unsigned int maxBuildJobs = 1;
 
 bool readOnlyMode = false;
 
+string currentTrustId;
+
 
 static bool settingsRead = false;
 

src/libstore/globals.hh

@@ -52,6 +52,10 @@ extern unsigned int maxBuildJobs;
    database. */
 extern bool readOnlyMode;
 
+/* Current trust ID.  !!! Of course, this shouldn't be a global
+   variable. */
+extern string currentTrustId;
+
 
 string querySetting(const string & name, const string & def);
 

src/libstore/misc.cc

@@ -1,4 +1,6 @@
 #include "build.hh"
+#include "misc.hh"
+#include "globals.hh"
 
 
 Derivation derivationFromPath(const Path & drvPath)
@@ -29,10 +31,224 @@ void computeFSClosure(const Path & storePath,
 }
 
 
- Path findOutput(const Derivation & drv, string id)
+OutputEqClass findOutputEqClass(const Derivation & drv, const string & id)
 {
-    for (DerivationOutputs::const_iterator i = drv.outputs.begin();
-         i != drv.outputs.end(); ++i)
-        if (i->first == id) return i->second.path;
-    throw Error(format("derivation has no output `%1%'") % id);
+    DerivationOutputs::const_iterator i = drv.outputs.find(id);
+    if (i == drv.outputs.end())
+        throw Error(format("derivation has no output `%1%'") % id);
+    return i->second.eqClass;
+}
+
+
+PathSet findTrustedEqClassMembers(const OutputEqClass & eqClass,
+    const TrustId & trustId)
+{
+    OutputEqMembers members;
+    queryOutputEqMembers(noTxn, eqClass, members);
+
+    PathSet result;
+    for (OutputEqMembers::iterator j = members.begin(); j != members.end(); ++j)
+        if (j->trustId == trustId || j->trustId == "root") result.insert(j->path);
+
+    return result;
+}
+
+
+Path findTrustedEqClassMember(const OutputEqClass & eqClass,
+    const TrustId & trustId)
+{
+    PathSet paths = findTrustedEqClassMembers(eqClass, trustId);
+    if (paths.size() == 0)
+        throw Error(format("no output path in equivalence class `%1%' is known") % eqClass);
+    return *(paths.begin());
+}
+
+
+typedef map<OutputEqClass, PathSet> ClassMap;
+typedef map<OutputEqClass, Path> FinalClassMap;
+
+
+static void findBestRewrite(const ClassMap::const_iterator & pos,
+    const ClassMap::const_iterator & end,
+    const PathSet & selection, const PathSet & unselection,
+    unsigned int & bestCost, PathSet & bestSelection)
+{
+    if (pos != end) {
+        for (PathSet::iterator i = pos->second.begin();
+             i != pos->second.end(); ++i)
+        {
+            PathSet selection2(selection);
+            selection2.insert(*i);
+            
+            PathSet unselection2(unselection);
+            for (PathSet::iterator j = pos->second.begin();
+                 j != pos->second.end(); ++j)
+                if (i != j) unselection2.insert(*j);
+            
+            ClassMap::const_iterator j = pos; ++j;
+            findBestRewrite(j, end, selection2, unselection2,
+                bestCost, bestSelection);
+        }
+        return;
+    }
+
+    PathSet badPaths;
+    for (PathSet::iterator i = selection.begin();
+         i != selection.end(); ++i)
+    {
+        PathSet closure;
+        computeFSClosure(*i, closure); 
+        for (PathSet::iterator j = closure.begin();
+             j != closure.end(); ++j)
+            if (unselection.find(*j) != unselection.end())
+                badPaths.insert(*i);
+    }
+    
+    //    printMsg(lvlError, format("cost %1% %2%") % badPaths.size() % showPaths(badPaths));
+
+    if (badPaths.size() < bestCost) {
+        bestCost = badPaths.size();
+        bestSelection = selection;
+    }
+}
+
+
+static Path maybeRewrite(const Path & path, const PathSet & selection,
+    const FinalClassMap & finalClassMap, const PathSet & sources,
+    Replacements & replacements,
+    unsigned int & nrRewrites)
+{
+    startNest(nest, lvlError, format("considering rewriting `%1%'") % path);
+
+    assert(selection.find(path) != selection.end());
+
+    if (replacements.find(path) != replacements.end()) return replacements[path];
+    
+    PathSet references;
+    queryReferences(noTxn, path, references);
+
+    HashRewrites rewrites;
+    PathSet newReferences;
+    
+    for (PathSet::iterator i = references.begin(); i != references.end(); ++i) {
+
+        if (*i == path || sources.find(*i) != sources.end()) {
+            newReferences.insert(*i);
+            continue; /* ignore self-references */
+        }
+
+        OutputEqClasses classes;
+        queryOutputEqClasses(noTxn, *i, classes);
+        assert(classes.size() > 0);
+
+        FinalClassMap::const_iterator j = finalClassMap.find(*(classes.begin()));
+        assert(j != finalClassMap.end());
+        
+        Path newPath = maybeRewrite(j->second, selection,
+            finalClassMap, sources, replacements, nrRewrites);
+
+        if (*i != newPath)
+            rewrites[hashPartOf(*i)] = hashPartOf(newPath);
+
+        newReferences.insert(newPath);
+    }
+
+    if (rewrites.size() == 0) {
+        replacements[path] = path;
+        return path;
+    }
+
+    printMsg(lvlError, format("rewriting `%1%'") % path);
+
+    Path newPath = addToStore(path,
+        hashPartOf(path), namePartOf(path),
+        references, rewrites);
+
+    /* !!! we don't know which eqClass `path' is in!  That is to say,
+       we don't know which one is intended here. */
+    OutputEqClasses classes;
+    queryOutputEqClasses(noTxn, path, classes);
+    for (PathSet::iterator i = classes.begin(); i != classes.end(); ++i) {
+        Transaction txn;
+        createStoreTransaction(txn);
+        addOutputEqMember(txn, *i, currentTrustId, newPath);
+        txn.commit();
+    }
+
+    nrRewrites++;
+
+    printMsg(lvlError, format("rewrote `%1%' to `%2%'") % path % newPath);
+
+    replacements[path] = newPath;
+        
+    return newPath;
+}
+
+
+PathSet consolidatePaths(const PathSet & paths, bool checkOnly,
+    Replacements & replacements)
+{
+    printMsg(lvlError, format("consolidating"));
+    
+    ClassMap classMap;
+    PathSet sources;
+    
+    for (PathSet::const_iterator i = paths.begin(); i != paths.end(); ++i) {
+        OutputEqClasses classes;
+        queryOutputEqClasses(noTxn, *i, classes);
+
+        if (classes.size() == 0)
+            sources.insert(*i);
+        else
+            for (OutputEqClasses::iterator j = classes.begin(); j != classes.end(); ++j)
+                classMap[*j].insert(*i);
+    }
+
+    printMsg(lvlError, format("found %1% sources %2%") % sources.size() % showPaths(sources));
+
+    bool conflict = false;
+    for (ClassMap::iterator i = classMap.begin(); i != classMap.end(); ++i)
+        if (i->second.size() >= 2) {
+            printMsg(lvlError, format("conflict in eq class `%1%'") % i->first);
+            conflict = true;
+        }
+
+    if (!conflict) return paths;
+    
+    assert(!checkOnly);
+
+    
+    /* !!! exponential-time algorithm! */
+    const unsigned int infinity = 1000000;
+    unsigned int bestCost = infinity;
+    PathSet bestSelection;
+    findBestRewrite(classMap.begin(), classMap.end(),
+        PathSet(), PathSet(), bestCost, bestSelection);
+
+    assert(bestCost != infinity);
+
+    printMsg(lvlError, format("cheapest selection %1% %2%")
+        % bestCost % showPaths(bestSelection));
+
+    FinalClassMap finalClassMap;
+    for (ClassMap::iterator i = classMap.begin(); i != classMap.end(); ++i)
+        for (PathSet::const_iterator j = i->second.begin(); j != i->second.end(); ++j)
+            if (bestSelection.find(*j) != bestSelection.end())
+                finalClassMap[i->first] = *j;
+
+    PathSet newPaths;
+    unsigned int nrRewrites = 0;
+    replacements.clear();
+    for (PathSet::iterator i = bestSelection.begin();
+         i != bestSelection.end(); ++i)
+        newPaths.insert(maybeRewrite(*i, bestSelection, finalClassMap,
+                            sources, replacements, nrRewrites));
+
+    newPaths.insert(sources.begin(), sources.end());
+
+    assert(nrRewrites == bestCost);
+
+    assert(newPaths.size() < paths.size());
+
+    return newPaths;
 }

src/libstore/misc.hh

@@ -0,0 +1,47 @@
+#ifndef __MISC_H
+#define __MISC_H
+
+#include "derivations.hh"
+#include "store.hh"
+
+
+/* Read a derivation, after ensuring its existence through
+   ensurePath(). */
+Derivation derivationFromPath(const Path & drvPath);
+
+
+/* Place in `paths' the set of all store paths in the file system
+   closure of `storePath'; that is, all paths than can be directly or
+   indirectly reached from it.  `paths' is not cleared.  If
+   `flipDirection' is true, the set of paths that can reach
+   `storePath' is returned; that is, the closures under the `referers'
+   relation instead of the `references' relation is returned. */
+void computeFSClosure(const Path & storePath,
+    PathSet & paths, bool flipDirection = false);
+
+
+/* Return the output equivalence class denoted by `id' in the
+   derivation `drv'. */
+OutputEqClass findOutputEqClass(const Derivation & drv,
+    const string & id);
+
+
+/* Return anll trusted path (wrt to the given trust ID) in the given
+   output path equivalence class, or an empty set if no such paths
+   currently exist. */
+PathSet findTrustedEqClassMembers(const OutputEqClass & eqClass,
+    const TrustId & trustId);
+
+/* Like `findTrustedEqClassMembers', but returns an arbitrary trusted
+   path, or throws an exception if no such path currently exists. */
+Path findTrustedEqClassMember(const OutputEqClass & eqClass,
+    const TrustId & trustId);
+
+
+typedef map<Path, Path> Replacements;
+
+PathSet consolidatePaths(const PathSet & paths, bool checkOnly,
+    Replacements & replacements);
+
+
+#endif /* !__MISC_H */

src/libstore/store.cc

@@ -39,6 +39,21 @@ static TableId dbReferences = 0;
    This table is just the reverse mapping of dbReferences. */
 static TableId dbReferers = 0;
 
+/* dbEquivalences :: OutputEqClass -> [(TrustId, Path)]
+
+   Lists the output paths that have been produced for each extension
+   class; i.e., the extension of an extension class. */
+static TableId dbEquivalences = 0;
+
+/* dbEquivalenceClasses :: Path -> [OutputEqClass]
+
+   !!! should be [(TrustId, OutputEqClass)] ?
+
+   Lists for each output path the extension classes that it is in. */
+static TableId dbEquivalenceClasses = 0;
+
+
+#if 0
 /* dbSubstitutes :: Path -> [[Path]]
 
    Each pair $(p, subs)$ tells Nix that it can use any of the
@@ -61,13 +76,16 @@ static TableId dbSubstitutes = 0;
    only be multiple such paths for fixed-output derivations (i.e.,
    derivations specifying an expected hash). */
 static TableId dbDerivers = 0;
+#endif
 
 
+#if 0
 bool Substitute::operator == (const Substitute & sub) const
 {
     return program == sub.program
         && args == sub.args;
 }
+#endif
 
 
 static void upgradeStore();
@@ -87,8 +105,12 @@ void openDB()
     dbValidPaths = nixDB.openTable("validpaths");
     dbReferences = nixDB.openTable("references");
     dbReferers = nixDB.openTable("referers");
+#if 0    
     dbSubstitutes = nixDB.openTable("substitutes");
     dbDerivers = nixDB.openTable("derivers");
+#endif
+    dbEquivalences = nixDB.openTable("equivalences");
+    dbEquivalenceClasses = nixDB.openTable("equivalence-classes");
 
     int curSchema = 0;
     Path schemaFN = nixDBPath + "/schema";
@@ -121,6 +143,60 @@ void createStoreTransaction(Transaction & txn)
 }
 
 
+
+/* Path hashes. */
+
+const unsigned int pathHashLen = 32; /* characters */
+const string nullPathHashRef(pathHashLen, 0);
+
+
+PathHash::PathHash()
+{
+    rep = nullPathHashRef;
+}
+
+
+PathHash::PathHash(const Hash & h)
+{
+    assert(h.type == htSHA256);
+    rep = printHash32(compressHash(h, 20));
+}
+
+
+PathHash::PathHash(const string & h)
+{
+    /* !!! hacky; check whether this is a valid 160 bit hash */
+    assert(h.size() == pathHashLen);
+    parseHash32(htSHA1, h); 
+    rep = h;
+}
+
+
+string PathHash::toString() const
+{
+    return rep;
+}
+
+
+bool PathHash::isNull() const
+{
+    return rep == nullPathHashRef;
+}
+
+
+bool PathHash::operator ==(const PathHash & hash2) const
+{
+    return rep == hash2.rep;
+}
+
+
+bool PathHash::operator <(const PathHash & hash2) const
+{
+    return rep < hash2.rep;
+}
+
+
+
 /* Path copying. */
 
 struct CopySink : DumpSink
@@ -167,12 +243,14 @@ void copyPath(const Path & src, const Path & dst)
 }
 
 
+
 bool isInStore(const Path & path)
 {
     return path[0] == '/'
         && string(path, 0, nixStore.size()) == nixStore
         && path.size() >= nixStore.size() + 2
-        && path[nixStore.size()] == '/';
+        && path[nixStore.size()] == '/'
+        && path[nixStore.size() + 1 + pathHashLen] == '-';
 }
 
 
@@ -202,6 +280,20 @@ Path toStorePath(const Path & path)
 }
 
 
+PathHash hashPartOf(const Path & path)
+{
+    assertStorePath(path);
+    return PathHash(string(path, nixStore.size() + 1, pathHashLen));
+}
+
+
+string namePartOf(const Path & path)
+{
+    assertStorePath(path);
+    return string(path, nixStore.size() + 1 + pathHashLen + 1);
+}
+
+
 void checkStoreName(const string & name)
 {
     string validChars = "+-._?=";
@@ -275,14 +367,16 @@ bool isValidPath(const Path & path)
 }
 
 
+#if 0
 static Substitutes readSubstitutes(const Transaction & txn,
     const Path & srcPath);
+#endif
 
 
 static bool isRealisablePath(const Transaction & txn, const Path & path)
 {
     return isValidPathTxn(txn, path)
-        || readSubstitutes(txn, path).size() > 0;
+        /* !!! || readSubstitutes(txn, path).size() > 0 */;
 }
 
 
@@ -356,6 +450,73 @@ void queryReferers(const Transaction & txn,
 }
 
 
+void addOutputEqMember(const Transaction & txn,
+    const OutputEqClass & eqClass, const TrustId & trustId,
+    const Path & path)
+{
+    OutputEqMembers members;
+    queryOutputEqMembers(txn, eqClass, members);
+
+    for (OutputEqMembers::iterator i = members.begin();
+         i != members.end(); ++i)
+        if (i->trustId == trustId && i->path == path) return;
+    
+    OutputEqMember member;
+    member.trustId = trustId;
+    member.path = path;
+    members.push_back(member);
+
+    Strings ss;
+    
+    for (OutputEqMembers::iterator i = members.begin();
+         i != members.end(); ++i)
+    {
+        Strings ss2;
+        ss2.push_back(i->trustId);
+        ss2.push_back(i->path);
+        ss.push_back(packStrings(ss2));
+    }
+
+    nixDB.setStrings(txn, dbEquivalences, eqClass, ss);
+
+    OutputEqClasses classes;
+    queryOutputEqClasses(txn, path, classes);
+
+    classes.insert(eqClass);
+
+    nixDB.setStrings(txn, dbEquivalenceClasses, path,
+        Strings(classes.begin(), classes.end()));
+}
+
+
+void queryOutputEqMembers(const Transaction & txn,
+    const OutputEqClass & eqClass, OutputEqMembers & members)
+{
+    Strings ss;
+    nixDB.queryStrings(txn, dbEquivalences, eqClass, ss);
+
+    for (Strings::iterator i = ss.begin(); i != ss.end(); ++i) {
+        Strings ss2 = unpackStrings(*i);
+        if (ss2.size() != 2) continue;
+        Strings::iterator j = ss2.begin();
+        OutputEqMember member;
+        member.trustId = *j++;
+        member.path = *j++;
+        members.push_back(member);
+    }
+}
+
+
+void queryOutputEqClasses(const Transaction & txn,
+    const Path & path, OutputEqClasses & classes)
+{
+    Strings ss;
+    nixDB.queryStrings(txn, dbEquivalenceClasses, path, ss);
+    classes.insert(ss.begin(), ss.end());
+}
+
+
+#if 0
 void setDeriver(const Transaction & txn, const Path & storePath,
     const Path & deriver)
 {
@@ -378,8 +539,10 @@ Path queryDeriver(const Transaction & txn, const Path & storePath)
     else
         return "";
 }
+#endif
 
 
+#if 0
 const int substituteVersion = 2;
 
 
@@ -488,6 +651,7 @@ void clearSubstitutes()
     
     txn.commit();
 }
+#endif
 
 
 static void setHash(const Transaction & txn, const Path & storePath,
@@ -563,7 +727,9 @@ void registerValidPaths(const Transaction & txn,
                 throw Error(format("cannot register path `%1%' as valid, since its reference `%2%' is invalid")
                     % i->path % *j);
 
+#if 0
         setDeriver(txn, i->path, i->deriver);
+#endif
     }
 }
 
@@ -578,30 +744,39 @@ static void invalidatePath(Transaction & txn, const Path & path)
        inverse `referers' entries, and the `derivers' entry; but only
        if there are no substitutes for this path.  This maintains the
        cleanup invariant. */
-    if (querySubstitutes(txn, path).size() == 0) {
+    if (1 /*querySubstitutes(txn, path).size() == 0 !!! */) {
         setReferences(txn, path, PathSet());
-        nixDB.delPair(txn, dbDerivers, path);
+        // !!!        nixDB.delPair(txn, dbDerivers, path);
     }
     
     nixDB.delPair(txn, dbValidPaths, path);
 }
 
 
-Path makeStorePath(const string & type,
-    const Hash & hash, const string & suffix)
+void makeStorePath(const Hash & contentHash, const string & suffix,
+    Path & path, PathHash & pathHash)
 {
-    /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
-    string s = type + ":sha256:" + printHash(hash) + ":"
-        + nixStore + ":" + suffix;
-
     checkStoreName(suffix);
 
-    return nixStore + "/"
-        + printHash32(compressHash(hashString(htSHA256, s), 20))
-        + "-" + suffix;
+    /* e.g., "sha256:1abc...:foo.tar.gz" */
+    string s = "sha256:" + printHash(contentHash) + ":" + suffix;
+
+    pathHash = PathHash(hashString(htSHA256, s));
+    
+    path = nixStore + "/" + pathHash.toString() + "-" + suffix;
 }
 
 
+Path makeRandomStorePath(const string & suffix)
+{
+    Hash hash(htSHA256);
+    for (unsigned int i = 0; i < hash.hashSize; ++i)
+        hash.hash[i] = rand() % 256; // !!! improve
+    return nixStore + "/" + PathHash(hash).toString() + "-" + suffix;
+}
+
+
+#if 0
 Path makeFixedOutputPath(bool recursive,
     string hashAlgo, Hash hash, string name)
 {
@@ -612,37 +787,99 @@ Path makeFixedOutputPath(bool recursive,
         + "");
     return makeStorePath("output:out", h, name);
 }
+#endif
 
 
-static Path _addToStore(bool fixed, bool recursive,
-    string hashAlgo, const Path & _srcPath)
+typedef map<PathHash, PathHash> HashRewrites;
+
+string rewriteHashes(string s, const HashRewrites & rewrites,
+    vector<int> & positions)
 {
-    Path srcPath(absPath(_srcPath));
-    debug(format("adding `%1%' to the store") % srcPath);
-
-    Hash h(htSHA256);
+    for (HashRewrites::const_iterator i = rewrites.begin();
+         i != rewrites.end(); ++i)
     {
-        SwitchToOriginalUser sw;
-        h = hashPath(htSHA256, srcPath);
-    }
+        string from = i->first.toString(), to = i->second.toString();
 
-    string baseName = baseNameOf(srcPath);
+        assert(from.size() == to.size());
 
-    Path dstPath;
-    
-    if (fixed) {
-
-        HashType ht(parseHashType(hashAlgo));
-        Hash h2(ht);
-        {
-            SwitchToOriginalUser sw;
-            h2 = recursive ? hashPath(ht, srcPath) : hashFile(ht, srcPath);
+        unsigned int j = 0;
+        while ((j = s.find(from, j)) != string::npos) {
+            debug(format("rewriting @ %1%") % j);
+            positions.push_back(j);
+            s.replace(j, to.size(), to);
+            j += to.size();
         }
-        
-        dstPath = makeFixedOutputPath(recursive, hashAlgo, h2, baseName);
     }
-        
-    else dstPath = makeStorePath("source", h, baseName);
+
+    return s;
+}
+
+
+string rewriteHashes(const string & s, const HashRewrites & rewrites)
+{
+    vector<int> dummy;
+    return rewriteHashes(s, rewrites, dummy);
+}
+
+
+static Hash hashModulo(string s, const PathHash & modulus)
+{
+    vector<int> positions;
+    
+    if (!modulus.isNull()) {
+        /* Zero out occurences of `modulus'. */
+        HashRewrites rewrites;
+        rewrites[modulus] = PathHash(); /* = null hash */
+        s = rewriteHashes(s, rewrites, positions);
+    }
+
+    string positionPrefix;
+    
+    for (vector<int>::iterator i = positions.begin();
+         i != positions.end(); ++i)
+        positionPrefix += (format("|%1%") % *i).str();
+
+    positionPrefix += "||";
+
+    debug(format("positions %1%") % positionPrefix);
+    
+    return hashString(htSHA256, positionPrefix + s);
+}
+
+
+static PathSet rewriteReferences(const PathSet & references,
+    const HashRewrites & rewrites)
+{
+    PathSet result;
+    for (PathSet::const_iterator i = references.begin(); i != references.end(); ++i)
+        result.insert(rewriteHashes(*i, rewrites));
+    return result;
+}
+
+
+static Path _addToStore(const string & suffix, string dump,
+    const PathHash & selfHash, const PathSet & references)
+{
+    /* Hash the contents, modulo the previous hash reference (if it
+       had one). */
+    Hash contentHash = hashModulo(dump, selfHash);
+
+    /* Construct the new store path. */ 
+    Path dstPath;
+    PathHash pathHash;
+    makeStorePath(contentHash, suffix, dstPath, pathHash);
+
+    /* If the contents had a previous hash reference, rewrite those
+       references to the new hash. */
+    HashRewrites rewrites;
+    if (!selfHash.isNull()) {
+        rewrites[selfHash] = pathHash;
+        vector<int> positions;
+        dump = rewriteHashes(dump, rewrites, positions);
+        /* !!! debug code, remove */
+        PathHash contentHash2 = hashModulo(dump, pathHash);
+        assert(contentHash2 == contentHash);
+    }
 
     if (!readOnlyMode) addTempRoot(dstPath);
 
@@ -659,17 +896,18 @@ static Path _addToStore(bool fixed, bool recursive,
 
             if (pathExists(dstPath)) deletePath(dstPath);
 
-            copyPath(srcPath, dstPath);
-
-            Hash h2 = hashPath(htSHA256, dstPath);
-            if (h != h2)
-                throw Error(format("contents of `%1%' changed while copying it to `%2%' (%3% -> %4%)")
-                    % srcPath % dstPath % printHash(h) % printHash(h2));
+            CopySource source(dump);
+            restorePath(dstPath, source);
 
             canonicalisePathMetaData(dstPath);
+
+            /* Set the references for the new path.  Of course, any
+               hash rewrites have to be applied to the references,
+               too. */
+            PathSet references2 = rewriteReferences(references, rewrites);
             
             Transaction txn(nixDB);
-            registerValidPath(txn, dstPath, h, PathSet(), "");
+            registerValidPath(txn, dstPath, contentHash, references2, "");
             txn.commit();
         }
 
@@ -680,51 +918,41 @@ static Path _addToStore(bool fixed, bool recursive,
 }
 
 
-Path addToStore(const Path & srcPath)
+Path addToStore(const Path & _srcPath, const PathHash & selfHash,
+    const string & suffix, const PathSet & references, const HashRewrites & rewrites)
 {
-    return _addToStore(false, false, "", srcPath);
+    Path srcPath(absPath(_srcPath));
+    debug(format("adding `%1%' to the store") % srcPath);
+
+    CopySink sink;
+    {
+        SwitchToOriginalUser sw;
+        dumpPath(srcPath, sink);
+    }
+
+    if (rewrites.size() != 0) sink.s = rewriteHashes(sink.s, rewrites);
+
+    return _addToStore(suffix == "" ? baseNameOf(srcPath) : suffix,
+        sink.s, selfHash,
+        rewriteReferences(references, rewrites));
 }
 
 
+#if 0
 Path addToStoreFixed(bool recursive, string hashAlgo, const Path & srcPath)
 {
     return _addToStore(true, recursive, hashAlgo, srcPath);
 }
+#endif
 
 
 Path addTextToStore(const string & suffix, const string & s,
     const PathSet & references)
 {
-    Hash hash = hashString(htSHA256, s);
-
-    Path dstPath = makeStorePath("text", hash, suffix);
+    CopySink sink;
+    makeSingletonArchive(s, sink);
     
-    if (!readOnlyMode) addTempRoot(dstPath);
-
-    if (!readOnlyMode && !isValidPath(dstPath)) {
-
-        PathSet lockPaths;
-        lockPaths.insert(dstPath);
-        PathLocks outputLock(lockPaths);
-
-        if (!isValidPath(dstPath)) {
-
-            if (pathExists(dstPath)) deletePath(dstPath);
-
-            writeStringToFile(dstPath, s);
-
-            canonicalisePathMetaData(dstPath);
-            
-            Transaction txn(nixDB);
-            registerValidPath(txn, dstPath,
-                hashPath(htSHA256, dstPath), references, "");
-            txn.commit();
-        }
-
-        outputLock.setDeletion(true);
-    }
-
-    return dstPath;
+    return _addToStore(suffix, sink.s, PathHash(), references);
 }
 
 
@@ -751,6 +979,7 @@ void deleteFromStore(const Path & _path)
 
 void verifyStore(bool checkContents)
 {
+#if 0    
     Transaction txn(nixDB);
 
     Paths paths;
@@ -891,6 +1120,7 @@ void verifyStore(bool checkContents)
     }
 
     txn.commit();
+#endif    
 }
 
 
@@ -902,6 +1132,7 @@ void verifyStore(bool checkContents)
 static void upgradeStore()
 {
     printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)...");
+#if 0    
 
     Transaction txn(nixDB);
 
@@ -982,4 +1213,5 @@ static void upgradeStore()
 
     /* !!! maybe this transaction is way too big */
     txn.commit();
+#endif    
 }

src/libstore/store.hh

@@ -12,6 +12,27 @@ using namespace std;
 const int nixSchemaVersion = 2;
 
 
+/* Path hashes are the hash components of store paths, e.g., the
+   `zvhgns772jpj68l40mq1jb74wpfsf0ma' in
+   `/nix/store/zvhgns772jpj68l40mq1jb74wpfsf0ma-glibc'.  These are
+   truncated SHA-256 hashes of the path contents,  */
+struct PathHash
+{
+private:
+    string rep;
+public:
+    PathHash();
+    PathHash(const Hash & h);
+    PathHash(const string & h);
+    string toString() const;
+    bool PathHash::isNull() const;
+    bool operator ==(const PathHash & hash2) const;
+    bool operator <(const PathHash & hash2) const;
+};
+
+
+
+#if 0
 /* A substitute is a program invocation that constructs some store
    path (typically by fetching it from somewhere, e.g., from the
    network). */
@@ -32,6 +53,39 @@ struct Substitute
 };
 
 typedef list<Substitute> Substitutes;
+#endif
+
+
+/* A trust identifier, which is a name of an entity involved in a
+   trust relation.  Right now this is just a user ID (e.g.,
+   `root'). */
+typedef string TrustId;
+
+
+/* An output path equivalence class.  They represent outputs of
+   derivations.  That is, a derivation can have several outputs (e.g.,
+   `out', `lib', `man', etc.), each of which maps to a output path
+   equivalence class.  They can map to a number of concrete paths,
+   depending on what users built the derivation.
+
+   Equivalence classes are actually "placeholder" store paths that
+   never get built.  They do occur in derivations however in
+   command-line arguments and environment variables, but get
+   substituted with concrete paths when we actually build. */
+typedef Path OutputEqClass;
+
+typedef set<OutputEqClass> OutputEqClasses;
+
+
+/* A member of an output path equivalence class, i.e., a store path
+   that has been produced by a certain derivation. */
+struct OutputEqMember
+{
+    TrustId trustId;
+    Path path;
+};
+
+typedef list<OutputEqMember> OutputEqMembers;
 
 
 /* Open the database environment. */
@@ -43,9 +97,12 @@ void initDB();
 /* Get a transaction object. */
 void createStoreTransaction(Transaction & txn);
 
+
 /* Copy a path recursively. */
 void copyPath(const Path & src, const Path & dst);
 
+
+#if 0
 /* Register a substitute. */
 void registerSubstitute(const Transaction & txn,
     const Path & srcPath, const Substitute & sub);
@@ -55,6 +112,8 @@ Substitutes querySubstitutes(const Transaction & txn, const Path & srcPath);
 
 /* Deregister all substitutes. */
 void clearSubstitutes();
+#endif
+
 
 /* Register the validity of a path, i.e., that `path' exists, that the
    paths referenced by it exists, and in the case of an output path of
@@ -79,6 +138,7 @@ typedef list<ValidPathInfo> ValidPathInfos;
 void registerValidPaths(const Transaction & txn,
     const ValidPathInfos & infos);
 
+
 /* Throw an exception if `path' is not directly in the Nix store. */
 void assertStorePath(const Path & path);
 
@@ -91,6 +151,11 @@ void checkStoreName(const string & name);
    /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
 Path toStorePath(const Path & path);
 
+PathHash hashPartOf(const Path & path);
+
+string namePartOf(const Path & path);
+
+
 /* "Fix", or canonicalise, the meta-data of the files in a store path
    after it has been built.  In particular:
    - the last modification date on each file is set to 0 (i.e.,
@@ -105,6 +170,7 @@ void canonicalisePathMetaData(const Path & path);
 bool isValidPathTxn(const Transaction & txn, const Path & path);
 bool isValidPath(const Path & path);
 
+
 /* Queries the hash of a valid path. */ 
 Hash queryPathHash(const Path & path);
 
@@ -123,6 +189,17 @@ void queryReferences(const Transaction & txn,
 void queryReferers(const Transaction & txn,
     const Path & storePath, PathSet & referers);
 
+void addOutputEqMember(const Transaction & txn,
+    const OutputEqClass & eqClass, const TrustId & trustId,
+    const Path & path);
+
+void queryOutputEqMembers(const Transaction & txn,
+    const OutputEqClass & eqClass, OutputEqMembers & members);
+
+void queryOutputEqClasses(const Transaction & txn,
+    const Path & path, OutputEqClasses & classes);
+
+#if 0
 /* Sets the deriver of a store path.  Use with care! */
 void setDeriver(const Transaction & txn, const Path & storePath,
     const Path & deriver);
@@ -130,27 +207,48 @@ void setDeriver(const Transaction & txn, const Path & storePath,
 /* Query the deriver of a store path.  Return the empty string if no
    deriver has been set. */
 Path queryDeriver(const Transaction & txn, const Path & storePath);
+#endif
+
 
 /* Constructs a unique store path name. */
-Path makeStorePath(const string & type,
-    const Hash & hash, const string & suffix);
-    
+void makeStorePath(const Hash & contentHash, const string & suffix,
+    Path & path, PathHash & pathHash);
+
+/* Constructs a random store path name.  Only to be used for temporary
+   build outputs, since these will violate the hash invariant. */
+Path makeRandomStorePath(const string & suffix);
+
+
+/* Hash rewriting. */
+typedef map<PathHash, PathHash> HashRewrites;
+
+string rewriteHashes(string s, const HashRewrites & rewrites,
+    vector<int> & positions);
+
+string rewriteHashes(const string & s, const HashRewrites & rewrites);
+
+
 /* Copy the contents of a path to the store and register the validity
    the resulting path.  The resulting path is returned. */
-Path addToStore(const Path & srcPath);
+Path addToStore(const Path & srcPath, const PathHash & selfHash = PathHash(),
+    const string & suffix = "", const PathSet & references = PathSet(),
+    const HashRewrites & rewrites = HashRewrites());
 
+#if 0
 /* Like addToStore(), but for pre-adding the outputs of fixed-output
    derivations. */
 Path addToStoreFixed(bool recursive, string hashAlgo, const Path & srcPath);
 
 Path makeFixedOutputPath(bool recursive,
     string hashAlgo, Hash hash, string name);
+#endif
 
 /* Like addToStore, but the contents written to the output path is a
    regular file containing the given string. */
 Path addTextToStore(const string & suffix, const string & s,
     const PathSet & references);
 
+
 /* Delete a value from the nixStore directory. */
 void deleteFromStore(const Path & path);
 

src/libstore/test-builder-1.sh

@@ -1,3 +0,0 @@
-#! /bin/sh
-
-echo "Hello World" > $out

src/libutil/archive.cc

@@ -140,6 +140,26 @@ void dumpPath(const Path & path, DumpSink & sink)
 }
 
 
+void makeSingletonArchive(const string & contents, DumpSink & sink)
+{
+    /* !!! hacky; have to keep this synchronised with dumpPath().  It
+       would be better to parameterise dumpPath() with a file system
+       "traverser". */
+    writeString(archiveVersion1, sink);
+    writeString("(", sink);
+    writeString("type", sink);
+    writeString("regular", sink);
+
+    unsigned int size = contents.size();
+    writeString("contents", sink);
+    writeInt(size, sink);
+    sink((const unsigned char *) contents.c_str(), size);
+    writePadding(size, sink);
+
+    writeString(")", sink);
+}
+
+
 static Error badArchive(string s)
 {
     return Error("bad archive: " + s);

src/libutil/archive.hh

@@ -48,6 +48,10 @@ struct DumpSink
 
 void dumpPath(const Path & path, DumpSink & sink);
 
+/* Make an archive consisting of a single non-executable regular
+   file, with specified string contents. */
+void makeSingletonArchive(const string & contents, DumpSink & sink);
+
 
 struct RestoreSource
 {

src/libutil/util.cc

@@ -357,6 +357,19 @@ void printMsg_(Verbosity level, const format & f)
 }
 
 
+string showPaths(const PathSet & paths)
+{
+    string s;
+    for (PathSet::const_iterator i = paths.begin();
+         i != paths.end(); ++i)
+    {
+        if (s.size() != 0) s += ", ";
+        s += "`" + *i + "'";
+    }
+    return s;
+}
+
+
 void readFull(int fd, unsigned char * buf, size_t count)
 {
     while (count) {

src/libutil/util.hh

@@ -173,6 +173,8 @@ void printMsg_(Verbosity level, const format & f);
 
 #define debug(f) printMsg(lvlDebug, f)
 
+string showPaths(const PathSet & paths);
+
 
 /* Wrappers arount read()/write() that read/write exactly the
    requested number of bytes. */

src/nix-env/main.cc

@@ -3,6 +3,7 @@
 #include "globals.hh"
 #include "build.hh"
 #include "gc.hh"
+#include "misc.hh"
 #include "shared.hh"
 #include "parser.hh"
 #include "eval.hh"
@@ -383,7 +384,6 @@ static void queryInstSources(EvalState & state,
            (import ./foo.nix)' = `(import ./foo.nix).bar'. */
         case srcNixExprs: {
                 
-
             Expr e1 = parseExprFromFile(state,
                 absPath(instSource.nixExprPath));
 
@@ -416,7 +416,10 @@ static void queryInstSources(EvalState & state,
 
                 if (isDerivation(*i)) {
                     elem.setDrvPath(*i);
-                    elem.setOutPath(findOutput(derivationFromPath(*i), "out"));
+                    elem.setOutPath(
+                        /* XXX check this; may not give a result */
+                        findTrustedEqClassMember(
+                            findOutputEqClass(derivationFromPath(*i), "out"), currentTrustId));
                     if (name.size() >= drvExtension.size() &&
                         string(name, name.size() - drvExtension.size()) == drvExtension)
                         name = string(name, 0, name.size() - drvExtension.size());
@@ -735,12 +738,14 @@ static void opQuery(Globals & globals,
         Strings columns;
         
         if (printStatus) {
+#if 0            
             Substitutes subs = querySubstitutes(noTxn, i->queryDrvPath(globals.state));
+#endif            
             columns.push_back(
                 (string) (installed.find(i->queryOutPath(globals.state))
                     != installed.end() ? "I" : "-")
                 + (isValidPath(i->queryOutPath(globals.state)) ? "P" : "-")
-                + (subs.size() > 0 ? "S" : "-"));
+                + (/* XXX subs.size() > 0 */ false ? "S" : "-"));
         }
 
         if (printName) columns.push_back(i->name);

src/nix-store/main.cc

@@ -4,6 +4,7 @@
 #include "globals.hh"
 #include "build.hh"
 #include "gc.hh"
+#include "misc.hh"
 #include "archive.hh"
 #include "shared.hh"
 #include "dotgraph.hh"
@@ -45,7 +46,8 @@ static Path realisePath(const Path & path)
         PathSet paths;
         paths.insert(path);
         buildDerivations(paths);
-        Path outPath = findOutput(derivationFromPath(path), "out");
+        Path outPath = findTrustedEqClassMember(
+            findOutputEqClass(derivationFromPath(path), "out"), currentTrustId);
         
         if (gcRoot == "")
             printGCWarning();
@@ -96,6 +98,7 @@ static void opAdd(Strings opFlags, Strings opArgs)
 }
 
 
+#if 0
 /* Preload the output of a fixed-output derivation into the Nix
    store. */
 static void opAddFixed(Strings opFlags, Strings opArgs)
@@ -141,6 +144,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
     cout << format("%1%\n") %
         makeFixedOutputPath(recursive, hashAlgo, h, name);
 }
+#endif
 
 
 /* Place in `paths' the set of paths that are required to `realise'
@@ -165,6 +169,8 @@ static void storePathRequisites(const Path & storePath,
     computeFSClosure(storePath, paths);
 
     if (includeOutputs) {
+        assert(0);
+#if 0        
         for (PathSet::iterator i = paths.begin();
              i != paths.end(); ++i)
             if (isDerivation(*i)) {
@@ -174,6 +180,7 @@ static void storePathRequisites(const Path & storePath,
                     if (isValidPath(j->second.path))
                         computeFSClosure(j->second.path, paths);
             }
+#endif        
     }
 }
 
@@ -183,7 +190,8 @@ static Path maybeUseOutput(const Path & storePath, bool useOutput, bool forceRea
     if (forceRealise) realisePath(storePath);
     if (useOutput && isDerivation(storePath)) {
         Derivation drv = derivationFromPath(storePath);
-        return findOutput(drv, "out");
+        return findTrustedEqClassMember(
+            findOutputEqClass(drv, "out"), currentTrustId);
     }
     else return storePath;
 }
@@ -313,7 +321,9 @@ static void opQuery(Strings opFlags, Strings opArgs)
                 *i = fixPath(*i);
                 if (forceRealise) realisePath(*i);
                 Derivation drv = derivationFromPath(*i);
-                cout << format("%1%\n") % findOutput(drv, "out");
+                cout << format("%1%\n") % findTrustedEqClassMember(
+                    findOutputEqClass(drv, "out"),
+                    currentTrustId);
             }
             break;
         }
@@ -338,6 +348,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
         }
 
         case qDeriver:
+#if 0            
             for (Strings::iterator i = opArgs.begin();
                  i != opArgs.end(); ++i)
             {
@@ -345,6 +356,8 @@ static void opQuery(Strings opFlags, Strings opArgs)
                 cout << format("%1%\n") %
                     (deriver == "" ? "unknown-deriver" : deriver);
             }
+#endif
+            assert(0);
             break;
 
         case qBinding:
@@ -395,6 +408,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
 }
 
 
+#if 0
 static void opRegisterSubstitutes(Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
@@ -441,6 +455,7 @@ static void opClearSubstitutes(Strings opFlags, Strings opArgs)
 
     clearSubstitutes();
 }
+#endif
 
 
 static void opRegisterValidity(Strings opFlags, Strings opArgs)
@@ -602,16 +617,20 @@ void run(Strings args)
             op = opRealise;
         else if (arg == "--add" || arg == "-A")
             op = opAdd;
+#if 0        
         else if (arg == "--add-fixed")
             op = opAddFixed;
         else if (arg == "--print-fixed-path")
             op = opPrintFixedPath;
+#endif
         else if (arg == "--query" || arg == "-q")
             op = opQuery;
+#if 0        
         else if (arg == "--register-substitutes")
             op = opRegisterSubstitutes;
         else if (arg == "--clear-substitutes")
             op = opClearSubstitutes;
+#endif            
         else if (arg == "--register-validity")
             op = opRegisterValidity;
         else if (arg == "--check-validity")