Make the search path lazier with non-fatal errors

Thus, -I / $NIX_PATH entries are now downloaded only when they are
needed for evaluation. An error to download an entry is a non-fatal
warning (just like non-existant paths).

This does change the semantics of builtins.nixPath, which now returns
the original, rather than resulting path. E.g., before we had

  [ { path = "/nix/store/hgm3yxf1lrrwa3z14zpqaj5p9vs0qklk-nixexprs.tar.xz"; prefix = "nixpkgs"; } ... ]

but now

  [ { path = "https://nixos.org/channels/nixos-16.03/nixexprs.tar.xz"; prefix = "nixpkgs"; } ... ]

Fixes #792.

(cherry picked from commit 363f37d084)

src/libexpr/eval.cc

@@ -273,7 +273,7 @@ EvalState::EvalState(const Strings & _searchPath)
 
     /* Initialise the Nix expression search path. */
     Strings paths = parseNixPath(getEnv("NIX_PATH", ""));
-    for (auto & i : _searchPath) addToSearchPath(i, true);
+    for (auto & i : _searchPath) addToSearchPath(i);
     for (auto & i : paths) addToSearchPath(i);
     addToSearchPath("nix=" + settings.nixDataDir + "/nix/corepkgs");
 
@@ -296,11 +296,15 @@ Path EvalState::checkSourcePath(const Path & path_)
     if (!restricted) return path_;
 
     /* Resolve symlinks. */
+    debug(format("checking access to ‘%s’") % path_);
     Path path = canonPath(path_, true);
 
-    for (auto & i : searchPath)
+    for (auto & i : searchPath) {
-        if (path == i.second || isInDir(path, i.second))
+        auto r = resolveSearchPathElem(i);
+        if (!r.first) continue;
+        if (path == r.second || isInDir(path, r.second))
             return path;
+    }
 
     /* To support import-from-derivation, allow access to anything in
        the store. FIXME: only allow access to paths that have been

src/libexpr/eval.hh

@@ -55,7 +55,8 @@ typedef std::map<Path, Path> SrcToStore;
 std::ostream & operator << (std::ostream & str, const Value & v);
 
 
-typedef list<std::pair<string, Path> > SearchPath;
+typedef std::pair<std::string, std::string> SearchPathElem;
+typedef std::list<SearchPathElem> SearchPath;
 
 
 /* Initialise the Boehm GC, if applicable. */
@@ -95,12 +96,14 @@ private:
 
     SearchPath searchPath;
 
+    std::map<std::string, std::pair<bool, std::string>> searchPathResolved;
+
 public:
 
     EvalState(const Strings & _searchPath);
     ~EvalState();
 
-    void addToSearchPath(const string & s, bool warn = false);
+    void addToSearchPath(const string & s);
 
     Path checkSourcePath(const Path & path);
 
@@ -122,6 +125,9 @@ public:
     Path findFile(const string & path);
     Path findFile(SearchPath & searchPath, const string & path, const Pos & pos = noPos);
 
+    /* If the specified search path element is a URI, download it. */
+    std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem);
+
     /* Evaluate an expression to normal form, storing the result in
        value `v'. */
     void eval(Expr * e, Value & v);

src/libexpr/parser.y

@@ -590,7 +590,7 @@ Expr * EvalState::parseExprFromString(const string & s, const Path & basePath)
 }
 
 
-void EvalState::addToSearchPath(const string & s, bool warn)
+void EvalState::addToSearchPath(const string & s)
 {
     size_t pos = s.find('=');
     string prefix;
@@ -602,16 +602,7 @@ void EvalState::addToSearchPath(const string & s, bool warn)
         path = string(s, pos + 1);
     }
 
-    if (isUri(path))
+    searchPath.emplace_back(prefix, path);
-        path = downloadFileCached(path, true);
-
-    path = absPath(path);
-    if (pathExists(path)) {
-        debug(format("adding path ‘%1%’ to the search path") % path);
-        /* Resolve symlinks in the path to support restricted mode. */
-        searchPath.push_back(std::pair<string, Path>(prefix, canonPath(path, true)));
-    } else if (warn)
-        printMsg(lvlError, format("warning: Nix search path entry ‘%1%’ does not exist, ignoring") % path);
 }
 
 
@@ -624,17 +615,19 @@ Path EvalState::findFile(const string & path)
 Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos & pos)
 {
     for (auto & i : searchPath) {
-        assert(!isUri(i.second));
+        std::string suffix;
-        Path res;
         if (i.first.empty())
-            res = i.second + "/" + path;
+            suffix = "/" + path;
         else {
-            if (path.compare(0, i.first.size(), i.first) != 0 ||
+            auto s = i.first.size();
-                (path.size() > i.first.size() && path[i.first.size()] != '/'))
+            if (path.compare(0, s, i.first) != 0 ||
+                (path.size() > s && path[s] != '/'))
                 continue;
-            res = i.second +
+            suffix = path.size() == s ? "" : "/" + string(path, s);
-                (path.size() == i.first.size() ? "" : "/" + string(path, i.first.size()));
         }
+        auto r = resolveSearchPathElem(i);
+        if (!r.first) continue;
+        Path res = r.second + suffix;
         if (pathExists(res)) return canonPath(res);
     }
     format f = format(
@@ -645,4 +638,35 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
 }
 
 
+std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathElem & elem)
+{
+    auto i = searchPathResolved.find(elem.second);
+    if (i != searchPathResolved.end()) return i->second;
+
+    std::pair<bool, std::string> res;
+
+    if (isUri(elem.second)) {
+        try {
+            res = { true, downloadFileCached(elem.second, true) };
+        } catch (DownloadError & e) {
+            printMsg(lvlError, format("warning: Nix search path entry ‘%1%’ cannot be downloaded, ignoring") % elem.second);
+            res = { false, "" };
+        }
+    } else {
+        auto path = absPath(elem.second);
+        if (pathExists(path))
+            res = { true, path };
+        else {
+            printMsg(lvlError, format("warning: Nix search path entry ‘%1%’ does not exist, ignoring") % elem.second);
+            res = { false, "" };
+        }
+    }
+
+    debug(format("resolved search path element ‘%s’ to ‘%s’") % elem.second % res.second);
+
+    searchPathResolved[elem.second] = res;
+    return res;
+}
+
+
 }

src/libexpr/primops.cc

@@ -765,7 +765,6 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
 
     SearchPath searchPath;
 
-    PathSet context;
     for (unsigned int n = 0; n < args[0]->listSize(); ++n) {
         Value & v2(*args[0]->listElems()[n]);
         state.forceAttrs(v2, pos);
@@ -778,21 +777,23 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
         i = v2.attrs->find(state.symbols.create("path"));
         if (i == v2.attrs->end())
             throw EvalError(format("attribute ‘path’ missing, at %1%") % pos);
-        string path = state.coerceToPath(pos, *i->value, context);
 
-        searchPath.push_back(std::pair<string, Path>(prefix, state.checkSourcePath(path)));
+        PathSet context;
+        string path = state.coerceToString(pos, *i->value, context, false, false);
+
+        try {
+            realiseContext(context);
+        } catch (InvalidPathError & e) {
+            throw EvalError(format("cannot find ‘%1%’, since path ‘%2%’ is not valid, at %3%")
+                % path % e.path % pos);
+        }
+
+        searchPath.emplace_back(prefix, path);
     }
 
     string path = state.forceStringNoCtx(*args[1], pos);
 
-    try {
+    mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str());
-        realiseContext(context);
-    } catch (InvalidPathError & e) {
-        throw EvalError(format("cannot find ‘%1%’, since path ‘%2%’ is not valid, at %3%")
-            % path % e.path % pos);
-    }
-
-    mkPath(v, state.findFile(searchPath, path, pos).c_str());
 }
 
 /* Read a directory (without . or ..) */

tests/tarball.sh

@@ -20,4 +20,10 @@ nix-build file://$tarball
 
 nix-build '<foo>' -I foo=file://$tarball
 
-nix-build -E "import (fetchTarball file://$tarball)"
+nix-build -o $TMPDIR/result -E "import (fetchTarball file://$tarball)"
+
+nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar.xz
+nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball.tar.xz
+(! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball.tar.xz)
+
+nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file://no-such-tarball.tar.xz -I fnord=.