From 69d82e5c58bf6d7e16fc296f598c352da2a618d0 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Thu, 25 Jan 2018 07:05:57 -0800 Subject: [PATCH] Add path primop. builtins.path allows specifying the name of a path (which makes paths with store-illegal names now addable), allows adding paths with flat instead of recursive hashes, allows specifying a filter (so is a generalization of filterSource), and allows specifying an expected hash (enabling safe path adding in pure mode). --- doc/manual/expressions/builtins.xml | 74 ++++++++++++++++++++++- src/libexpr/eval.cc | 2 +- src/libexpr/primops.cc | 93 +++++++++++++++++++++++------ src/libstore/store-api.cc | 5 +- src/libstore/store-api.hh | 6 +- tests/lang/data | 1 + tests/lang/eval-okay-path.exp | 1 + tests/lang/eval-okay-path.nix | 7 +++ 8 files changed, 162 insertions(+), 27 deletions(-) create mode 100644 tests/lang/data create mode 100644 tests/lang/eval-okay-path.exp create mode 100644 tests/lang/eval-okay-path.nix diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 5a3a8645..81770bcf 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -308,8 +308,9 @@ stdenv.mkDerivation { … } - builtins.filterSource - e1 e2 + + builtins.filterSource + e1 e2 @@ -768,6 +769,75 @@ Evaluates to [ "foo" ]. + + + builtins.path + args + + + + + An enrichment of the built-in path type, based on the attributes + present in args. All are optional + except path: + + + + + path + + The underlying path. + + + + name + + + The name of the path when added to the store. This can + used to reference paths that have nix-illegal characters + in their names, like @. + + + + + filter + + + A function of the type expected by + builtins.filterSource, + with the same semantics. + + + + + recursive + + + When false, when + path is added to the store it is with a + flat hash, rather than a hash of the NAR serialization of + the file. Thus, path must refer to a + regular file, not a directory. This allows similar + behavior to fetchurl. Defaults to + true. + + + + + sha256 + + + When provided, this is the expected hash of the file at + the path. Evaluation will fail if the hash is incorrect, + and providing a hash allows + builtins.path to be used even when the + pure-eval nix config option is on. + + + + + + builtins.pathExists path diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 33a9bc61..9499ebe7 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1566,7 +1566,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path) dstPath = srcToStore[path]; else { dstPath = settings.readOnlyMode - ? store->computeStorePathForPath(checkSourcePath(path)).first + ? store->computeStorePathForPath(baseNameOf(path), checkSourcePath(path)).first : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair); srcToStore[path] = dstPath; printMsg(lvlChatty, format("copied source '%1%' -> '%2%'") diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 975f0e83..5c8dfd9d 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1009,20 +1009,13 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu } -static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, + Value * filterFun, bool recursive, const Hash & expectedHash, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, *args[1], context); - if (!context.empty()) - throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); - - state.forceValue(*args[0]); - if (args[0]->type != tLambda) - throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos); - - path = state.checkSourcePath(path); - - PathFilter filter = [&](const Path & path) { + const auto path = settings.pureEval && expectedHash ? + path_ : + state.checkSourcePath(path_); + PathFilter filter = filterFun ? ([&](const Path & path) { auto st = lstat(path); /* Call the filter function. The first argument is the path, @@ -1031,7 +1024,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args mkString(arg1, path); Value fun2; - state.callFunction(*args[0], arg1, fun2, noPos); + state.callFunction(*filterFun, arg1, fun2, noPos); Value arg2; mkString(arg2, @@ -1044,16 +1037,79 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args state.callFunction(fun2, arg2, res, noPos); return state.forceBool(res, pos); - }; + }) : defaultPathFilter; - Path dstPath = settings.readOnlyMode - ? state.store->computeStorePathForPath(path, true, htSHA256, filter).first - : state.store->addToStore(baseNameOf(path), path, true, htSHA256, filter, state.repair); + Path expectedStorePath; + if (expectedHash) { + expectedStorePath = + state.store->makeFixedOutputPath(recursive, expectedHash, name); + } + Path dstPath; + if (!expectedHash || !state.store->isValidPath(expectedStorePath)) { + dstPath = settings.readOnlyMode + ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first + : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair); + if (expectedHash && expectedStorePath != dstPath) { + throw Error(format("store path mismatch in (possibly filtered) path added from '%1%'") % path); + } + } else + dstPath = expectedStorePath; mkString(v, dstPath, {dstPath}); } +static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + Path path = state.coerceToPath(pos, *args[1], context); + if (!context.empty()) + throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); + + state.forceValue(*args[0]); + if (args[0]->type != tLambda) + throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos); + + addPath(state, pos, baseNameOf(path), path, args[0], true, Hash(), v); +} + +static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos); + Path path; + string name; + Value * filterFun = nullptr; + auto recursive = true; + Hash expectedHash; + + for (auto & attr : *args[0]->attrs) { + const string & n(attr.name); + if (n == "path") { + PathSet context; + path = state.coerceToPath(*attr.pos, *attr.value, context); + if (!context.empty()) + throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % *attr.pos); + } else if (attr.name == state.sName) + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "filter") { + state.forceValue(*attr.value); + filterFun = attr.value; + } else if (n == "recursive") + recursive = state.forceBool(*attr.value, *attr.pos); + else if (n == "sha256") + expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + else + throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos); + } + if (path.empty()) + throw EvalError(format("'path' required, at %1%") % pos); + if (name.empty()) + name = baseNameOf(path); + + addPath(state, pos, name, path, filterFun, recursive, expectedHash, v); +} + + /************************************************************* * Sets *************************************************************/ @@ -2071,6 +2127,7 @@ void EvalState::createBaseEnv() addPrimOp("__fromJSON", 1, prim_fromJSON); addPrimOp("__toFile", 2, prim_toFile); addPrimOp("__filterSource", 2, prim_filterSource); + addPrimOp("__path", 1, prim_path); // Sets addPrimOp("__attrNames", 1, prim_attrNames); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 77ab87ef..7abb300a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -222,11 +222,10 @@ Path Store::makeTextPath(const string & name, const Hash & hash, } -std::pair Store::computeStorePathForPath(const Path & srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter) const +std::pair Store::computeStorePathForPath(const string & name, + const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const { Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - string name = baseNameOf(srcPath); Path dstPath = makeFixedOutputPath(recursive, h, name); return std::pair(dstPath, h); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index c0e735cd..bf0862ef 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -305,9 +305,9 @@ public: /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store path and the cryptographic hash of the contents of srcPath. */ - std::pair computeStorePathForPath(const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter) const; + std::pair computeStorePathForPath(const string & name, + const Path & srcPath, bool recursive = true, + HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; /* Preparatory part of addTextToStore(). diff --git a/tests/lang/data b/tests/lang/data new file mode 100644 index 00000000..257cc564 --- /dev/null +++ b/tests/lang/data @@ -0,0 +1 @@ +foo diff --git a/tests/lang/eval-okay-path.exp b/tests/lang/eval-okay-path.exp new file mode 100644 index 00000000..6827d49f --- /dev/null +++ b/tests/lang/eval-okay-path.exp @@ -0,0 +1 @@ +"/run/user/1000/nix-test/store/wjagrv37lfvfx92g2gf3yqflwypj0q1y-output" diff --git a/tests/lang/eval-okay-path.nix b/tests/lang/eval-okay-path.nix new file mode 100644 index 00000000..e67168cf --- /dev/null +++ b/tests/lang/eval-okay-path.nix @@ -0,0 +1,7 @@ +builtins.path + { path = ./.; + filter = path: _: baseNameOf path == "data"; + recursive = true; + sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw"; + name = "output"; + }