picnoir/Nix
1
0
Fork 0
Code Issues Pull requests 1 Releases Wiki Activity

Introduce MemoryInputAccessor and use it for corepkgs

Browse source 
MemoryInputAccessor is an in-memory virtual filesystem that returns
files like <nix/fetchurl.nix>. This removes the need for special hacks
to handle those files.
This commit is contained in:
Eelco Dolstra 2023-10-18 17:34:58 +02:00
parent ea38605d11
commit df73c6eb8c
11 changed files with 176 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
doc/manual/generate-manpage.nix
View file

@ -2,7 +2,7 @@ let
inherit (builtins)
attrNames attrValues fromJSON listToAttrs mapAttrs groupBy
concatStringsSep concatMap length lessThan replaceStrings sort;
inherit (import ./utils.nix) attrsToList concatStrings optionalString filterAttrs trim squash unique;
inherit (import <nix/utils.nix>) attrsToList concatStrings optionalString filterAttrs trim squash unique;
showStoreDocs = import ./generate-store-info.nix;
in

2
doc/manual/local.mk
View file

@ -32,7 +32,7 @@ dummy-env = env -i \
NIX_STATE_DIR=/dummy \
NIX_CONFIG='cores = 0'
nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw
nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw
# re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution
define process-includes

77
src/libexpr/eval.cc
View file

@ -13,6 +13,7 @@
#include "profiles.hh"
#include "print.hh"
#include "fs-input-accessor.hh"
#include "memory-input-accessor.hh"
#include <algorithm>
#include <chrono>
@ -516,7 +517,16 @@ EvalState::EvalState(
: "in restricted mode";
throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation);
}))
, derivationInternal(rootPath(CanonPath("/builtin/derivation.nix")))
, corepkgsFS(makeMemoryInputAccessor())
, internalFS(makeMemoryInputAccessor())
, derivationInternal{corepkgsFS->addFile(
CanonPath("derivation-internal.nix"),
#include "primops/derivation.nix.gen.hh"
)}
, callFlakeInternal{internalFS->addFile(
CanonPath("call-flake.nix"),
#include "flake/call-flake.nix.gen.hh"
)}
, store(store)
, buildStore(buildStore ? buildStore : store)
, debugRepl(nullptr)
@ -531,7 +541,8 @@ EvalState::EvalState(
, baseEnv(allocEnv(128))
, staticBaseEnv{std::make_shared<StaticEnv>(false, nullptr)}
{
rootFS->allowPath(CanonPath::root); // FIXME
// For now, don't rely on FSInputAccessor for access control.
rootFS->allowPath(CanonPath::root);
countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0";
@ -570,6 +581,11 @@ EvalState::EvalState(
}
}
corepkgsFS->addFile(
CanonPath("fetchurl.nix"),
#include "fetchurl.nix.gen.hh"
);
createBaseEnv();
}
@ -600,6 +616,7 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value &
SourcePath EvalState::checkSourcePath(const SourcePath & path_)
{
if (path_.accessor != rootFS) return path_;
if (!allowedPaths) return path_;
auto i = resolvedPaths.find(path_.path.abs());
@ -614,8 +631,6 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_)
*/
Path abspath = canonPath(path_.path.abs());
if (hasPrefix(abspath, corepkgsPrefix)) return rootPath(CanonPath(abspath));
for (auto & i : *allowedPaths) {
if (isDirOrInDir(abspath, i)) {
found = true;
@ -1180,24 +1195,6 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial
if (!e)
e = parseExprFromFile(checkSourcePath(resolvedPath));
cacheFile(path, resolvedPath, e, v, mustBeTrivial);
}
void EvalState::resetFileCache()
{
fileEvalCache.clear();
fileParseCache.clear();
}
void EvalState::cacheFile(
const SourcePath & path,
const SourcePath & resolvedPath,
Expr * e,
Value & v,
bool mustBeTrivial)
{
fileParseCache[resolvedPath] = e;
try {
@ -1226,6 +1223,13 @@ void EvalState::cacheFile(
}
void EvalState::resetFileCache()
{
fileEvalCache.clear();
fileParseCache.clear();
}
void EvalState::eval(Expr * e, Value & v)
{
e->eval(*this, baseEnv, v);
@ -2341,10 +2345,31 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx)
{
auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
if (path == "" || path[0] != '/')
error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow<EvalError>();
return rootPath(CanonPath(path));
try {
forceValue(v, pos);
if (v.type() == nString) {
copyContext(v, context);
auto s = v.string_view();
if (!hasPrefix(s, "/"))
error("string '%s' doesn't represent an absolute path", s).atPos(pos).debugThrow<EvalError>();
return rootPath(CanonPath(s));
}
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
throw;
}
if (v.type() == nPath)
return v.path();
if (v.type() == nAttrs) {
auto i = v.attrs->find(sOutPath);
if (i != v.attrs->end())
return coerceToPath(pos, *i->value, context, errorCtx);
}
error("cannot coerce %1% to a path", showType(v)).withTrace(pos, errorCtx).debugThrow<TypeError>();
}

32
src/libexpr/eval.hh
View file

@ -25,6 +25,7 @@ class StorePath;
struct SingleDerivedPath;
enum RepairFlag : bool;
struct FSInputAccessor;
struct MemoryInputAccessor;
/**
@ -212,10 +213,26 @@ public:
Bindings emptyBindings;
/**
* The accessor for the root filesystem.
*/
const ref<FSInputAccessor> rootFS;
/**
* The in-memory filesystem for <nix/...> paths.
*/
const ref<MemoryInputAccessor> corepkgsFS;
/**
* In-memory filesystem for internal, non-user-callable Nix
* expressions like call-flake.nix.
*/
const ref<MemoryInputAccessor> internalFS;
const SourcePath derivationInternal;
const SourcePath callFlakeInternal;
/**
* Store used to materialise .drv files.
*/
@ -226,7 +243,6 @@ public:
*/
const ref<Store> buildStore;
RootValue vCallFlake = nullptr;
RootValue vImportedDrvToDerivation = nullptr;
/**
@ -408,16 +424,6 @@ public:
*/
void evalFile(const SourcePath & path, Value & v, bool mustBeTrivial = false);
/**
* Like `evalFile`, but with an already parsed expression.
*/
void cacheFile(
const SourcePath & path,
const SourcePath & resolvedPath,
Expr * e,
Value & v,
bool mustBeTrivial = false);
void resetFileCache();
/**
@ -427,7 +433,7 @@ public:
SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
/**
* Try to resolve a search path value (not the optinal key part)
* Try to resolve a search path value (not the optional key part)
*
* If the specified search path element is a URI, download it.
*
@ -832,8 +838,6 @@ struct InvalidPathError : EvalError
#endif
};
static const std::string corepkgsPrefix{"/__corepkgs__/"};
template<class ErrorType>
void ErrorBuilder::debugThrow()
{

10
src/libexpr/flake/flake.cc
View file

@ -737,14 +737,10 @@ void callFlake(EvalState & state,
vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir);
if (!state.vCallFlake) {
state.vCallFlake = allocRootValue(state.allocValue());
state.eval(state.parseExprFromString(
#include "call-flake.nix.gen.hh"
, state.rootPath(CanonPath::root)), **state.vCallFlake);
}
auto vCallFlake = state.allocValue();
state.evalFile(state.callFlakeInternal, *vCallFlake);
state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos);
state.callFunction(*vCallFlake, *vLocks, *vTmp1, noPos);
state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
}

5
src/libexpr/parser.y
View file

@ -64,7 +64,6 @@ struct StringToken {
#include "parser-tab.hh"
#include "lexer-tab.hh"
#include "fs-input-accessor.hh"
YY_DECL;
@ -650,6 +649,8 @@ formal
#include "fetchers.hh"
#include "store-api.hh"
#include "flake/flake.hh"
#include "fs-input-accessor.hh"
#include "memory-input-accessor.hh"
namespace nix {
@ -761,7 +762,7 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_
}
if (hasPrefix(path, "nix/"))
return rootPath(CanonPath(concatStrings(corepkgsPrefix, path.substr(4))));
return {corepkgsFS, CanonPath(path.substr(3))};
debugThrow(ThrownError({
.msg = hintfmt(evalSettings.pureEval

29
src/libexpr/primops.cc
View file

@ -121,13 +121,15 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, co
auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path");
try {
StringMap rewrites = state.realiseContext(context);
auto realPath = state.rootPath(CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context)));
if (!context.empty()) {
auto rewrites = state.realiseContext(context);
auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context);
return {path.accessor, CanonPath(realPath)};
}
return flags.checkForPureEval
? state.checkSourcePath(realPath)
: realPath;
? state.checkSourcePath(path)
: path;
} catch (Error & e) {
e.addTrace(state.positions[pos], "while realising the context of path '%s'", path);
throw;
@ -210,12 +212,6 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh");
}
else if (path2 == corepkgsPrefix + "fetchurl.nix") {
state.eval(state.parseExprFromString(
#include "fetchurl.nix.gen.hh"
, state.rootPath(CanonPath::root)), v);
}
else {
if (!vScope)
state.evalFile(path, v);
@ -1486,7 +1482,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args,
}));
NixStringContext context;
auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")).path;
auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'")).path;
/* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink
directly in the store. The latter condition is necessary so
e.g. nix-push does the right thing. */
@ -2257,7 +2253,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg
{
NixStringContext context;
auto path = state.coerceToPath(pos, *args[1], context,
"while evaluating the second argument (the path to filter) passed to builtins.filterSource");
"while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'");
state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource");
addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
}
@ -4444,12 +4440,7 @@ void EvalState::createBaseEnv()
/* Note: we have to initialize the 'derivation' constant *after*
building baseEnv/staticBaseEnv because it uses 'builtins'. */
char code[] =
#include "primops/derivation.nix.gen.hh"
// the parser needs two NUL bytes as terminators; one of them
// is implied by being a C string.
"\0";
eval(parse(code, sizeof(code), derivationInternal, rootPath(CanonPath::root), staticBaseEnv), *vDerivation);
evalFile(derivationInternal, *vDerivation);
}

14
src/libexpr/tests/error_traces.cc
View file

@ -295,7 +295,7 @@ namespace nix {
TEST_F(ErrorTraceTest, toPath) {
ASSERT_TRACE2("toPath []",
TypeError,
hintfmt("cannot coerce %s to a string", "a list"),
hintfmt("cannot coerce %s to a path", "a list"),
hintfmt("while evaluating the first argument passed to builtins.toPath"));
ASSERT_TRACE2("toPath \"foo\"",
@ -309,8 +309,8 @@ namespace nix {
TEST_F(ErrorTraceTest, storePath) {
ASSERT_TRACE2("storePath true",
TypeError,
hintfmt("cannot coerce %s to a string", "a Boolean"),
hintfmt("while evaluating the first argument passed to builtins.storePath"));
hintfmt("cannot coerce %s to a path", "a Boolean"),
hintfmt("while evaluating the first argument passed to 'builtins.storePath'"));
}
@ -318,7 +318,7 @@ namespace nix {
TEST_F(ErrorTraceTest, pathExists) {
ASSERT_TRACE2("pathExists []",
TypeError,
hintfmt("cannot coerce %s to a string", "a list"),
hintfmt("cannot coerce %s to a path", "a list"),
hintfmt("while realising the context of a path"));
ASSERT_TRACE2("pathExists \"zorglub\"",
@ -377,13 +377,13 @@ namespace nix {
TEST_F(ErrorTraceTest, filterSource) {
ASSERT_TRACE2("filterSource [] []",
TypeError,
hintfmt("cannot coerce %s to a string", "a list"),
hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource"));
hintfmt("cannot coerce %s to a path", "a list"),
hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"));
ASSERT_TRACE2("filterSource [] \"foo\"",
EvalError,
hintfmt("string '%s' doesn't represent an absolute path", "foo"),
hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource"));
hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"));
ASSERT_TRACE2("filterSource [] ./.",
TypeError,

54
src/libfetchers/memory-input-accessor.cc Normal file
View file

@ -0,0 +1,54 @@
#include "memory-input-accessor.hh"
namespace nix {
struct MemoryInputAccessorImpl : MemoryInputAccessor
{
std::map<CanonPath, std::string> files;
std::string readFile(const CanonPath & path) override
{
auto i = files.find(path);
if (i == files.end())
throw Error("file '%s' does not exist", path);
return i->second;
}
bool pathExists(const CanonPath & path) override
{
auto i = files.find(path);
return i != files.end();
}
Stat lstat(const CanonPath & path) override
{
auto i = files.find(path);
if (i != files.end())
return Stat { .type = tRegular, .isExecutable = false };
throw Error("file '%s' does not exist", path);
}
DirEntries readDirectory(const CanonPath & path) override
{
return {};
}
std::string readLink(const CanonPath & path) override
{
throw UnimplementedError("MemoryInputAccessor::readLink");
}
SourcePath addFile(CanonPath path, std::string && contents) override
{
files.emplace(path, std::move(contents));
return {ref(shared_from_this()), std::move(path)};
}
};
ref<MemoryInputAccessor> makeMemoryInputAccessor()
{
return make_ref<MemoryInputAccessorImpl>();
}
}

15
src/libfetchers/memory-input-accessor.hh Normal file
View file

@ -0,0 +1,15 @@
#include "input-accessor.hh"
namespace nix {
/**
* An input accessor for an in-memory file system.
*/
struct MemoryInputAccessor : InputAccessor
{
virtual SourcePath addFile(CanonPath path, std::string && contents) = 0;
};
ref<MemoryInputAccessor> makeMemoryInputAccessor();
}

34
src/nix/main.cc
View file

@ -12,6 +12,7 @@
#include "finally.hh"
#include "loggers.hh"
#include "markdown.hh"
#include "memory-input-accessor.hh"
#include <sys/types.h>
#include <sys/socket.h>
@ -206,29 +207,20 @@ static void showHelp(std::vector<std::string> subcommand, NixArgs & toplevel)
#include "generate-manpage.nix.gen.hh"
, state.rootPath(CanonPath::root)), *vGenerateManpage);
auto vUtils = state.allocValue();
state.cacheFile(
state.rootPath(CanonPath("/utils.nix")), state.rootPath(CanonPath("/utils.nix")),
state.parseExprFromString(
#include "utils.nix.gen.hh"
, state.rootPath(CanonPath::root)),
*vUtils);
state.corepkgsFS->addFile(
CanonPath("utils.nix"),
#include "utils.nix.gen.hh"
);
auto vSettingsInfo = state.allocValue();
state.cacheFile(
state.rootPath(CanonPath("/generate-settings.nix")), state.rootPath(CanonPath("/generate-settings.nix")),
state.parseExprFromString(
#include "generate-settings.nix.gen.hh"
, state.rootPath(CanonPath::root)),
*vSettingsInfo);
state.corepkgsFS->addFile(
CanonPath("/generate-settings.nix"),
#include "generate-settings.nix.gen.hh"
);
auto vStoreInfo = state.allocValue();
state.cacheFile(
state.rootPath(CanonPath("/generate-store-info.nix")), state.rootPath(CanonPath("/generate-store-info.nix")),
state.parseExprFromString(
#include "generate-store-info.nix.gen.hh"
, state.rootPath(CanonPath::root)),
*vStoreInfo);
state.corepkgsFS->addFile(
CanonPath("/generate-store-info.nix"),
#include "generate-store-info.nix.gen.hh"
);
auto vDump = state.allocValue();
vDump->mkString(toplevel.dumpCli());