picnoir/Nix
1
0
Fork 0
Code Issues Pull requests 1 Releases Wiki Activity

Squashed get CA derivations building

Browse source
This commit is contained in:
John Ericson 2020-08-07 19:09:26 +00:00
parent f7ba16f9cb
commit e913a2989f
29 changed files with 1021 additions and 485 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
src/libexpr/get-drvs.cc
View file

@ -39,7 +39,9 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName);
outPath = store->printStorePath(i->second.path(*store, drv.name));
auto optStorePath = i->second.pathOpt(*store, drv.name);
if (optStorePath)
outPath = store->printStorePath(*optStorePath);
}
@ -77,12 +79,15 @@ string DrvInfo::queryDrvPath() const
string DrvInfo::queryOutPath() const
{
if (outPath == "" && attrs) {
if (!outPath && attrs) {
Bindings::iterator i = attrs->find(state->sOutPath);
PathSet context;
outPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : "";
if (i != attrs->end())
outPath = state->coerceToPath(*i->pos, *i->value, context);
}
return outPath;
if (!outPath)
throw UnimplementedError("CA derivations are not yet supported");
return *outPath;
}

2
src/libexpr/get-drvs.hh
View file

@ -20,7 +20,7 @@ private:
mutable string name;
mutable string system;
mutable string drvPath;
mutable string outPath;
mutable std::optional<string> outPath;
mutable string outputName;
Outputs outputs;

59
src/libexpr/primops.cc
View file

@ -44,16 +44,6 @@ void EvalState::realiseContext(const PathSet & context)
throw InvalidPathError(store->printStorePath(ctx));
if (!outputName.empty() && ctx.isDerivation()) {
drvs.push_back(StorePathWithOutputs{ctx, {outputName}});
/* Add the output of this derivation to the allowed
paths. */
if (allowedPaths) {
auto drv = store->derivationFromPath(ctx);
DerivationOutputs::iterator i = drv.outputs.find(outputName);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName);
allowedPaths->insert(store->printStorePath(i->second.path(*store, drv.name)));
}
}
}
@ -69,8 +59,38 @@ void EvalState::realiseContext(const PathSet & context)
store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
store->buildPaths(drvs);
/* Add the output of this derivations to the allowed
paths. */
if (allowedPaths) {
for (auto & [drvPath, outputs] : drvs) {
auto outputPaths = store->queryDerivationOutputMapAssumeTotal(drvPath);
for (auto & outputName : outputs) {
if (outputPaths.count(outputName) == 0)
throw Error("derivation '%s' does not have an output named '%s'",
store->printStorePath(drvPath), outputName);
allowedPaths->insert(store->printStorePath(outputPaths.at(outputName)));
}
}
}
}
static void mkOutputString(EvalState & state, Value & v,
const StorePath & drvPath, const BasicDerivation & drv,
std::pair<string, DerivationOutput> o)
{
auto optOutputPath = o.second.pathOpt(*state.store, drv.name);
mkString(
*state.allocAttr(v, state.symbols.create(o.first)),
state.store->printStorePath(optOutputPath
? *optOutputPath
/* Downstream we would substitute this for an actual path once
we build the floating CA derivation */
/* FIXME: we need to depend on the basic derivation, not
derivation */
: downstreamPlaceholder(*state.store, drvPath, o.first)),
{"!" + o.first + "!" + state.store->printStorePath(drvPath)});
}
/* Load and evaluate an expression from path specified by the
argument. */
@ -114,8 +134,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
unsigned int outputs_index = 0;
for (const auto & o : drv.outputs) {
v2 = state.allocAttr(w, state.symbols.create(o.first));
mkString(*v2, state.store->printStorePath(o.second.path(*state.store, drv.name)), {"!" + o.first + "!" + path});
mkOutputString(state, w, storePath, drv, o);
outputsVal->listElems()[outputs_index] = state.allocValue();
mkString(*(outputsVal->listElems()[outputs_index++]), o.first);
}
@ -846,16 +865,18 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
/* Optimisation, but required in read-only mode! because in that
case we don't actually write store derivations, so we can't
read them later. */
drvHashes.insert_or_assign(drvPath,
hashDerivationModulo(*state.store, Derivation(drv), false));
read them later.
However, we don't bother doing this for floating CA derivations because
their "hash modulo" is indeterminate until built. */
if (drv.type() != DerivationType::CAFloating)
drvHashes.insert_or_assign(drvPath,
hashDerivationModulo(*state.store, Derivation(drv), false));
state.mkAttrs(v, 1 + drv.outputs.size());
mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
for (auto & i : drv.outputs) {
mkString(*state.allocAttr(v, state.symbols.create(i.first)),
state.store->printStorePath(i.second.path(*state.store, drv.name)), {"!" + i.first + "!" + drvPathS});
}
for (auto & i : drv.outputs)
mkOutputString(state, v, drvPath, drv, i);
v.attrs->sort();
}

964
src/libstore/build.cc
View file

File diff suppressed because it is too large Load diff

158
src/libstore/derivations.cc
View file

@ -15,7 +15,8 @@ std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::str
},
[&](DerivationOutputCAFixed dof) -> std::optional<StorePath> {
return {
store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName)
// FIXME if we intend to support multiple CA outputs.
dof.path(store, drvName, "out")
};
},
[](DerivationOutputCAFloating dof) -> std::optional<StorePath> {
@ -25,6 +26,13 @@ std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::str
}
StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
return store.makeFixedOutputPath(
hash.method, hash.hash,
outputPathName(drvName, outputName));
}
bool derivationIsCA(DerivationType dt) {
switch (dt) {
case DerivationType::InputAddressed: return false;
@ -106,12 +114,15 @@ static string parseString(std::istream & str)
return res;
}
static void validatePath(std::string_view s) {
if (s.size() == 0 || s[0] != '/')
throw FormatError("bad path '%1%' in derivation", s);
}
static Path parsePath(std::istream & str)
{
string s = parseString(str);
if (s.size() == 0 || s[0] != '/')
throw FormatError("bad path '%1%' in derivation", s);
auto s = parseString(str);
validatePath(s);
return s;
}
@ -141,7 +152,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
{
expect(str, ","); auto path = store.parseStorePath(parsePath(str));
expect(str, ","); auto pathS = parseString(str);
expect(str, ","); auto hashAlgo = parseString(str);
expect(str, ","); const auto hash = parseString(str);
expect(str, ")");
@ -152,30 +163,35 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings
method = FileIngestionMethod::Recursive;
hashAlgo = string(hashAlgo, 2);
}
const HashType hashType = parseHashType(hashAlgo);
return hash != ""
? DerivationOutput {
.output = DerivationOutputCAFixed {
.hash = FixedOutputHash {
.method = std::move(method),
.hash = Hash::parseNonSRIUnprefixed(hash, hashType),
},
}
}
: (settings.requireExperimentalFeature("ca-derivations"),
DerivationOutput {
.output = DerivationOutputCAFloating {
.method = std::move(method),
.hashType = std::move(hashType),
},
});
} else
const auto hashType = parseHashType(hashAlgo);
if (hash != "") {
validatePath(pathS);
return DerivationOutput {
.output = DerivationOutputCAFixed {
.hash = FixedOutputHash {
.method = std::move(method),
.hash = Hash::parseNonSRIUnprefixed(hash, hashType),
},
},
};
} else {
settings.requireExperimentalFeature("ca-derivations");
assert(pathS == "");
return DerivationOutput {
.output = DerivationOutputCAFloating {
.method = std::move(method),
.hashType = std::move(hashType),
},
};
}
} else {
validatePath(pathS);
return DerivationOutput {
.output = DerivationOutputInputAddressed {
.path = std::move(path),
.path = store.parseStorePath(pathS),
}
};
}
}
@ -316,17 +332,19 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
for (auto & i : outputs) {
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first);
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name)));
std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path));
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
},
[&](DerivationOutputCAFixed dof) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
},
[&](DerivationOutputCAFloating dof) {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
s += ','; printUnquotedString(s, "");
},
@ -382,6 +400,16 @@ bool isDerivation(const string & fileName)
}
std::string outputPathName(std::string_view drvName, std::string_view outputName) {
std::string res { drvName };
if (outputName != "out") {
res += "-";
res += outputName;
}
return res;
}
DerivationType BasicDerivation::type() const
{
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs;
@ -479,7 +507,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
auto hash = hashString(htSHA256, "fixed:out:"
+ dof.hash.printMethodAlgo() + ":"
+ dof.hash.hash.to_string(Base16, false) + ":"
+ store.printStorePath(i.second.path(store, drv.name)));
+ store.printStorePath(dof.path(store, drv.name, i.first)));
outputHashes.insert_or_assign(i.first, std::move(hash));
}
return outputHashes;
@ -530,17 +558,9 @@ bool wantOutput(const string & output, const std::set<string> & wanted)
}
StorePathSet BasicDerivation::outputPaths(const Store & store) const
{
StorePathSet paths;
for (auto & i : outputs)
paths.insert(i.second.path(store, name));
return paths;
}
static DerivationOutput readDerivationOutput(Source & in, const Store & store)
{
auto path = store.parseStorePath(readString(in));
auto pathS = readString(in);
auto hashAlgo = readString(in);
auto hash = readString(in);
@ -550,29 +570,35 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
method = FileIngestionMethod::Recursive;
hashAlgo = string(hashAlgo, 2);
}
auto hashType = parseHashType(hashAlgo);
return hash != ""
? DerivationOutput {
.output = DerivationOutputCAFixed {
.hash = FixedOutputHash {
.method = std::move(method),
.hash = Hash::parseNonSRIUnprefixed(hash, hashType),
},
}
}
: (settings.requireExperimentalFeature("ca-derivations"),
DerivationOutput {
.output = DerivationOutputCAFloating {
.method = std::move(method),
.hashType = std::move(hashType),
},
});
} else
const auto hashType = parseHashType(hashAlgo);
if (hash != "") {
validatePath(pathS);
return DerivationOutput {
.output = DerivationOutputCAFixed {
.hash = FixedOutputHash {
.method = std::move(method),
.hash = Hash::parseNonSRIUnprefixed(hash, hashType),
},
},
};
} else {
settings.requireExperimentalFeature("ca-derivations");
assert(pathS == "");
return DerivationOutput {
.output = DerivationOutputCAFloating {
.method = std::move(method),
.hashType = std::move(hashType),
},
};
}
} else {
validatePath(pathS);
return DerivationOutput {
.output = DerivationOutputInputAddressed {
.path = std::move(path),
.path = store.parseStorePath(pathS),
}
};
}
}
StringSet BasicDerivation::outputNames() const
@ -624,18 +650,21 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
{
out << drv.outputs.size();
for (auto & i : drv.outputs) {
out << i.first
<< store.printStorePath(i.second.path(store, drv.name));
out << i.first;
std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) {
out << "" << "";
out << store.printStorePath(doi.path)
<< ""
<< "";
},
[&](DerivationOutputCAFixed dof) {
out << dof.hash.printMethodAlgo()
out << store.printStorePath(dof.path(store, drv.name, i.first))
<< dof.hash.printMethodAlgo()
<< dof.hash.hash.to_string(Base16, false);
},
[&](DerivationOutputCAFloating dof) {
out << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
out << ""
<< (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
<< "";
},
}, i.second.output);
@ -654,5 +683,14 @@ std::string hashPlaceholder(const std::string & outputName)
return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false);
}
StorePath downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName)
{
auto drvNameWithExtension = drvPath.name();
auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
return store.makeStorePath(
"downstream-placeholder:" + std::string { drvPath.name() } + ":" + std::string { outputName },
"compressed:" + std::string { drvPath.hashPart() },
outputPathName(drvName, outputName));
}
}

14
src/libstore/derivations.hh
View file

@ -27,6 +27,7 @@ struct DerivationOutputInputAddressed
struct DerivationOutputCAFixed
{
FixedOutputHash hash; /* hash used for expected hash computation */
StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
};
/* Floating-output derivations, whose output paths are content addressed, but
@ -48,12 +49,6 @@ struct DerivationOutput
> output;
std::optional<HashType> hashAlgoOpt(const Store & store) const;
std::optional<StorePath> pathOpt(const Store & store, std::string_view drvName) const;
/* DEPRECATED: Remove after CA drvs are fully implemented */
StorePath path(const Store & store, std::string_view drvName) const {
auto p = pathOpt(store, drvName);
if (!p) throw UnimplementedError("floating content-addressed derivations are not yet implemented");
return *p;
}
};
typedef std::map<string, DerivationOutput> DerivationOutputs;
@ -101,9 +96,6 @@ struct BasicDerivation
/* Return true iff this is a fixed-output derivation. */
DerivationType type() const;
/* Return the output paths of a derivation. */
StorePathSet outputPaths(const Store & store) const;
/* Return the output names of a derivation. */
StringSet outputNames() const;
@ -136,6 +128,8 @@ Derivation readDerivation(const Store & store, const Path & drvPath, std::string
// FIXME: remove
bool isDerivation(const string & fileName);
std::string outputPathName(std::string_view drvName, std::string_view outputName);
// known CA drv's output hashes, current just for fixed-output derivations
// whose output hashes are always known since they are fixed up-front.
typedef std::map<std::string, Hash> CaOutputHashes;
@ -185,4 +179,6 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
std::string hashPlaceholder(const std::string & outputName);
StorePath downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
}

31
src/libstore/local-store.cc
View file

@ -578,13 +578,32 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
envHasRightPath(path, i.first);
},
[&](DerivationOutputCAFloating _) {
throw UnimplementedError("floating CA output derivations are not yet implemented");
/* Nothing to check */
},
}, i.second.output);
}
}
void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output)
{
auto state(_state.lock());
return linkDeriverToPath(*state, queryValidPathId(*state, deriver), outputName, output);
}
void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output)
{
retrySQLite<void>([&]() {
state.stmtAddDerivationOutput.use()
(deriver)
(outputName)
(printStorePath(output))
.exec();
});
}
uint64_t LocalStore::addValidPath(State & state,
const ValidPathInfo & info, bool checkOutputs)
{
@ -619,11 +638,11 @@ uint64_t LocalStore::addValidPath(State & state,
if (checkOutputs) checkDerivationOutputs(info.path, drv);
for (auto & i : drv.outputs) {
state.stmtAddDerivationOutput.use()
(id)
(i.first)
(printStorePath(i.second.path(*this, drv.name)))
.exec();
/* Floating CA derivations have indeterminate output paths until
they are built, so don't register anything in that case */
auto optPath = i.second.pathOpt(*this, drv.name);
if (optPath)
linkDeriverToPath(state, id, i.first, *optPath);
}
}

6
src/libstore/local-store.hh
View file

@ -282,6 +282,12 @@ private:
specified by the ‘secret-key-files’ option. */
void signPathInfo(ValidPathInfo & info);
/* Add a mapping from the deriver of the path info (if specified) to its
* out path
*/
void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output);
void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output);
Path getRealStoreDir() override { return realStoreDir; }
void createUser(const std::string & userName, uid_t userId) override;

23
src/libstore/misc.cc
View file

@ -203,17 +203,24 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
return;
}
PathSet invalid;
/* true for regular derivations, and CA derivations for which we
have a trust mapping for all wanted outputs. */
auto knownOutputPaths = true;
for (auto & [outputName, pathOpt] : queryDerivationOutputMap(path.path)) {
if (!pathOpt) {
knownOutputPaths = false;
break;
}
if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt))
invalid.insert(printStorePath(*pathOpt));
}
if (knownOutputPaths && invalid.empty()) return;
auto drv = make_ref<Derivation>(derivationFromPath(path.path));
ParsedDerivation parsedDrv(StorePath(path.path), *drv);
PathSet invalid;
for (auto & j : drv->outputs)
if (wantOutput(j.first, path.outputs)
&& !isValidPath(j.second.path(*this, drv->name)))
invalid.insert(printStorePath(j.second.path(*this, drv->name)));
if (invalid.empty()) return;
if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
for (auto & output : invalid)
pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState));

16
src/libstore/references.cc
View file

@ -79,9 +79,17 @@ void RefScanSink::operator () (const unsigned char * data, size_t len)
std::pair<PathSet, HashResult> scanForReferences(const string & path,
const PathSet & refs)
{
RefScanSink refsSink;
HashSink hashSink { htSHA256 };
TeeSink sink { refsSink, hashSink };
auto found = scanForReferences(hashSink, path, refs);
auto hash = hashSink.finish();
return std::pair<PathSet, HashResult>(found, hash);
}
PathSet scanForReferences(Sink & toTee,
const string & path, const PathSet & refs)
{
RefScanSink refsSink;
TeeSink sink { refsSink, toTee };
std::map<string, Path> backMap;
/* For efficiency (and a higher hit rate), just search for the
@ -111,9 +119,7 @@ std::pair<PathSet, HashResult> scanForReferences(const string & path,
found.insert(j->second);
}
auto hash = hashSink.finish();
return std::pair<PathSet, HashResult>(found, hash);
return found;
}

2
src/libstore/references.hh
View file

@ -7,6 +7,8 @@ namespace nix {
std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs);
PathSet scanForReferences(Sink & toTee, const Path & path, const PathSet & refs);
struct RewritingSink : Sink
{
std::string from, to, prev;

2
src/libstore/remote-store.cc
View file

@ -74,7 +74,7 @@ void write(const Store & store, Sink & out, const StorePath & storePath)
template<>
std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
{
auto s = readString(from);
auto s = readString(from);
return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
}

19
src/libstore/store-api.cc
View file

@ -140,21 +140,28 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
*/
StorePath Store::makeStorePath(const string & type,
const Hash & hash, std::string_view name) const
StorePath Store::makeStorePath(std::string_view type,
std::string_view hash, std::string_view name) const
{
/* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name);
string s = std::string { type } + ":" + std::string { hash }
+ ":" + storeDir + ":" + std::string { name };
auto h = compressHash(hashString(htSHA256, s), 20);
return StorePath(h, name);
}
StorePath Store::makeOutputPath(const string & id,
StorePath Store::makeStorePath(std::string_view type,
const Hash & hash, std::string_view name) const
{
return makeStorePath("output:" + id, hash,
std::string(name) + (id == "out" ? "" : "-" + id));
return makeStorePath(type, hash.to_string(Base16, true), name);
}
StorePath Store::makeOutputPath(std::string_view id,
const Hash & hash, std::string_view name) const
{
return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id));
}

6
src/libstore/store-api.hh
View file

@ -244,10 +244,12 @@ public:
StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const;
/* Constructs a unique store path name. */
StorePath makeStorePath(const string & type,
StorePath makeStorePath(std::string_view type,
std::string_view hash, std::string_view name) const;
StorePath makeStorePath(std::string_view type,
const Hash & hash, std::string_view name) const;
StorePath makeOutputPath(const string & id,
StorePath makeOutputPath(std::string_view id,
const Hash & hash, std::string_view name) const;
StorePath makeFixedOutputPath(FileIngestionMethod method,

6
src/libutil/serialise.hh
View file

@ -22,6 +22,12 @@ struct Sink
}
};
/* Just throws away data. */
struct NullSink : Sink
{
void operator () (const unsigned char * data, size_t len) override
{ }
};
/* A buffered abstract sink. */
struct BufferedSink : virtual Sink

56
src/nix-build/nix-build.cc
View file

@ -487,50 +487,56 @@ static void _main(int argc, char * * argv)
std::vector<StorePathWithOutputs> pathsToBuild;
std::map<Path, Path> drvPrefixes;
std::map<Path, Path> resultSymlinks;
std::vector<Path> outPaths;
std::map<StorePath, std::pair<size_t, StringSet>> drvMap;
for (auto & drvInfo : drvs) {
auto drvPath = drvInfo.queryDrvPath();
auto outPath = drvInfo.queryOutPath();
auto drvPath = store->parseStorePath(drvInfo.queryDrvPath());
auto outputName = drvInfo.queryOutputName();
if (outputName == "")
throw Error("derivation '%s' lacks an 'outputName' attribute", drvPath);
throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath));
pathsToBuild.push_back({store->parseStorePath(drvPath), {outputName}});
pathsToBuild.push_back({drvPath, {outputName}});
std::string drvPrefix;
auto i = drvPrefixes.find(drvPath);
if (i != drvPrefixes.end())
drvPrefix = i->second;
auto i = drvMap.find(drvPath);
if (i != drvMap.end())
i->second.second.insert(outputName);
else {
drvPrefix = outLink;
if (drvPrefixes.size())
drvPrefix += fmt("-%d", drvPrefixes.size() + 1);
drvPrefixes[drvPath] = drvPrefix;
drvMap[drvPath] = {drvMap.size(), {outputName}};
}
std::string symlink = drvPrefix;
if (outputName != "out") symlink += "-" + outputName;
resultSymlinks[symlink] = outPath;
outPaths.push_back(outPath);
}
buildPaths(pathsToBuild);
if (dryRun) return;
for (auto & symlink : resultSymlinks)
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
store2->addPermRoot(store->parseStorePath(symlink.second), absPath(symlink.first), true);
std::vector<StorePath> outPaths;
for (auto & [drvPath, info] : drvMap) {
auto & [counter, wantedOutputs] = info;
std::string drvPrefix = outLink;
if (counter)
drvPrefix += fmt("-%d", counter + 1);
auto builtOutputs = store->queryDerivationOutputMapAssumeTotal(drvPath);
for (auto & outputName : wantedOutputs) {
auto outputPath = builtOutputs.at(outputName);
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
std::string symlink = drvPrefix;
if (outputName != "out") symlink += "-" + outputName;
store2->addPermRoot(outputPath, absPath(symlink), true);
}
outPaths.push_back(outputPath);
}
}
logger->stop();
for (auto & path : outPaths)
std::cout << path << '\n';
std::cout << store->printStorePath(path) << '\n';
}
}

26
src/nix-store/nix-store.cc
View file

@ -65,6 +65,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
if (path.path.isDerivation()) {
if (build) store->buildPaths({path});
auto outputPaths = store->queryDerivationOutputMapAssumeTotal(path.path);
Derivation drv = store->derivationFromPath(path.path);
rootNr++;
@ -77,7 +78,8 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have an output named '%s'",
store2->printStorePath(path.path), j);
auto outPath = store2->printStorePath(i->second.path(*store, drv.name));
auto outPath = outputPaths.at(i->first);
auto retPath = store->printStorePath(outPath);
if (store2) {
if (gcRoot == "")
printGCWarning();
@ -85,10 +87,10 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
Path rootName = gcRoot;
if (rootNr > 1) rootName += "-" + std::to_string(rootNr);
if (i->first != "out") rootName += "-" + i->first;
outPath = store2->addPermRoot(store->parseStorePath(outPath), rootName, indirectRoot);
retPath = store2->addPermRoot(outPath, rootName, indirectRoot);
}
}
outputs.insert(outPath);
outputs.insert(retPath);
}
return outputs;
}
@ -218,8 +220,14 @@ static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput,
if (useOutput && storePath.isDerivation()) {
auto drv = store->derivationFromPath(storePath);
StorePathSet outputs;
for (auto & i : drv.outputs)
outputs.insert(i.second.path(*store, drv.name));
if (forceRealise)
return store->queryDerivationOutputs(storePath);
for (auto & i : drv.outputs) {
auto optPath = i.second.pathOpt(*store, drv.name);
if (!optPath)
throw UsageError("Cannot use output path of floating content-addressed derivation until we know what it is (e.g. by building it)");
outputs.insert(*optPath);
}
return outputs;
}
else return {storePath};
@ -309,11 +317,9 @@ static void opQuery(Strings opFlags, Strings opArgs)
case qOutputs: {
for (auto & i : opArgs) {
auto i2 = store->followLinksToStorePath(i);
if (forceRealise) realisePath({i2});
Derivation drv = store->derivationFromPath(i2);
for (auto & j : drv.outputs)
cout << fmt("%1%\n", store->printStorePath(j.second.path(*store, drv.name)));
auto outputs = maybeUseOutputs(store->followLinksToStorePath(i), true, forceRealise);
for (auto & outputPath : outputs)
cout << fmt("%1%\n", store->printStorePath(outputPath));
}
break;
}

3
src/nix/build.cc
View file

@ -74,7 +74,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
store2->addPermRoot(bo.path, absPath(symlink), true);
},
[&](BuildableFromDrv bfd) {
for (auto & output : bfd.outputs) {
auto builtOutputs = store->queryDerivationOutputMapAssumeTotal(bfd.drvPath);
for (auto & output : builtOutputs) {
std::string symlink = outLink;
if (i) symlink += fmt("-%d", i);
if (output.first != "out") symlink += fmt("-%s", output.first);

4
src/nix/command.cc
View file

@ -137,7 +137,9 @@ void MixProfile::updateProfile(const Buildables & buildables)
},
[&](BuildableFromDrv bfd) {
for (auto & output : bfd.outputs) {
result.push_back(output.second);
if (!output.second)
throw Error("output path should be known because we just tried to build it");
result.push_back(*output.second);
}
},
}, buildable);

13
src/nix/installables.cc
View file

@ -302,10 +302,10 @@ struct InstallableStorePath : Installable
Buildables toBuildables() override
{
if (storePath.isDerivation()) {
std::map<std::string, StorePath> outputs;
std::map<std::string, std::optional<StorePath>> outputs;
auto drv = store->readDerivation(storePath);
for (auto & [name, output] : drv.outputs)
outputs.emplace(name, output.path(*store, drv.name));
outputs.emplace(name, output.pathOpt(*store, drv.name));
return {
BuildableFromDrv {
.drvPath = storePath,
@ -331,7 +331,7 @@ Buildables InstallableValue::toBuildables()
{
Buildables res;
std::map<StorePath, OutputPathMap> drvsToOutputs;
std::map<StorePath, std::map<std::string, std::optional<StorePath>>> drvsToOutputs;
// Group by derivation, helps with .all in particular
for (auto & drv : toDerivations()) {
@ -674,8 +674,11 @@ StorePathSet toStorePaths(ref<Store> store,
outPaths.insert(bo.path);
},
[&](BuildableFromDrv bfd) {
for (auto & output : bfd.outputs)
outPaths.insert(output.second);
for (auto & output : bfd.outputs) {
if (!output.second)
throw Error("Cannot operate on output of unbuilt CA drv");
outPaths.insert(*output.second);
}
},
}, b);
} else {

4
src/nix/installables.hh
View file

@ -20,7 +20,7 @@ struct BuildableOpaque {
struct BuildableFromDrv {
StorePath drvPath;
std::map<std::string, StorePath> outputs;
std::map<std::string, std::optional<StorePath>> outputs;
};
typedef std::variant<
@ -82,7 +82,7 @@ struct InstallableValue : Installable
struct DerivationInfo
{
StorePath drvPath;
StorePath outPath;
std::optional<StorePath> outPath;
std::string outputName;
};

10
src/nix/profile.cc
View file

@ -178,7 +178,9 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
auto [attrPath, resolvedRef, drv] = installable2->toDerivation();
ProfileElement element;
element.storePaths = {drv.outPath}; // FIXME
if (!drv.outPath)
throw UnimplementedError("CA derivations are not yet supported by 'nix profile'");
element.storePaths = {*drv.outPath}; // FIXME
element.source = ProfileElementSource{
installable2->flakeRef,
resolvedRef,
@ -189,7 +191,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
manifest.elements.emplace_back(std::move(element));
} else
throw Error("'nix profile install' does not support argument '%s'", installable->what());
throw UnimplementedError("'nix profile install' does not support argument '%s'", installable->what());
}
store->buildPaths(pathsToBuild);
@ -347,7 +349,9 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
printInfo("upgrading '%s' from flake '%s' to '%s'",
element.source->attrPath, element.source->resolvedRef, resolvedRef);
element.storePaths = {drv.outPath}; // FIXME
if (!drv.outPath)
throw UnimplementedError("CA derivations are not yet supported by 'nix profile'");
element.storePaths = {*drv.outPath}; // FIXME
element.source = ProfileElementSource{
installable.flakeRef,
resolvedRef,

6
src/nix/repl.cc
View file

@ -488,10 +488,10 @@ bool NixRepl::processLine(string line)
but doing it in a child makes it easier to recover from
problems / SIGINT. */
if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) {
auto drv = readDerivation(*state->store, drvPath, Derivation::nameFromPath(state->store->parseStorePath(drvPath)));
auto outputs = state->store->queryDerivationOutputMapAssumeTotal(state->store->parseStorePath(drvPath));
std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
for (auto & i : drv.outputs)
std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path(*state->store, drv.name)));
for (auto & i : outputs)
std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second));
}
} else if (command == ":i") {
runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath});

4
src/nix/show-derivation.cc
View file

@ -69,12 +69,12 @@ struct CmdShowDerivation : InstallablesCommand
auto outputsObj(drvObj.object("outputs"));
for (auto & output : drv.outputs) {
auto outputObj(outputsObj.object(output.first));
outputObj.attr("path", store->printStorePath(output.second.path(*store, drv.name)));
std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) {
outputObj.attr("path", store->printStorePath(doi.path));
},
[&](DerivationOutputCAFixed dof) {
outputObj.attr("path", store->printStorePath(dof.path(*store, drv.name, output.first)));
outputObj.attr("hashAlgo", dof.hash.printMethodAlgo());
outputObj.attr("hash", dof.hash.hash.to_string(Base16, false));
},

19
tests/content-addressed.nix Normal file
View file

@ -0,0 +1,19 @@
with import ./config.nix;
{ seed ? 0 }:
# A simple content-addressed derivation.
# The derivation can be arbitrarily modified by passing a different `seed`,
# but the output will always be the same
mkDerivation {
name = "simple-content-addressed";
buildCommand = ''
set -x
echo "Building a CA derivation"
echo "The seed is ${toString seed}"
mkdir -p $out
echo "Hello World" > $out/hello
'';
__contentAddressed = true;
outputHashMode = "recursive";
outputHashAlgo = "sha256";
}

16
tests/content-addressed.sh Normal file
View file

@ -0,0 +1,16 @@
#!/usr/bin/env bash
source common.sh
clearStore
clearCache
export REMOTE_STORE=file://$cacheDir
drv=$(nix-instantiate --experimental-features ca-derivations ./content-addressed.nix --arg seed 1)
nix --experimental-features 'nix-command ca-derivations' show-derivation --derivation "$drv" --arg seed 1
out1=$(nix-build --experimental-features ca-derivations ./content-addressed.nix --arg seed 1 --no-out-link)
out2=$(nix-build --experimental-features ca-derivations ./content-addressed.nix --arg seed 2 --no-out-link)
test $out1 == $out2

3
tests/local.mk
View file

@ -32,7 +32,8 @@ nix_tests = \
post-hook.sh \
function-trace.sh \
recursive.sh \
flakes.sh
flakes.sh \
content-addressed.sh
# parallel.sh
install-tests += $(foreach x, $(nix_tests), tests/$(x))

15
tests/multiple-outputs.nix
View file

@ -2,6 +2,21 @@ with import ./config.nix;
rec {
# Want to ensure that "out" doesn't get a suffix on it's path.
nameCheck = mkDerivation {
name = "multiple-outputs-a";
outputs = [ "out" "dev" ];
builder = builtins.toFile "builder.sh"
''
mkdir $first $second
test -z $all
echo "first" > $first/file
echo "second" > $second/file
ln -s $first $second/link
'';
helloString = "Hello, world!";
};
a = mkDerivation {
name = "multiple-outputs-a";
outputs = [ "first" "second" ];

6
tests/multiple-outputs.sh
View file

@ -4,6 +4,12 @@ clearStore
rm -f $TEST_ROOT/result*
# Test whether the output names match our expectations
outPath=$(nix-instantiate multiple-outputs.nix --eval -A nameCheck.out.outPath)
[ "$(echo "$outPath" | sed -E 's_^".*/[^-/]*-([^/]*)"$_\1_')" = "multiple-outputs-a" ]
outPath=$(nix-instantiate multiple-outputs.nix --eval -A nameCheck.dev.outPath)
[ "$(echo "$outPath" | sed -E 's_^".*/[^-/]*-([^/]*)"$_\1_')" = "multiple-outputs-a-dev" ]
# Test whether read-only evaluation works when referring to the
# ‘drvPath’ attribute.
echo "evaluating c..."