downloadCached: Return ETag

This allows fetchFlake() to return the Git revision of a GitHub
archive.
This commit is contained in:
Eelco Dolstra 2019-02-25 23:20:50 +08:00
parent 6e9182fbc2
commit 529add316c
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
7 changed files with 64 additions and 26 deletions

View file

@ -46,7 +46,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
Path lookupFileArg(EvalState & state, string s) Path lookupFileArg(EvalState & state, string s)
{ {
if (isUri(s)) if (isUri(s))
return getDownloader()->downloadCached(state.store, s, true); return getDownloader()->downloadCached(state.store, s, true).path;
else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
Path p = s.substr(1, s.size() - 2); Path p = s.substr(1, s.size() - 2);
return state.findFile(p); return state.findFile(p);

View file

@ -657,7 +657,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
if (isUri(elem.second)) { if (isUri(elem.second)) {
try { try {
res = { true, getDownloader()->downloadCached(store, elem.second, true) }; res = { true, getDownloader()->downloadCached(store, elem.second, true).path };
} catch (DownloadError & e) { } catch (DownloadError & e) {
printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second);
res = { false, "" }; res = { false, "" };

View file

@ -2112,7 +2112,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
if (evalSettings.pureEval && !expectedHash) if (evalSettings.pureEval && !expectedHash)
throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash); Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash).path;
if (state.allowedPaths) if (state.allowedPaths)
state.allowedPaths->insert(res); state.allowedPaths->insert(res);

View file

@ -87,7 +87,13 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef,
return flakeRef; return flakeRef;
} }
static Path fetchFlake(EvalState & state, const FlakeRef & flakeRef) struct FlakeSourceInfo
{
Path storePath;
std::optional<Hash> rev;
};
static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef)
{ {
assert(flakeRef.isDirect()); assert(flakeRef.isDirect());
@ -99,25 +105,36 @@ static Path fetchFlake(EvalState & state, const FlakeRef & flakeRef)
// FIXME: support passing auth tokens for private repos. // FIXME: support passing auth tokens for private repos.
auto storePath = getDownloader()->downloadCached(state.store, auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
fmt("https://api.github.com/repos/%s/%s/tarball/%s", refData->owner, refData->repo,
refData->owner, refData->repo, refData->rev
refData->rev ? refData->rev->to_string(Base16, false)
? refData->rev->to_string(Base16, false) : refData->ref
: refData->ref ? *refData->ref
? *refData->ref : "master");
: "master"),
true, "source");
// FIXME: extract revision hash from ETag. auto result = getDownloader()->downloadCached(state.store, url, true, "source");
return storePath; if (!result.etag)
throw Error("did not receive an ETag header from '%s'", url);
if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"')
throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url);
FlakeSourceInfo info;
info.storePath = result.path;
info.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1);
return info;
} }
else if (auto refData = std::get_if<FlakeRef::IsGit>(&flakeRef.data)) { else if (auto refData = std::get_if<FlakeRef::IsGit>(&flakeRef.data)) {
auto gitInfo = exportGit(state.store, refData->uri, refData->ref, auto gitInfo = exportGit(state.store, refData->uri, refData->ref,
refData->rev ? refData->rev->to_string(Base16, false) : "", "source"); refData->rev ? refData->rev->to_string(Base16, false) : "", "source");
return gitInfo.storePath; FlakeSourceInfo info;
info.storePath = gitInfo.storePath;
info.rev = Hash(gitInfo.rev, htSHA1);
return info;
} }
else abort(); else abort();
@ -138,7 +155,11 @@ struct Flake
static Flake getFlake(EvalState & state, const FlakeRef & flakeRef) static Flake getFlake(EvalState & state, const FlakeRef & flakeRef)
{ {
auto flakePath = fetchFlake(state, flakeRef); auto sourceInfo = fetchFlake(state, flakeRef);
debug("got flake source '%s' with revision %s",
sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false));
auto flakePath = sourceInfo.storePath;
state.store->assertStorePath(flakePath); state.store->assertStorePath(flakePath);
if (state.allowedPaths) if (state.allowedPaths)

View file

@ -771,7 +771,7 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
} }
} }
Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) CachedDownloadResult Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl)
{ {
auto url = resolveUri(url_); auto url = resolveUri(url_);
@ -783,8 +783,11 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
Path expectedStorePath; Path expectedStorePath;
if (expectedHash) { if (expectedHash) {
expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name); expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name);
if (store->isValidPath(expectedStorePath)) if (store->isValidPath(expectedStorePath)) {
return store->toRealPath(expectedStorePath); CachedDownloadResult result;
result.path = store->toRealPath(expectedStorePath);
return result;
}
} }
Path cacheDir = getCacheDir() + "/nix/tarballs"; Path cacheDir = getCacheDir() + "/nix/tarballs";
@ -803,6 +806,8 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
bool skip = false; bool skip = false;
CachedDownloadResult result;
if (pathExists(fileLink) && pathExists(dataFile)) { if (pathExists(fileLink) && pathExists(dataFile)) {
storePath = readLink(fileLink); storePath = readLink(fileLink);
store->addTempRoot(storePath); store->addTempRoot(storePath);
@ -814,6 +819,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
skip = true; skip = true;
if (effectiveUrl) if (effectiveUrl)
*effectiveUrl = url_; *effectiveUrl = url_;
result.etag = ss[1];
} else if (!ss[1].empty()) { } else if (!ss[1].empty()) {
debug(format("verifying previous ETag '%1%'") % ss[1]); debug(format("verifying previous ETag '%1%'") % ss[1]);
expectedETag = ss[1]; expectedETag = ss[1];
@ -831,6 +837,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
auto res = download(request); auto res = download(request);
if (effectiveUrl) if (effectiveUrl)
*effectiveUrl = res.effectiveUrl; *effectiveUrl = res.effectiveUrl;
result.etag = res.etag;
if (!res.cached) { if (!res.cached) {
ValidPathInfo info; ValidPathInfo info;
@ -852,6 +859,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
} catch (DownloadError & e) { } catch (DownloadError & e) {
if (storePath.empty()) throw; if (storePath.empty()) throw;
printError(format("warning: %1%; using cached result") % e.msg()); printError(format("warning: %1%; using cached result") % e.msg());
result.etag = expectedETag;
} }
} }
@ -885,7 +893,8 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
url, expectedHash.to_string(), gotHash.to_string()); url, expectedHash.to_string(), gotHash.to_string());
} }
return store->toRealPath(storePath); result.path = store->toRealPath(storePath);
return result;
} }

View file

@ -41,6 +41,12 @@ struct DownloadResult
uint64_t bodySize = 0; uint64_t bodySize = 0;
}; };
struct CachedDownloadResult
{
Path path;
std::optional<std::string> etag;
};
class Store; class Store;
struct Downloader struct Downloader
@ -64,8 +70,10 @@ struct Downloader
and is more recent than tarball-ttl seconds. Otherwise, and is more recent than tarball-ttl seconds. Otherwise,
use the recorded ETag to verify if the server has a more use the recorded ETag to verify if the server has a more
recent version, and if so, download it to the Nix store. */ recent version, and if so, download it to the Nix store. */
Path downloadCached(ref<Store> store, const string & uri, bool unpack, string name = "", CachedDownloadResult downloadCached(
const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, int ttl = settings.tarballTtl); ref<Store> store, const string & uri, bool unpack, string name = "",
const Hash & expectedHash = Hash(), string * effectiveUri = nullptr,
int ttl = settings.tarballTtl);
enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
}; };

View file

@ -88,7 +88,7 @@ static void update(const StringSet & channelNames)
// definition from a consistent location if the redirect changes mid-download. // definition from a consistent location if the redirect changes mid-download.
std::string effectiveUrl; std::string effectiveUrl;
auto dl = getDownloader(); auto dl = getDownloader();
auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0); auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0).path;
url = chomp(std::move(effectiveUrl)); url = chomp(std::move(effectiveUrl));
// If the URL contains a version number, append it to the name // If the URL contains a version number, append it to the name
@ -123,10 +123,10 @@ static void update(const StringSet & channelNames)
// Download the channel tarball. // Download the channel tarball.
auto fullURL = url + "/nixexprs.tar.xz"; auto fullURL = url + "/nixexprs.tar.xz";
try { try {
filename = dl->downloadCached(store, fullURL, false); filename = dl->downloadCached(store, fullURL, false).path;
} catch (DownloadError & e) { } catch (DownloadError & e) {
fullURL = url + "/nixexprs.tar.bz2"; fullURL = url + "/nixexprs.tar.bz2";
filename = dl->downloadCached(store, fullURL, false); filename = dl->downloadCached(store, fullURL, false).path;
} }
chomp(filename); chomp(filename);
} }