BinaryCacheStore: Add index-debug-info option

This integrates the functionality of the index-debuginfo program in
nixos-channel-scripts to maintain an index of DWARF debuginfo files in
a format usable by dwarffs. Thus the debug info index is updated by
Hydra rather than by the channel mirroring script.

Example usage:

  $ nix copy --to 'file:///tmp/binary-cache?index-debug-info=true' /nix/store/vr9mhcch3fljzzkjld3kvkggvpq38cva-nix-2.2.2-debug

  $ cat /tmp/binary-cache/debuginfo/036b210b03bad75ab2d8fc80b7a146f98e7f1ecf.debug
  {"archive":"../nar/0313h2kdhk4v73xna9ysiksp2v8xrsk5xsw79mmwr3rg7byb4ka8.nar.xz","member":"lib/debug/.build-id/03/6b210b03bad75ab2d8fc80b7a146f98e7f1ecf.debug"}

Fixes #3083.
This commit is contained in:
Eelco Dolstra 2019-09-04 19:24:35 +02:00
parent d20f814cde
commit 5dafde28db
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
3 changed files with 71 additions and 12 deletions

View file

@ -10,10 +10,13 @@
#include "nar-info-disk-cache.hh" #include "nar-info-disk-cache.hh"
#include "nar-accessor.hh" #include "nar-accessor.hh"
#include "json.hh" #include "json.hh"
#include "thread-pool.hh"
#include <chrono> #include <chrono>
#include <future> #include <future>
#include <regex>
#include <nlohmann/json.hpp>
namespace nix { namespace nix {
@ -139,6 +142,11 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor); auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
auto narAccessor = makeNarAccessor(nar);
if (accessor_)
accessor_->addToCache(info.path, *nar, narAccessor);
/* Optionally write a JSON file containing a listing of the /* Optionally write a JSON file containing a listing of the
contents of the NAR. */ contents of the NAR. */
if (writeNARListing) { if (writeNARListing) {
@ -148,11 +156,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
JSONObject jsonRoot(jsonOut); JSONObject jsonRoot(jsonOut);
jsonRoot.attr("version", 1); jsonRoot.attr("version", 1);
auto narAccessor = makeNarAccessor(nar);
if (accessor_)
accessor_->addToCache(info.path, *nar, narAccessor);
{ {
auto res = jsonRoot.placeholder("root"); auto res = jsonRoot.placeholder("root");
listNar(res, narAccessor, "", true); listNar(res, narAccessor, "", true);
@ -162,11 +165,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json"); upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json");
} }
else {
if (accessor_)
accessor_->addToCache(info.path, *nar, makeNarAccessor(nar));
}
/* Compress the NAR. */ /* Compress the NAR. */
narInfo->compression = compression; narInfo->compression = compression;
auto now1 = std::chrono::steady_clock::now(); auto now1 = std::chrono::steady_clock::now();
@ -181,12 +179,70 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
% ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0) % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0)
% duration); % duration);
/* Atomically write the NAR file. */
narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" : + (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" : compression == "bzip2" ? ".bz2" :
compression == "br" ? ".br" : compression == "br" ? ".br" :
""); "");
/* Optionally maintain an index of DWARF debug info files
consisting of JSON files named 'debuginfo/<build-id>' that
specify the NAR file and member containing the debug info. */
if (writeDebugInfo) {
std::string buildIdDir = "/lib/debug/.build-id";
if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) {
ThreadPool threadPool(25);
auto doFile = [&](std::string member, std::string key, std::string target) {
checkInterrupt();
nlohmann::json json;
json["archive"] = target;
json["member"] = member;
// FIXME: or should we overwrite? The previous link may point
// to a GC'ed file, so overwriting might be useful...
if (fileExists(key)) return;
printMsg(lvlTalkative, "creating debuginfo link from '%s' to '%s'", key, target);
upsertFile(key, json.dump(), "application/json");
};
std::regex regex1("^[0-9a-f]{2}$");
std::regex regex2("^[0-9a-f]{38}\\.debug$");
for (auto & s1 : narAccessor->readDirectory(buildIdDir)) {
auto dir = buildIdDir + "/" + s1;
if (narAccessor->stat(dir).type != FSAccessor::tDirectory
|| !std::regex_match(s1, regex1))
continue;
for (auto & s2 : narAccessor->readDirectory(dir)) {
auto debugPath = dir + "/" + s2;
if (narAccessor->stat(debugPath).type != FSAccessor::tRegular
|| !std::regex_match(s2, regex2))
continue;
auto buildId = s1 + s2;
std::string key = "debuginfo/" + buildId;
std::string target = "../" + narInfo->url;
threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target));
}
}
threadPool.process();
}
}
/* Atomically write the NAR file. */
if (repair || !fileExists(narInfo->url)) { if (repair || !fileExists(narInfo->url)) {
stats.narWrite++; stats.narWrite++;
upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");

View file

@ -17,6 +17,7 @@ public:
const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
const Setting<bool> writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
const Setting<bool> parallelCompression{this, false, "parallel-compression", const Setting<bool> parallelCompression{this, false, "parallel-compression",

View file

@ -63,6 +63,8 @@ protected:
void LocalBinaryCacheStore::init() void LocalBinaryCacheStore::init()
{ {
createDirs(binaryCacheDir + "/nar"); createDirs(binaryCacheDir + "/nar");
if (writeDebugInfo)
createDirs(binaryCacheDir + "/debuginfo");
BinaryCacheStore::init(); BinaryCacheStore::init();
} }