Add function queryPathFromHashPart()

To implement binary caches efficiently, Hydra needs to be able to map
the hash part of a store path (e.g. "gbg...zr7") to the full store
path (e.g. "/nix/store/gbg...kzr7-subversion-1.7.5").  (The binary
cache mechanism uses hash parts as a key for looking up store paths to
ensure privacy.)  However, doing a search in the Nix store for
/nix/store/<hash>* is expensive since it requires reading the entire
directory.  queryPathFromHashPart() prevents this by doing a cheap
database lookup.
This commit is contained in:
Eelco Dolstra 2012-07-17 18:55:39 -04:00
parent 220818f758
commit ccc52adfb2
9 changed files with 67 additions and 0 deletions

View file

@ -14,6 +14,7 @@ our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } );
our @EXPORT = qw(
isValidPath queryReferences queryPathInfo queryDeriver queryPathHash
queryPathFromHashPart
topoSortPaths computeFSClosure followLinksToStorePath exportPaths
hashPath hashFile hashString
addToStore makeFixedOutputPath

View file

@ -108,6 +108,17 @@ SV * queryPathInfo(char * path, int base32)
}
SV * queryPathFromHashPart(char * hashPart)
PPCODE:
try {
doInit();
Path path = store->queryPathFromHashPart(hashPart);
XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0)));
} catch (Error & e) {
croak(e.what());
}
SV * computeFSClosure(int flipDirection, int includeOutputs, ...)
PPCODE:
try {

View file

@ -405,6 +405,10 @@ void LocalStore::openDB(bool create)
"select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;");
stmtQueryDerivationOutputs.create(db,
"select id, path from DerivationOutputs where drv = ?;");
// Use "path >= ?" with limit 1 rather than "path like '?%'" to
// ensure efficient lookup.
stmtQueryPathFromHashPart.create(db,
"select path from ValidPaths where path >= ? limit 1;");
}
@ -865,6 +869,26 @@ StringSet LocalStore::queryDerivationOutputNames(const Path & path)
}
Path LocalStore::queryPathFromHashPart(const string & hashPart)
{
if (hashPart.size() != 32) throw Error("invalid hash part");
SQLiteTxn txn(db);
Path prefix = nixStore + "/" + hashPart;
SQLiteStmtUse use(stmtQueryPathFromHashPart);
stmtQueryPathFromHashPart.bind(prefix);
int res = sqlite3_step(stmtQueryPathFromHashPart);
if (res == SQLITE_DONE) return "";
if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database");
const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0);
return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : "";
}
void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run)
{
if (run.pid != -1) return;

View file

@ -121,6 +121,8 @@ public:
StringSet queryDerivationOutputNames(const Path & path);
Path queryPathFromHashPart(const string & hashPart);
PathSet querySubstitutablePaths();
bool hasSubstitutes(const Path & path);
@ -217,6 +219,7 @@ private:
SQLiteStmt stmtAddDerivationOutput;
SQLiteStmt stmtQueryValidDerivers;
SQLiteStmt stmtQueryDerivationOutputs;
SQLiteStmt stmtQueryPathFromHashPart;
int getSchema();

View file

@ -341,6 +341,18 @@ PathSet RemoteStore::queryDerivationOutputNames(const Path & path)
}
Path RemoteStore::queryPathFromHashPart(const string & hashPart)
{
openConnection();
writeInt(wopQueryPathFromHashPart, to);
writeString(hashPart, to);
processStderr();
Path path = readString(from);
if (!path.empty()) assertStorePath(path);
return path;
}
Path RemoteStore::addToStore(const Path & _srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter)
{

View file

@ -43,6 +43,8 @@ public:
StringSet queryDerivationOutputNames(const Path & path);
Path queryPathFromHashPart(const string & hashPart);
bool hasSubstitutes(const Path & path);
bool querySubstitutablePathInfo(const Path & path,

View file

@ -139,6 +139,10 @@ public:
/* Query the output names of the derivation denoted by `path'. */
virtual StringSet queryDerivationOutputNames(const Path & path) = 0;
/* Query the full store path given the hash part of a valid store
path, or "" if the path doesn't exist. */
virtual Path queryPathFromHashPart(const string & hashPart) = 0;
/* Query whether a path has substitutes. */
virtual bool hasSubstitutes(const Path & path) = 0;

View file

@ -40,6 +40,7 @@ typedef enum {
wopQueryPathInfo = 26,
wopImportPaths = 27,
wopQueryDerivationOutputNames = 28,
wopQueryPathFromHashPart = 29,
} WorkerOp;

View file

@ -350,6 +350,15 @@ static void performOp(unsigned int clientVersion,
break;
}
case wopQueryPathFromHashPart: {
string hashPart = readString(from);
startWork();
Path path = store->queryPathFromHashPart(hashPart);
stopWork();
writeString(path, to);
break;
}
case wopAddToStore: {
string baseName = readString(from);
bool fixed = readInt(from) == 1; /* obsolete */