* addToStore() in nix-worker: don't write the NAR dump received from

the client to a temporary directory, as that is highly inefficient.
This commit is contained in:
Eelco Dolstra 2008-12-03 18:05:14 +00:00
parent 5eaf644c99
commit 82ae85de27
3 changed files with 85 additions and 25 deletions

View file

@ -655,24 +655,12 @@ void LocalStore::invalidatePath(const Path & path)
}
Path LocalStore::addToStore(const Path & _srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter)
Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
bool recursive, HashType hashAlgo)
{
Path srcPath(absPath(_srcPath));
debug(format("adding `%1%' to the store") % srcPath);
Hash h = hashString(hashAlgo, dump);
/* Read the whole path into memory. This is not a very scalable
method for very large paths, but `copyPath' is mainly used for
small files. */
StringSink sink;
if (recursive)
dumpPath(srcPath, sink, filter);
else
sink.s = readFile(srcPath);
Hash h = hashString(hashAlgo, sink.s);
Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, baseNameOf(srcPath));
Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name);
addTempRoot(dstPath);
@ -688,10 +676,10 @@ Path LocalStore::addToStore(const Path & _srcPath,
if (pathExists(dstPath)) deletePathWrapped(dstPath);
if (recursive) {
StringSource source(sink.s);
StringSource source(dump);
restorePath(dstPath, source);
} else
writeStringToFile(dstPath, sink.s);
writeStringToFile(dstPath, dump);
canonicalisePathMetaData(dstPath);
@ -701,7 +689,7 @@ Path LocalStore::addToStore(const Path & _srcPath,
sha256); otherwise, compute it here. */
registerValidPath(dstPath,
(recursive && hashAlgo == htSHA256) ? h :
(recursive ? hashString(htSHA256, sink.s) : hashPath(htSHA256, dstPath)),
(recursive ? hashString(htSHA256, dump) : hashPath(htSHA256, dstPath)),
PathSet(), "");
}
@ -712,6 +700,25 @@ Path LocalStore::addToStore(const Path & _srcPath,
}
Path LocalStore::addToStore(const Path & _srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter)
{
Path srcPath(absPath(_srcPath));
debug(format("adding `%1%' to the store") % srcPath);
/* Read the whole path into memory. This is not a very scalable
method for very large paths, but `copyPath' is mainly used for
small files. */
StringSink sink;
if (recursive)
dumpPath(srcPath, sink, filter);
else
sink.s = readFile(srcPath);
return addToStoreFromDump(sink.s, baseNameOf(srcPath), recursive, hashAlgo);
}
Path LocalStore::addTextToStore(const string & name, const string & s,
const PathSet & references)
{

View file

@ -93,6 +93,13 @@ public:
bool recursive = true, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter);
/* Like addToStore(), but the contents of the path are contained
in `dump', which is either a NAR serialisation (if recursive ==
true) or simply the contents of a regular file (if recursive ==
false). */
Path addToStoreFromDump(const string & dump, const string & name,
bool recursive = true, HashType hashAlgo = htSHA256);
Path addTextToStore(const string & name, const string & s,
const PathSet & references);

View file

@ -223,6 +223,43 @@ struct TunnelSource : Source
};
/* If the NAR archive contains a single file at top-level, then save
the contents of the file to `s'. Otherwise barf. */
struct RetrieveRegularNARSink : ParseSink
{
string s;
void createDirectory(const Path & path)
{
throw Error("regular file expected");
}
void receiveContents(unsigned char * data, unsigned int len)
{
s.append((const char *) data, len);
}
void createSymlink(const Path & path, const string & target)
{
throw Error("regular file expected");
}
};
/* Adapter class of a Source that saves all data read to `s'. */
struct SavingSourceAdapter : Source
{
Source & orig;
string s;
SavingSourceAdapter(Source & orig) : orig(orig) { }
void operator () (unsigned char * data, unsigned int len)
{
orig(data, len);
s.append((const char *) data, len);
}
};
static void performOp(unsigned int clientVersion,
Source & from, Sink & to, unsigned int op)
{
@ -299,13 +336,22 @@ static void performOp(unsigned int clientVersion,
}
HashType hashAlgo = parseHashType(s);
Path tmp = createTempDir();
AutoDelete delTmp(tmp);
Path tmp2 = tmp + "/" + baseName;
restorePath(tmp2, from);
SavingSourceAdapter savedNAR(from);
RetrieveRegularNARSink savedRegular;
if (recursive) {
/* Get the entire NAR dump from the client and save it to
a string so that we can pass it to
addToStoreFromDump(). */
ParseSink sink; /* null sink; just parse the NAR */
parseDump(sink, savedNAR);
} else {
parseDump(savedRegular, from);
}
startWork();
Path path = store->addToStore(tmp2, recursive, hashAlgo);
Path path = dynamic_cast<LocalStore *>(store.get())
->addToStoreFromDump(recursive ? savedNAR.s : savedRegular.s, baseName, recursive, hashAlgo);
stopWork();
writeString(path, to);