* NAR archives: handle files larger than 2^32 bytes. Previously it

would just silently store only (fileSize % 2^32) bytes.
* Use posix_fallocate if available when unpacking archives.
* Provide a better error message when trying to unpack something that
  isn't a NAR archive.
This commit is contained in:
Eelco Dolstra 2009-03-22 17:36:43 +00:00
parent 7e05b8b75e
commit 77d272623f
7 changed files with 48 additions and 17 deletions

View file

@ -266,6 +266,7 @@ AC_CHECK_FUNCS([setresuid setreuid lchown])
# Nice to have, but not essential. # Nice to have, but not essential.
AC_CHECK_FUNCS([strsignal]) AC_CHECK_FUNCS([strsignal])
AC_CHECK_FUNCS([posix_fallocate])
# This is needed if ATerm, Berkeley DB or bzip2 are static libraries, # This is needed if ATerm, Berkeley DB or bzip2 are static libraries,

View file

@ -865,7 +865,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
store path follows the archive data proper), and besides, we store path follows the archive data proper), and besides, we
don't know yet whether the signature is valid. */ don't know yet whether the signature is valid. */
Path tmpDir = createTempDir(nixStore); Path tmpDir = createTempDir(nixStore);
AutoDelete delTmp(tmpDir); AutoDelete delTmp(tmpDir); /* !!! could be GC'ed! */
Path unpacked = tmpDir + "/unpacked"; Path unpacked = tmpDir + "/unpacked";
restorePath(unpacked, hashAndReadSource); restorePath(unpacked, hashAndReadSource);

View file

@ -2,6 +2,7 @@
#include <algorithm> #include <algorithm>
#include <vector> #include <vector>
#define _XOPEN_SOURCE 600
#include <sys/types.h> #include <sys/types.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <unistd.h> #include <unistd.h>
@ -11,6 +12,8 @@
#include "archive.hh" #include "archive.hh"
#include "util.hh" #include "util.hh"
#include "config.h"
namespace nix { namespace nix {
@ -47,17 +50,17 @@ static void dumpEntries(const Path & path, Sink & sink, PathFilter & filter)
} }
static void dumpContents(const Path & path, unsigned int size, static void dumpContents(const Path & path, off_t size,
Sink & sink) Sink & sink)
{ {
writeString("contents", sink); writeString("contents", sink);
writeInt(size, sink); writeLongLong(size, sink);
AutoCloseFD fd = open(path.c_str(), O_RDONLY); AutoCloseFD fd = open(path.c_str(), O_RDONLY);
if (fd == -1) throw SysError(format("opening file `%1%'") % path); if (fd == -1) throw SysError(format("opening file `%1%'") % path);
unsigned char buf[65536]; unsigned char buf[65536];
unsigned int left = size; off_t left = size;
while (left > 0) { while (left > 0) {
size_t n = left > sizeof(buf) ? sizeof(buf) : left; size_t n = left > sizeof(buf) ? sizeof(buf) : left;
@ -101,7 +104,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
writeString(readLink(path), sink); writeString(readLink(path), sink);
} }
else throw Error("unknown file type: " + path); else throw Error(format("file `%1%' has an unknown type") % path);
writeString(")", sink); writeString(")", sink);
} }
@ -114,9 +117,9 @@ void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
} }
static Error badArchive(string s) static SerialisationError badArchive(string s)
{ {
return Error("bad archive: " + s); return SerialisationError("bad archive: " + s);
} }
@ -162,14 +165,17 @@ static void parseEntry(ParseSink & sink, Source & source, const Path & path)
static void parseContents(ParseSink & sink, Source & source, const Path & path) static void parseContents(ParseSink & sink, Source & source, const Path & path)
{ {
unsigned int size = readInt(source); unsigned long long size = readLongLong(source);
unsigned int left = size;
sink.preallocateContents(size);
unsigned long long left = size;
unsigned char buf[65536]; unsigned char buf[65536];
while (left) { while (left) {
checkInterrupt(); checkInterrupt();
unsigned int n = sizeof(buf); unsigned int n = sizeof(buf);
if (n > left) n = left; if ((unsigned long long) n > left) n = left;
source(buf, n); source(buf, n);
sink.receiveContents(buf, n); sink.receiveContents(buf, n);
left -= n; left -= n;
@ -248,8 +254,15 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
void parseDump(ParseSink & sink, Source & source) void parseDump(ParseSink & sink, Source & source)
{ {
if (readString(source) != archiveVersion1) string version;
throw badArchive("expected Nix archive"); try {
version = readString(source);
} catch (SerialisationError & e) {
/* This generally means the integer at the start couldn't be
decoded. Ignore and throw the exception below. */
}
if (version != archiveVersion1)
throw badArchive("input doesn't look like a Nix archive");
parse(sink, source, ""); parse(sink, source, "");
} }
@ -282,6 +295,16 @@ struct RestoreSink : ParseSink
throw SysError("fchmod"); throw SysError("fchmod");
} }
void preallocateContents(unsigned long long len)
{
#if HAVE_POSIX_FALLOCATE
if (len) {
errno = posix_fallocate(fd, 0, len);
if (errno) throw SysError(format("preallocating file of %1% bytes") % len);
}
#endif
}
void receiveContents(unsigned char * data, unsigned int len) void receiveContents(unsigned char * data, unsigned int len)
{ {
writeFull(fd, data, len); writeFull(fd, data, len);

View file

@ -62,6 +62,7 @@ struct ParseSink
virtual void createRegularFile(const Path & path) { }; virtual void createRegularFile(const Path & path) { };
virtual void isExecutable() { }; virtual void isExecutable() { };
virtual void preallocateContents(unsigned long long size) { };
virtual void receiveContents(unsigned char * data, unsigned int len) { }; virtual void receiveContents(unsigned char * data, unsigned int len) { };
virtual void createSymlink(const Path & path, const string & target) { }; virtual void createSymlink(const Path & path, const string & target) { };

View file

@ -80,7 +80,7 @@ void readPadding(unsigned int len, Source & source)
unsigned int n = 8 - (len % 8); unsigned int n = 8 - (len % 8);
source(zero, n); source(zero, n);
for (unsigned int i = 0; i < n; i++) for (unsigned int i = 0; i < n; i++)
if (zero[i]) throw Error("non-zero padding"); if (zero[i]) throw SerialisationError("non-zero padding");
} }
} }
@ -90,7 +90,7 @@ unsigned int readInt(Source & source)
unsigned char buf[8]; unsigned char buf[8];
source(buf, sizeof(buf)); source(buf, sizeof(buf));
if (buf[4] || buf[5] || buf[6] || buf[7]) if (buf[4] || buf[5] || buf[6] || buf[7])
throw Error("implementation cannot deal with > 32-bit integers"); throw SerialisationError("implementation cannot deal with > 32-bit integers");
return return
buf[0] | buf[0] |
(buf[1] << 8) | (buf[1] << 8) |

View file

@ -106,6 +106,9 @@ string readString(Source & source);
StringSet readStringSet(Source & source); StringSet readStringSet(Source & source);
MakeError(SerialisationError, Error)
} }

View file

@ -617,16 +617,19 @@ static void opExport(Strings opFlags, Strings opArgs)
static void opImport(Strings opFlags, Strings opArgs) static void opImport(Strings opFlags, Strings opArgs)
{ {
bool requireSignature = false; bool requireSignature = false;
for (Strings::iterator i = opFlags.begin(); foreach (Strings::iterator, i, opFlags)
i != opFlags.end(); ++i)
if (*i == "--require-signature") requireSignature = true; if (*i == "--require-signature") requireSignature = true;
else throw UsageError(format("unknown flag `%1%'") % *i); else throw UsageError(format("unknown flag `%1%'") % *i);
if (!opArgs.empty()) throw UsageError("no arguments expected"); if (!opArgs.empty()) throw UsageError("no arguments expected");
FdSource source(STDIN_FILENO); FdSource source(STDIN_FILENO);
while (readInt(source) == 1) while (true) {
int n = readInt(source);
if (n == 0) break;
if (n != 1) throw Error("input doesn't look like something created by `nix-store --export'");
cout << format("%1%\n") % store->importPath(requireSignature, source) << std::flush; cout << format("%1%\n") % store->importPath(requireSignature, source) << std::flush;
}
} }