diff --git a/Makefile.config.in b/Makefile.config.in index 7e3b35b98..fe609ce06 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -18,6 +18,7 @@ SODIUM_LIBS = @SODIUM_LIBS@ LIBLZMA_LIBS = @LIBLZMA_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ +LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ EDITLINE_LIBS = @EDITLINE_LIBS@ bash = @bash@ bindir = @bindir@ diff --git a/configure.ac b/configure.ac index 99248b90c..b868343d2 100644 --- a/configure.ac +++ b/configure.ac @@ -178,6 +178,8 @@ AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true], [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) AC_CHECK_HEADERS([bzlib.h], [true], [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) +# Checks for libarchive +PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"]) # Look for SQLite, a required dependency. PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"]) diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock index 6e5ecf45d..957c01e5a 100644 --- a/nix-rust/Cargo.lock +++ b/nix-rust/Cargo.lock @@ -54,17 +54,6 @@ dependencies = [ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "filetime" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "fnv" version = "1.0.6" @@ -109,7 +98,6 @@ dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)", - "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -319,17 +307,6 @@ dependencies = [ "wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "tar" -version = "0.4.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", - "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "tempfile" version = "3.1.0" @@ -375,14 +352,6 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "xattr" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", -] - [metadata] "checksum assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7deb0a829ca7bcfaf5da70b073a8d128619259a7be8216a355e23f00763059e5" "checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" @@ -393,7 +362,6 @@ dependencies = [ "checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb" "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -"checksum filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1ff6d4dab0aa0c8e6346d46052e93b13a16cf847b54ed357087c35011048cc7d" "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" "checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" "checksum getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407" @@ -423,11 +391,9 @@ dependencies = [ "checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716" "checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" "checksum rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3dd93264e10c577503e926bd1430193eeb5d21b059148910082245309b424fae" -"checksum tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)" = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3" "checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" "checksum wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" "checksum wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d" "checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -"checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml index cd388bed6..1372e5a73 100644 --- a/nix-rust/Cargo.toml +++ b/nix-rust/Cargo.toml @@ -9,7 +9,6 @@ name = "nixrust" crate-type = ["cdylib"] [dependencies] -tar = "0.4" libc = "0.2" #futures-preview = { version = "=0.3.0-alpha.19" } #hyper = "0.13.0-alpha.4" diff --git a/nix-rust/src/c.rs b/nix-rust/src/c.rs index 8d2507d37..c1358545f 100644 --- a/nix-rust/src/c.rs +++ b/nix-rust/src/c.rs @@ -1,22 +1,4 @@ -use super::{ - error, - foreign::{self}, - store::path, - store::StorePath, - util, -}; - -#[no_mangle] -pub unsafe extern "C" fn unpack_tarfile( - source: foreign::Source, - dest_dir: &str, - out: *mut Result<(), error::CppException>, -) { - out.write( - util::tarfile::unpack_tarfile(source, std::path::Path::new(dest_dir)) - .map_err(|err| err.into()), - ); -} +use super::{error, store::path, store::StorePath, util}; #[no_mangle] pub unsafe extern "C" fn ffi_String_new(s: &str, out: *mut String) { diff --git a/nix-rust/src/foreign.rs b/nix-rust/src/foreign.rs deleted file mode 100644 index 7bce7753c..000000000 --- a/nix-rust/src/foreign.rs +++ /dev/null @@ -1,14 +0,0 @@ -/// A wrapper around Nix's Source class that provides the Read trait. -#[repr(C)] -pub struct Source { - fun: extern "C" fn(this: *mut libc::c_void, data: &mut [u8]) -> usize, - this: *mut libc::c_void, -} - -impl std::io::Read for Source { - fn read(&mut self, buf: &mut [u8]) -> std::result::Result { - let n = (self.fun)(self.this, buf); - assert!(n <= buf.len()); - Ok(n) - } -} diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index 9935cd27a..e62613ba8 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -12,7 +12,6 @@ extern crate proptest; #[cfg(not(test))] mod c; mod error; -mod foreign; #[cfg(unused)] mod nar; mod store; diff --git a/nix-rust/src/util/mod.rs b/nix-rust/src/util/mod.rs index cd852c55f..eaad9d406 100644 --- a/nix-rust/src/util/mod.rs +++ b/nix-rust/src/util/mod.rs @@ -1,2 +1 @@ pub mod base32; -pub mod tarfile; diff --git a/nix-rust/src/util/tarfile.rs b/nix-rust/src/util/tarfile.rs deleted file mode 100644 index 74d60692c..000000000 --- a/nix-rust/src/util/tarfile.rs +++ /dev/null @@ -1,56 +0,0 @@ -use crate::{foreign::Source, Error}; -use std::fs; -use std::io; -use std::os::unix::fs::OpenOptionsExt; -use std::path::{Component, Path}; -use tar::Archive; - -pub fn unpack_tarfile(source: Source, dest_dir: &Path) -> Result<(), Error> { - fs::create_dir_all(dest_dir)?; - - let mut tar = Archive::new(source); - - for file in tar.entries()? { - let mut file = file?; - - let path = file.path()?; - - for i in path.components() { - if let Component::Prefix(_) | Component::RootDir | Component::ParentDir = i { - return Err(Error::BadTarFileMemberName( - file.path()?.to_str().unwrap().to_string(), - )); - } - } - - let dest_file = dest_dir.join(path); - - fs::create_dir_all(dest_file.parent().unwrap())?; - - match file.header().entry_type() { - tar::EntryType::Directory => { - fs::create_dir(dest_file)?; - } - tar::EntryType::Regular => { - let mode = if file.header().mode()? & (libc::S_IXUSR as u32) == 0 { - 0o666 - } else { - 0o777 - }; - let mut f = fs::OpenOptions::new() - .create(true) - .write(true) - .mode(mode) - .open(dest_file)?; - io::copy(&mut file, &mut f)?; - } - tar::EntryType::Symlink => { - std::os::unix::fs::symlink(file.header().link_name()?.unwrap(), dest_file)?; - } - tar::EntryType::XGlobalHeader | tar::EntryType::XHeader => {} - t => return Err(Error::Misc(format!("unsupported tar entry type '{:?}'", t))), - } - } - - Ok(()) -} diff --git a/release-common.nix b/release-common.nix index 3765e61b3..60c8849d1 100644 --- a/release-common.nix +++ b/release-common.nix @@ -49,6 +49,7 @@ rec { [ curl bzip2 xz brotli zlib editline openssl pkgconfig sqlite boehmgc + libarchive boost nlohmann_json rustc cargo diff --git a/src/libstore/download.cc b/src/libstore/download.cc index af058ba42..0c231ca1d 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -908,7 +908,7 @@ CachedDownloadResult Downloader::downloadCached( printInfo("unpacking '%s'...", url); Path tmpDir = createTempDir(); AutoDelete autoDelete(tmpDir, true); - unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir, std::string(baseNameOf(url))); + unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir); auto members = readDirectory(tmpDir); if (members.size() != 1) throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 35c1f6c13..16c1fa03f 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,6 +6,6 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) -libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context +libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context libutil_LIBS = libnixrust diff --git a/src/libutil/rust-ffi.cc b/src/libutil/rust-ffi.cc index 8b8b7b75d..6f36b3192 100644 --- a/src/libutil/rust-ffi.cc +++ b/src/libutil/rust-ffi.cc @@ -19,15 +19,4 @@ std::ostream & operator << (std::ostream & str, const String & s) return str; } -size_t Source::sourceWrapper(void * _this, rust::Slice data) -{ - try { - // FIXME: how to propagate exceptions? - auto n = ((nix::Source *) _this)->read((unsigned char *) data.ptr, data.size); - return n; - } catch (...) { - abort(); - } -} - } diff --git a/src/libutil/rust-ffi.hh b/src/libutil/rust-ffi.hh index 3b51661c2..469a5fba3 100644 --- a/src/libutil/rust-ffi.hh +++ b/src/libutil/rust-ffi.hh @@ -131,18 +131,6 @@ struct String : Vec std::ostream & operator << (std::ostream & str, const String & s); -struct Source -{ - size_t (*fun)(void * source_this, rust::Slice data); - nix::Source * _this; - - Source(nix::Source & _this) - : fun(sourceWrapper), _this(&_this) - {} - - static size_t sourceWrapper(void * _this, rust::Slice data); -}; - /* C++ representation of Rust's Result. */ template struct Result diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 1be0ba24c..68e918891 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -1,38 +1,132 @@ -#include "rust-ffi.hh" -#include "compression.hh" +#include +#include -extern "C" { - rust::Result> * - unpack_tarfile(rust::Source source, rust::StringSlice dest_dir, rust::Result> & out); -} +#include "serialise.hh" namespace nix { +struct TarArchive { + struct archive *archive; + Source *source; + std::vector buffer; + + void check(int err, const char *reason = "Failed to extract archive (%s)") { + if (err == ARCHIVE_EOF) + throw EndOfFile("reached end of archive"); + else if (err != ARCHIVE_OK) + throw Error(reason, archive_error_string(this->archive)); + } + + TarArchive(Source& source) : buffer(4096) { + this->archive = archive_read_new(); + this->source = &source; + + archive_read_support_filter_all(archive); + archive_read_support_format_all(archive); + check(archive_read_open(archive, (void *)this, TarArchive::callback_open, TarArchive::callback_read, TarArchive::callback_close), "Failed to open archive (%s)"); + } + + TarArchive(const Path &path) { + this->archive = archive_read_new(); + + archive_read_support_filter_all(archive); + archive_read_support_format_all(archive); + check(archive_read_open_filename(archive, path.c_str(), 16384), "Failed to open archive (%s)"); + } + + // disable copy constructor + TarArchive(const TarArchive&) = delete; + + void close() { + check(archive_read_close(archive), "Failed to close archive (%s)"); + } + + ~TarArchive() { + if (this->archive) archive_read_free(this->archive); + } + +private: + static int callback_open(struct archive *, void *self) { + return ARCHIVE_OK; + } + + static ssize_t callback_read(struct archive *archive, void *_self, const void **buffer) { + TarArchive *self = (TarArchive *)_self; + *buffer = self->buffer.data(); + + try { + return self->source->read(self->buffer.data(), 4096); + } catch (EndOfFile &) { + return 0; + } catch (std::exception &err) { + archive_set_error(archive, EIO, "Source threw exception: %s", err.what()); + + return -1; + } + } + + static int callback_close(struct archive *, void *self) { + return ARCHIVE_OK; + } +}; + +struct PushD { + char * oldDir; + + PushD(const std::string &newDir) { + oldDir = getcwd(0, 0); + if (!oldDir) throw SysError("getcwd"); + int r = chdir(newDir.c_str()); + if (r != 0) throw SysError("changing directory to tar output path"); + } + + ~PushD() { + int r = chdir(oldDir); + free(oldDir); + if (r != 0) + std::cerr << "warning: failed to change directory back after tar extraction"; + /* can't throw out of a destructor */ + } +}; + +static void extract_archive(TarArchive &archive, const Path & destDir) { + // need to chdir back *after* archive closing + PushD newDir(destDir); + struct archive_entry *entry; + int flags = ARCHIVE_EXTRACT_FFLAGS + | ARCHIVE_EXTRACT_PERM + | ARCHIVE_EXTRACT_SECURE_SYMLINKS + | ARCHIVE_EXTRACT_SECURE_NODOTDOT + | ARCHIVE_EXTRACT_SECURE_NOABSOLUTEPATHS; + + for(;;) { + int r = archive_read_next_header(archive.archive, &entry); + if (r == ARCHIVE_EOF) break; + else if (r == ARCHIVE_WARN) + std::cerr << "warning: " << archive_error_string(archive.archive) << std::endl; + else + archive.check(r); + + archive.check(archive_read_extract(archive.archive, entry, flags)); + } + + archive.close(); +} + void unpackTarfile(Source & source, const Path & destDir) { - rust::Source source2(source); - rust::Result> res; - unpack_tarfile(source2, destDir, res); - res.unwrap(); + auto archive = TarArchive(source); + + createDirs(destDir); + extract_archive(archive, destDir); } -void unpackTarfile(const Path & tarFile, const Path & destDir, - std::optional baseName) +void unpackTarfile(const Path & tarFile, const Path & destDir) { - if (!baseName) baseName = std::string(baseNameOf(tarFile)); + auto archive = TarArchive(tarFile); - auto source = sinkToSource([&](Sink & sink) { - // FIXME: look at first few bytes to determine compression type. - auto decompressor = - hasSuffix(*baseName, ".bz2") ? makeDecompressionSink("bzip2", sink) : - hasSuffix(*baseName, ".gz") ? makeDecompressionSink("gzip", sink) : - hasSuffix(*baseName, ".xz") ? makeDecompressionSink("xz", sink) : - makeDecompressionSink("none", sink); - readFile(tarFile, *decompressor); - decompressor->finish(); - }); - - unpackTarfile(*source, destDir); + createDirs(destDir); + extract_archive(archive, destDir); } } diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index ce0911e2a..89a024f1d 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -4,7 +4,6 @@ namespace nix { void unpackTarfile(Source & source, const Path & destDir); -void unpackTarfile(const Path & tarFile, const Path & destDir, - std::optional baseName = {}); +void unpackTarfile(const Path & tarFile, const Path & destDir); } diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 5040b6fb8..2ec0b07ac 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -190,10 +190,7 @@ static int _main(int argc, char * * argv) printInfo("unpacking..."); Path unpacked = (Path) tmpDir + "/unpacked"; createDirs(unpacked); - if (hasSuffix(baseNameOf(uri), ".zip")) - runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked}); - else - unpackTarfile(tmpFile, unpacked, std::string(baseNameOf(uri))); + unpackTarfile(tmpFile, unpacked); /* If the archive unpacks to a single file/directory, then use that as the top-level. */