From 11da5b2816e11b081d8ff38db4330addd2014f7e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Mar 2019 14:12:20 +0100 Subject: [PATCH 01/23] Add some Rust code --- .gitignore | 2 ++ Makefile | 1 + nix-rust/Cargo.lock | 59 +++++++++++++++++++++++++++++++++++++++++++ nix-rust/Cargo.toml | 12 +++++++++ nix-rust/local.mk | 7 ++++++ nix-rust/src/lib.rs | 48 +++++++++++++++++++++++++++++++++++ release-common.nix | 1 + shell.nix | 2 +- src/nix/local.mk | 4 +-- src/nix/test.cc | 61 +++++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 194 insertions(+), 3 deletions(-) create mode 100644 nix-rust/Cargo.lock create mode 100644 nix-rust/Cargo.toml create mode 100644 nix-rust/local.mk create mode 100644 nix-rust/src/lib.rs create mode 100644 src/nix/test.cc diff --git a/.gitignore b/.gitignore index fd62dfb38..f58e1f90f 100644 --- a/.gitignore +++ b/.gitignore @@ -119,3 +119,5 @@ GPATH GRTAGS GSYMS GTAGS + +nix-rust/target diff --git a/Makefile b/Makefile index 866c0961e..469070533 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ makefiles = \ mk/precompiled-headers.mk \ local.mk \ + nix-rust/local.mk \ src/libutil/local.mk \ src/libstore/local.mk \ src/libmain/local.mk \ diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock new file mode 100644 index 000000000..cfaedc3ed --- /dev/null +++ b/nix-rust/Cargo.lock @@ -0,0 +1,59 @@ +[[package]] +name = "cfg-if" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "filetime" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "libc" +version = "0.2.50" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "nix-rust" +version = "0.1.0" +dependencies = [ + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "tar 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "redox_syscall" +version = "0.1.51" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "tar" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)", + "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "xattr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[metadata] +"checksum cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "11d43355396e872eefb45ce6342e4374ed7bc2b3a502d1b28e36d6e23c05d1f4" +"checksum filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a2df5c1a8c4be27e7707789dc42ae65976e60b394afd293d1419ab915833e646" +"checksum libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)" = "aab692d7759f5cd8c859e169db98ae5b52c924add2af5fbbca11d12fefb567c1" +"checksum redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)" = "423e376fffca3dfa06c9e9790a9ccd282fafb3cc6e6397d01dbf64f9bacc6b85" +"checksum tar 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)" = "c2167ff53da2a661702b3299f71a91b61b1dffef36b4b2884b1f9c67254c0133" +"checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml new file mode 100644 index 000000000..2d2bf6752 --- /dev/null +++ b/nix-rust/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "nix-rust" +version = "0.1.0" +authors = ["Eelco Dolstra "] + +[lib] +name = "nixrust" +crate-type = ["staticlib"] + +[dependencies] +tar = "0.4" +libc = "0.2" diff --git a/nix-rust/local.mk b/nix-rust/local.mk new file mode 100644 index 000000000..c69e3b9ea --- /dev/null +++ b/nix-rust/local.mk @@ -0,0 +1,7 @@ +libnixrust_PATH := $(d)/target/release/libnixrust.a +libnixrust_INSTALL_PATH := $(libnixrust_PATH) +libnixrust_LDFLAGS_USE := -L$(d)/target/release -lnixrust -ldl +libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE) + +$(d)/target/release/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml + $(trace-gen) cd nix-rust && RUSTC_BOOTSTRAP=1 cargo build --release && touch target/release/libnixrust.a diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs new file mode 100644 index 000000000..abcb89055 --- /dev/null +++ b/nix-rust/src/lib.rs @@ -0,0 +1,48 @@ +extern crate libc; +extern crate tar; + +use std::fs; +use std::io; +use std::os::unix::fs::OpenOptionsExt; +use std::path::Path; +use tar::Archive; + +#[no_mangle] +pub extern "C" fn unpack_tarfile(data: &[u8], dest_dir: &str) -> bool { + // FIXME: handle errors. + + let dest_dir = Path::new(dest_dir); + + let mut tar = Archive::new(data); + + for file in tar.entries().unwrap() { + let mut file = file.unwrap(); + + let dest_file = dest_dir.join(file.header().path().unwrap()); + + fs::create_dir_all(dest_file.parent().unwrap()).unwrap(); + + match file.header().entry_type() { + tar::EntryType::Directory => { + fs::create_dir(dest_file).unwrap(); + } + tar::EntryType::Regular => { + let mode = if file.header().mode().unwrap() & libc::S_IXUSR == 0 { + 0o666 + } else { + 0o777 + }; + let mut f = fs::OpenOptions::new() + .create(true) + .write(true) + .mode(mode) + .open(dest_file) + .unwrap(); + io::copy(&mut file, &mut f).unwrap(); + } + t => panic!("Unsupported tar entry type '{:?}'.", t), + } + } + + true +} diff --git a/release-common.nix b/release-common.nix index 63f39f005..dd5f939d9 100644 --- a/release-common.nix +++ b/release-common.nix @@ -51,6 +51,7 @@ rec { openssl pkgconfig sqlite boehmgc boost nlohmann_json + rustc cargo # Tests git diff --git a/shell.nix b/shell.nix index 9c504f024..21ed47121 100644 --- a/shell.nix +++ b/shell.nix @@ -7,7 +7,7 @@ with import ./release-common.nix { inherit pkgs; }; (if useClang then clangStdenv else stdenv).mkDerivation { name = "nix"; - buildInputs = buildDeps ++ tarballDeps ++ perlDeps; + buildInputs = buildDeps ++ tarballDeps ++ perlDeps ++ [ pkgs.rustfmt ]; inherit configureFlags; diff --git a/src/nix/local.mk b/src/nix/local.mk index c09efd1fc..a145cf9b1 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -15,9 +15,9 @@ nix_SOURCES := \ $(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-store/*.cc) \ -nix_LIBS = libexpr libmain libstore libutil +nix_LIBS = libexpr libmain libstore libutil libnixrust -nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system +nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system -Lnix-rust/target/release $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ diff --git a/src/nix/test.cc b/src/nix/test.cc new file mode 100644 index 000000000..70e0a903d --- /dev/null +++ b/src/nix/test.cc @@ -0,0 +1,61 @@ +#include "command.hh" +#include "store-api.hh" +#include "common-args.hh" + +using namespace nix; + +namespace rust { + +// Depending on the internal representation of Rust slices is slightly +// evil... +template struct Slice +{ + const T * ptr; + size_t size; + + Slice(const T * ptr, size_t size) : ptr(ptr), size(size) + { + assert(ptr); + } +}; + +struct StringSlice : Slice +{ + StringSlice(const std::string & s): Slice(s.data(), s.size()) { } +}; + +} + +extern "C" { + bool unpack_tarfile(rust::Slice data, rust::StringSlice dest_dir); +} + +struct CmdTest : StoreCommand +{ + CmdTest() + { + } + + std::string name() override + { + return "test"; + } + + std::string description() override + { + return "bla bla"; + } + + void run(ref store) override + { + auto data = readFile("./nix-2.2.tar"); + + std::string destDir = "./dest"; + + deletePath(destDir); + + unpack_tarfile({(uint8_t*) data.data(), data.size()}, destDir); + } +}; + +static RegisterCommand r(make_ref()); From e60f6bd4ce831ced94fafeb527c429b6f88159ac Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Mar 2019 20:45:56 +0100 Subject: [PATCH 02/23] Enable Rust code to call C++ Source objects --- nix-rust/src/lib.rs | 19 +++++++++++++++++-- src/nix/test.cc | 34 ++++++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index abcb89055..799d52e31 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -7,13 +7,28 @@ use std::os::unix::fs::OpenOptionsExt; use std::path::Path; use tar::Archive; +/// A wrapper around Nix's Source class that provides the Read trait. +#[repr(C)] +pub struct Source { + fun: extern "C" fn(this: *mut libc::c_void, data: &mut [u8]) -> usize, + this: *mut libc::c_void, +} + +impl std::io::Read for Source { + fn read(&mut self, buf: &mut [u8]) -> std::result::Result { + let n = (self.fun)(self.this, buf); + assert!(n <= buf.len()); + Ok(n) + } +} + #[no_mangle] -pub extern "C" fn unpack_tarfile(data: &[u8], dest_dir: &str) -> bool { +pub extern "C" fn unpack_tarfile(source: Source, dest_dir: &str) -> bool { // FIXME: handle errors. let dest_dir = Path::new(dest_dir); - let mut tar = Archive::new(data); + let mut tar = Archive::new(source); for file in tar.entries().unwrap() { let mut file = file.unwrap(); diff --git a/src/nix/test.cc b/src/nix/test.cc index 70e0a903d..7f1f0d47a 100644 --- a/src/nix/test.cc +++ b/src/nix/test.cc @@ -1,6 +1,7 @@ #include "command.hh" #include "store-api.hh" #include "common-args.hh" +#include "compression.hh" using namespace nix; @@ -10,10 +11,10 @@ namespace rust { // evil... template struct Slice { - const T * ptr; + T * ptr; size_t size; - Slice(const T * ptr, size_t size) : ptr(ptr), size(size) + Slice(T * ptr, size_t size) : ptr(ptr), size(size) { assert(ptr); } @@ -21,13 +22,30 @@ template struct Slice struct StringSlice : Slice { - StringSlice(const std::string & s): Slice(s.data(), s.size()) { } + StringSlice(const std::string & s): Slice((char *) s.data(), s.size()) {} +}; + +struct Source +{ + size_t (*fun)(void * source_this, rust::Slice data); + nix::Source * _this; + + Source(nix::Source & _this) + : fun(sourceWrapper), _this(&_this) + {} + + // FIXME: how to propagate exceptions? + static size_t sourceWrapper(void * _this, rust::Slice data) + { + auto n = ((nix::Source *) _this)->read(data.ptr, data.size); + return n; + } }; } extern "C" { - bool unpack_tarfile(rust::Slice data, rust::StringSlice dest_dir); + bool unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); } struct CmdTest : StoreCommand @@ -48,13 +66,17 @@ struct CmdTest : StoreCommand void run(ref store) override { - auto data = readFile("./nix-2.2.tar"); + auto source = sinkToSource([&](Sink & sink) { + auto decompressor = makeDecompressionSink("bzip2", sink); + readFile("./nix-2.2.tar.bz2", *decompressor); + decompressor->finish(); + }); std::string destDir = "./dest"; deletePath(destDir); - unpack_tarfile({(uint8_t*) data.data(), data.size()}, destDir); + unpack_tarfile(*source, destDir); } }; From 045708db4343174f30f3647776971c852f72a9e8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Mar 2019 23:40:35 +0100 Subject: [PATCH 03/23] Make a builtin builder This was the last function using a shell script, so this allows us to get rid of tar, coreutils, bash etc. --- corepkgs/config.nix.in | 10 +----- corepkgs/unpack-channel.nix | 35 +++------------------ nix-rust/src/lib.rs | 5 ++- src/libstore/build.cc | 2 ++ src/libstore/builtins.hh | 1 + src/libstore/builtins/unpack-channel.cc | 39 +++++++++++++++++++++++ src/libstore/local.mk | 2 +- src/{nix/test.cc => libstore/rust.hh} | 41 +------------------------ src/nix/local.mk | 4 +-- 9 files changed, 55 insertions(+), 84 deletions(-) create mode 100644 src/libstore/builtins/unpack-channel.cc rename src/{nix/test.cc => libstore/rust.hh} (53%) diff --git a/corepkgs/config.nix.in b/corepkgs/config.nix.in index 32ce6b399..4ea182d8f 100644 --- a/corepkgs/config.nix.in +++ b/corepkgs/config.nix.in @@ -1,3 +1,4 @@ +# FIXME: remove this file? let fromEnv = var: def: let val = builtins.getEnv var; in @@ -17,13 +18,4 @@ in rec { nixLocalstateDir = "@localstatedir@"; nixSysconfDir = "@sysconfdir@"; nixStoreDir = fromEnv "NIX_STORE_DIR" "@storedir@"; - - # If Nix is installed in the Nix store, then automatically add it as - # a dependency to the core packages. This ensures that they work - # properly in a chroot. - chrootDeps = - if dirOf nixPrefix == builtins.storeDir then - [ (builtins.storePath nixPrefix) ] - else - [ ]; } diff --git a/corepkgs/unpack-channel.nix b/corepkgs/unpack-channel.nix index d39a20637..10515bc8b 100644 --- a/corepkgs/unpack-channel.nix +++ b/corepkgs/unpack-channel.nix @@ -1,39 +1,12 @@ -with import ; - -let - - builder = builtins.toFile "unpack-channel.sh" - '' - mkdir $out - cd $out - xzpat="\.xz\$" - gzpat="\.gz\$" - if [[ "$src" =~ $xzpat ]]; then - ${xz} -d < $src | ${tar} xf - ${tarFlags} - elif [[ "$src" =~ $gzpat ]]; then - ${gzip} -d < $src | ${tar} xf - ${tarFlags} - else - ${bzip2} -d < $src | ${tar} xf - ${tarFlags} - fi - if [ * != $channelName ]; then - mv * $out/$channelName - fi - ''; - -in - { name, channelName, src }: derivation { - system = builtins.currentSystem; - builder = shell; - args = [ "-e" builder ]; - inherit name channelName src; + builder = "builtin:unpack-channel"; - PATH = "${nixBinDir}:${coreutils}"; + system = "builtin"; + + inherit name channelName src; # No point in doing this remotely. preferLocalBuild = true; - - inherit chrootDeps; } diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index 799d52e31..ac6dee543 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -33,7 +33,7 @@ pub extern "C" fn unpack_tarfile(source: Source, dest_dir: &str) -> bool { for file in tar.entries().unwrap() { let mut file = file.unwrap(); - let dest_file = dest_dir.join(file.header().path().unwrap()); + let dest_file = dest_dir.join(file.path().unwrap()); fs::create_dir_all(dest_file.parent().unwrap()).unwrap(); @@ -55,6 +55,9 @@ pub extern "C" fn unpack_tarfile(source: Source, dest_dir: &str) -> bool { .unwrap(); io::copy(&mut file, &mut f).unwrap(); } + tar::EntryType::Symlink => { + std::os::unix::fs::symlink(file.header().link_name().unwrap().unwrap(), dest_file).unwrap(); + } t => panic!("Unsupported tar entry type '{:?}'.", t), } } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 51a9fa35b..efbb7fc9f 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3128,6 +3128,8 @@ void DerivationGoal::runChild() builtinFetchurl(drv2, netrcData); else if (drv->builder == "builtin:buildenv") builtinBuildenv(drv2); + else if (drv->builder == "builtin:unpack-channel") + builtinUnpackChannel(drv2); else throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8)); _exit(0); diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index 0d2da873e..87d6ce665 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -7,5 +7,6 @@ namespace nix { // TODO: make pluggable. void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); void builtinBuildenv(const BasicDerivation & drv); +void builtinUnpackChannel(const BasicDerivation & drv); } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc new file mode 100644 index 000000000..88202ec6b --- /dev/null +++ b/src/libstore/builtins/unpack-channel.cc @@ -0,0 +1,39 @@ +#include "rust.hh" +#include "builtins.hh" +#include "compression.hh" + +namespace nix { + +void builtinUnpackChannel(const BasicDerivation & drv) +{ + auto getAttr = [&](const string & name) { + auto i = drv.env.find(name); + if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + return i->second; + }; + + Path out = getAttr("out"); + auto channelName = getAttr("channelName"); + auto src = getAttr("src"); + + createDirs(out); + + auto source = sinkToSource([&](Sink & sink) { + auto decompressor = + hasSuffix(src, ".bz2") ? makeDecompressionSink("bzip2", sink) : + hasSuffix(src, ".xz") ? makeDecompressionSink("xz", sink) : + makeDecompressionSink("none", sink); + readFile(src, *decompressor); + decompressor->finish(); + }); + + unpack_tarfile(*source, out); + + auto entries = readDirectory(out); + if (entries.size() != 1) + throw Error("channel tarball '%s' contains more than one file", src); + if (rename((out + "/" + entries[0].name).c_str(), (out + "/" + channelName).c_str()) == -1) + throw SysError("renaming channel directory"); +} + +} diff --git a/src/libstore/local.mk b/src/libstore/local.mk index d690fea28..d3254554d 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -6,7 +6,7 @@ libstore_DIR := $(d) libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) -libstore_LIBS = libutil +libstore_LIBS = libutil libnixrust libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread ifneq ($(OS), FreeBSD) diff --git a/src/nix/test.cc b/src/libstore/rust.hh similarity index 53% rename from src/nix/test.cc rename to src/libstore/rust.hh index 7f1f0d47a..7e6c2f54d 100644 --- a/src/nix/test.cc +++ b/src/libstore/rust.hh @@ -1,9 +1,4 @@ -#include "command.hh" -#include "store-api.hh" -#include "common-args.hh" -#include "compression.hh" - -using namespace nix; +#include "serialise.hh" namespace rust { @@ -47,37 +42,3 @@ struct Source extern "C" { bool unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); } - -struct CmdTest : StoreCommand -{ - CmdTest() - { - } - - std::string name() override - { - return "test"; - } - - std::string description() override - { - return "bla bla"; - } - - void run(ref store) override - { - auto source = sinkToSource([&](Sink & sink) { - auto decompressor = makeDecompressionSink("bzip2", sink); - readFile("./nix-2.2.tar.bz2", *decompressor); - decompressor->finish(); - }); - - std::string destDir = "./dest"; - - deletePath(destDir); - - unpack_tarfile(*source, destDir); - } -}; - -static RegisterCommand r(make_ref()); diff --git a/src/nix/local.mk b/src/nix/local.mk index a145cf9b1..c09efd1fc 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -15,9 +15,9 @@ nix_SOURCES := \ $(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-store/*.cc) \ -nix_LIBS = libexpr libmain libstore libutil libnixrust +nix_LIBS = libexpr libmain libstore libutil -nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system -Lnix-rust/target/release +nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ From 6a9c815734de3549a5116796830bd4ebfd006025 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Mar 2019 23:49:14 +0100 Subject: [PATCH 04/23] Remove most of This is no longer needed. --- .gitignore | 1 + corepkgs/config.nix.in | 8 -------- tests/{config.nix => config.nix.in} | 6 ++---- tests/local.mk | 2 +- 4 files changed, 4 insertions(+), 13 deletions(-) rename tests/{config.nix => config.nix.in} (86%) diff --git a/.gitignore b/.gitignore index f58e1f90f..f64031e12 100644 --- a/.gitignore +++ b/.gitignore @@ -86,6 +86,7 @@ perl/Makefile.config /tests/restricted-innocent /tests/shell /tests/shell.drv +/tests/config.nix # /tests/lang/ /tests/lang/*.out diff --git a/corepkgs/config.nix.in b/corepkgs/config.nix.in index 4ea182d8f..cb9945944 100644 --- a/corepkgs/config.nix.in +++ b/corepkgs/config.nix.in @@ -4,14 +4,6 @@ let let val = builtins.getEnv var; in if val != "" then val else def; in rec { - shell = "@bash@"; - coreutils = "@coreutils@"; - bzip2 = "@bzip2@"; - gzip = "@gzip@"; - xz = "@xz@"; - tar = "@tar@"; - tarFlags = "@tarFlags@"; - tr = "@tr@"; nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@"; nixPrefix = "@prefix@"; nixLibexecDir = fromEnv "NIX_LIBEXEC_DIR" "@libexecdir@"; diff --git a/tests/config.nix b/tests/config.nix.in similarity index 86% rename from tests/config.nix rename to tests/config.nix.in index 6ba91065b..51aed539c 100644 --- a/tests/config.nix +++ b/tests/config.nix.in @@ -1,9 +1,7 @@ -with import ; - rec { - inherit shell; + shell = "@bash@"; - path = coreutils; + path = "@coreutils@"; system = builtins.currentSystem; diff --git a/tests/local.mk b/tests/local.mk index 187f96ea2..f1d215eec 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -39,4 +39,4 @@ tests-environment = NIX_REMOTE= $(bash) -e clean-files += $(d)/common.sh -installcheck: $(d)/common.sh $(d)/plugins/libplugintest.$(SO_EXT) +installcheck: $(d)/common.sh $(d)/config.nix $(d)/plugins/libplugintest.$(SO_EXT) From 87b7b25e13206bcfe200f45df822787612f159f7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 28 Mar 2019 00:02:10 +0100 Subject: [PATCH 05/23] Clean up the configure script --- configure.ac | 20 +++----------------- perl/lib/Nix/Config.pm.in | 4 ---- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/configure.ac b/configure.ac index 26f16b846..8e38f2b8e 100644 --- a/configure.ac +++ b/configure.ac @@ -117,26 +117,15 @@ fi ]) NEED_PROG(bash, bash) -NEED_PROG(patch, patch) AC_PATH_PROG(xmllint, xmllint, false) AC_PATH_PROG(xsltproc, xsltproc, false) AC_PATH_PROG(flex, flex, false) AC_PATH_PROG(bison, bison, false) -NEED_PROG(sed, sed) -NEED_PROG(tar, tar) -NEED_PROG(bzip2, bzip2) -NEED_PROG(gzip, gzip) -NEED_PROG(xz, xz) AC_PATH_PROG(dot, dot) AC_PATH_PROG(lsof, lsof, lsof) -NEED_PROG(cat, cat) -NEED_PROG(tr, tr) -AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH], - [path of cat, mkdir, etc.]), - coreutils=$withval, coreutils=$(dirname $cat)) -AC_SUBST(coreutils) +AC_SUBST(coreutils, [$(dirname $(type -p cat))]) AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH], @@ -167,7 +156,8 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then LIBS="-latomic $LIBS" fi -# Look for OpenSSL, a required dependency. +# Look for OpenSSL, a required dependency. FIXME: this is only (maybe) +# used by S3BinaryCacheStore. PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) @@ -177,11 +167,9 @@ AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true], AC_CHECK_HEADERS([bzlib.h], [true], [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) - # Look for SQLite, a required dependency. PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"]) - # Look for libcurl, a required dependency. PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"]) @@ -204,13 +192,11 @@ PKG_CHECK_MODULES([SODIUM], [libsodium], have_sodium=1], [have_sodium=]) AC_SUBST(HAVE_SODIUM, [$have_sodium]) - # Look for liblzma, a required dependency. PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"]) AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt], [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])]) - # Look for libbrotli{enc,dec}. PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]) diff --git a/perl/lib/Nix/Config.pm.in b/perl/lib/Nix/Config.pm.in index 67a20c3f4..bc1749e60 100644 --- a/perl/lib/Nix/Config.pm.in +++ b/perl/lib/Nix/Config.pm.in @@ -11,10 +11,6 @@ $logDir = $ENV{"NIX_LOG_DIR"} || "@nixlocalstatedir@/log/nix"; $confDir = $ENV{"NIX_CONF_DIR"} || "@nixsysconfdir@/nix"; $storeDir = $ENV{"NIX_STORE_DIR"} || "@nixstoredir@"; -$bzip2 = "@bzip2@"; -$xz = "@xz@"; -$curl = "@curl@"; - $useBindings = 1; %config = (); From 0dbb249b36f1d058e63982d26746fde48911cce4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 4 Jul 2019 17:06:38 +0200 Subject: [PATCH 06/23] Update Rust dependencies --- nix-rust/Cargo.lock | 61 +++++++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock index cfaedc3ed..ea7ac72db 100644 --- a/nix-rust/Cargo.lock +++ b/nix-rust/Cargo.lock @@ -1,59 +1,82 @@ [[package]] name = "cfg-if" -version = "0.1.7" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "filetime" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "libc" -version = "0.2.50" +version = "0.2.58" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "nix-rust" version = "0.1.0" dependencies = [ - "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", - "tar 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "redox_syscall" -version = "0.1.51" +version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "tar" -version = "0.4.22" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)", + "filetime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "winapi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "xattr" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", ] [metadata] -"checksum cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "11d43355396e872eefb45ce6342e4374ed7bc2b3a502d1b28e36d6e23c05d1f4" -"checksum filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a2df5c1a8c4be27e7707789dc42ae65976e60b394afd293d1419ab915833e646" -"checksum libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)" = "aab692d7759f5cd8c859e169db98ae5b52c924add2af5fbbca11d12fefb567c1" -"checksum redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)" = "423e376fffca3dfa06c9e9790a9ccd282fafb3cc6e6397d01dbf64f9bacc6b85" -"checksum tar 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)" = "c2167ff53da2a661702b3299f71a91b61b1dffef36b4b2884b1f9c67254c0133" +"checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33" +"checksum filetime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "450537dc346f0c4d738dda31e790da1da5d4bd12145aad4da0d03d713cb3794f" +"checksum libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)" = "6281b86796ba5e4366000be6e9e18bf35580adf9e63fbe2294aadb587613a319" +"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +"checksum tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)" = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3" +"checksum winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "f10e386af2b13e47c89e7236a7a14a086791a2b88ebad6df9bf42040195cf770" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" "checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" From d722e2175ef826cd60f05608fb7d58aa82261549 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Jul 2019 01:01:18 +0200 Subject: [PATCH 07/23] Include cargo dependencies in the Nix tarball --- nix-rust/local.mk | 4 +++- release.nix | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/nix-rust/local.mk b/nix-rust/local.mk index c69e3b9ea..ce88fa51b 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -4,4 +4,6 @@ libnixrust_LDFLAGS_USE := -L$(d)/target/release -lnixrust -ldl libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE) $(d)/target/release/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml - $(trace-gen) cd nix-rust && RUSTC_BOOTSTRAP=1 cargo build --release && touch target/release/libnixrust.a + $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release -Z offline && touch target/release/libnixrust.a + +dist-files += $(d)/vendor diff --git a/release.nix b/release.nix index 1d7c2e2d6..64aa35e11 100644 --- a/release.nix +++ b/release.nix @@ -10,6 +10,43 @@ let jobs = rec { + # Create a "vendor" directory that contains the crates listed in + # Cargo.lock, and include it in the Nix tarball. This allows Nix + # to be built without network access. + vendoredCrates = + let + lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock); + + files = map (pkg: import { + url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download"; + sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)"; + }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package); + + in pkgs.runCommand "cargo-vendor-dir" {} + '' + mkdir -p $out/vendor + + cat > $out/vendor/config < "$dir/.cargo-checksum.json" + + mv "$dir" $out/vendor/ + rmdir $out/vendor/tmp + '') files)} + ''; + tarball = with pkgs; @@ -38,6 +75,8 @@ let distPhase = '' + cp -prd ${vendoredCrates}/vendor/ nix-rust/vendor/ + runHook preDist make dist mkdir -p $out/tarballs From afb021893beb598213d4225de5edd746baa81d7a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Jul 2019 01:19:30 +0200 Subject: [PATCH 08/23] Reduce the size of the vendor directory by removing some winapi cruft --- release.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/release.nix b/release.nix index 64aa35e11..9cf4c74f2 100644 --- a/release.nix +++ b/release.nix @@ -42,8 +42,15 @@ let # Add just enough metadata to keep Cargo happy. printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json" + # Clean up some cruft from the winapi crates. FIXME: find + # a way to remove winapi* from our dependencies. + if [[ $dir =~ /winapi ]]; then + find $dir -name "*.a" -print0 | xargs -0 rm -f -- + fi + mv "$dir" $out/vendor/ - rmdir $out/vendor/tmp + + rm -rf $out/vendor/tmp '') files)} ''; From 343ebcc048c69e103db3f87a9339388bd5dd93cb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Jul 2019 01:50:35 +0200 Subject: [PATCH 09/23] Only pass '-Z offline' to cargo if we have a vendor directory --- nix-rust/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix-rust/local.mk b/nix-rust/local.mk index ce88fa51b..a5e017cef 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -4,6 +4,6 @@ libnixrust_LDFLAGS_USE := -L$(d)/target/release -lnixrust -ldl libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE) $(d)/target/release/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml - $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release -Z offline && touch target/release/libnixrust.a + $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release $$(if [[ -d vendor ]]; then echo -Z offline; fi) && touch target/release/libnixrust.a dist-files += $(d)/vendor From 8110b4ebb29174ecd4b22510da0285abf604b8a7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Sep 2019 21:55:32 +0200 Subject: [PATCH 10/23] Rust cleanup --- nix-rust/Cargo.toml | 1 + nix-rust/src/error.rs | 5 +++ nix-rust/src/foreign.rs | 14 +++++++++ nix-rust/src/lib.rs | 68 ++++------------------------------------- nix-rust/src/tarfile.rs | 47 ++++++++++++++++++++++++++++ src/libstore/rust.hh | 2 +- 6 files changed, 74 insertions(+), 63 deletions(-) create mode 100644 nix-rust/src/error.rs create mode 100644 nix-rust/src/foreign.rs create mode 100644 nix-rust/src/tarfile.rs diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml index 2d2bf6752..8b56dc89c 100644 --- a/nix-rust/Cargo.toml +++ b/nix-rust/Cargo.toml @@ -2,6 +2,7 @@ name = "nix-rust" version = "0.1.0" authors = ["Eelco Dolstra "] +edition = "2018" [lib] name = "nixrust" diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs new file mode 100644 index 000000000..28d0abdef --- /dev/null +++ b/nix-rust/src/error.rs @@ -0,0 +1,5 @@ +#[derive(Debug)] +pub enum Error { + Misc(String), + Foreign(libc::c_void), // == std::exception_ptr +} diff --git a/nix-rust/src/foreign.rs b/nix-rust/src/foreign.rs new file mode 100644 index 000000000..7bce7753c --- /dev/null +++ b/nix-rust/src/foreign.rs @@ -0,0 +1,14 @@ +/// A wrapper around Nix's Source class that provides the Read trait. +#[repr(C)] +pub struct Source { + fun: extern "C" fn(this: *mut libc::c_void, data: &mut [u8]) -> usize, + this: *mut libc::c_void, +} + +impl std::io::Read for Source { + fn read(&mut self, buf: &mut [u8]) -> std::result::Result { + let n = (self.fun)(self.this, buf); + assert!(n <= buf.len()); + Ok(n) + } +} diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index ac6dee543..192ca29e4 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -1,66 +1,10 @@ -extern crate libc; -extern crate tar; +mod error; +mod foreign; +mod tarfile; -use std::fs; -use std::io; -use std::os::unix::fs::OpenOptionsExt; -use std::path::Path; -use tar::Archive; - -/// A wrapper around Nix's Source class that provides the Read trait. -#[repr(C)] -pub struct Source { - fun: extern "C" fn(this: *mut libc::c_void, data: &mut [u8]) -> usize, - this: *mut libc::c_void, -} - -impl std::io::Read for Source { - fn read(&mut self, buf: &mut [u8]) -> std::result::Result { - let n = (self.fun)(self.this, buf); - assert!(n <= buf.len()); - Ok(n) - } -} +pub use error::Error; #[no_mangle] -pub extern "C" fn unpack_tarfile(source: Source, dest_dir: &str) -> bool { - // FIXME: handle errors. - - let dest_dir = Path::new(dest_dir); - - let mut tar = Archive::new(source); - - for file in tar.entries().unwrap() { - let mut file = file.unwrap(); - - let dest_file = dest_dir.join(file.path().unwrap()); - - fs::create_dir_all(dest_file.parent().unwrap()).unwrap(); - - match file.header().entry_type() { - tar::EntryType::Directory => { - fs::create_dir(dest_file).unwrap(); - } - tar::EntryType::Regular => { - let mode = if file.header().mode().unwrap() & libc::S_IXUSR == 0 { - 0o666 - } else { - 0o777 - }; - let mut f = fs::OpenOptions::new() - .create(true) - .write(true) - .mode(mode) - .open(dest_file) - .unwrap(); - io::copy(&mut file, &mut f).unwrap(); - } - tar::EntryType::Symlink => { - std::os::unix::fs::symlink(file.header().link_name().unwrap().unwrap(), dest_file).unwrap(); - } - t => panic!("Unsupported tar entry type '{:?}'.", t), - } - } - - true +pub extern "C" fn unpack_tarfile(source: foreign::Source, dest_dir: &str) { + tarfile::unpack_tarfile(source, dest_dir).unwrap(); } diff --git a/nix-rust/src/tarfile.rs b/nix-rust/src/tarfile.rs new file mode 100644 index 000000000..696118e4d --- /dev/null +++ b/nix-rust/src/tarfile.rs @@ -0,0 +1,47 @@ +use crate::{foreign::Source, Error}; +use std::fs; +use std::io; +use std::os::unix::fs::OpenOptionsExt; +use std::path::Path; +use tar::Archive; + +pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> { + let dest_dir = Path::new(dest_dir); + + let mut tar = Archive::new(source); + + for file in tar.entries().unwrap() { + let mut file = file.unwrap(); + + let dest_file = dest_dir.join(file.path().unwrap()); + + fs::create_dir_all(dest_file.parent().unwrap()).unwrap(); + + match file.header().entry_type() { + tar::EntryType::Directory => { + fs::create_dir(dest_file).unwrap(); + } + tar::EntryType::Regular => { + let mode = if file.header().mode().unwrap() & libc::S_IXUSR == 0 { + 0o666 + } else { + 0o777 + }; + let mut f = fs::OpenOptions::new() + .create(true) + .write(true) + .mode(mode) + .open(dest_file) + .unwrap(); + io::copy(&mut file, &mut f).unwrap(); + } + tar::EntryType::Symlink => { + std::os::unix::fs::symlink(file.header().link_name().unwrap().unwrap(), dest_file) + .unwrap(); + } + t => return Err(Error::Misc(format!("unsupported tar entry type '{:?}'", t))), + } + } + + Ok(()) +} diff --git a/src/libstore/rust.hh b/src/libstore/rust.hh index 7e6c2f54d..4366e4723 100644 --- a/src/libstore/rust.hh +++ b/src/libstore/rust.hh @@ -40,5 +40,5 @@ struct Source } extern "C" { - bool unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); + void unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); } From f738cd4d976f4f72159bbcbfa7b451c33f0ea74a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Sep 2019 01:15:20 +0200 Subject: [PATCH 11/23] More Rust FFI adventures We can now convert Rust Errors to C++ exceptions. At the Rust->C++ FFI boundary, Result will cause Error to be converted to and thrown as a C++ exception. --- nix-rust/src/error.rs | 27 ++++++++++- nix-rust/src/lib.rs | 29 ++++++++++-- nix-rust/src/tarfile.rs | 20 ++++---- src/libstore/builtins/unpack-channel.cc | 2 +- src/libstore/rust.cc | 12 +++++ src/libstore/rust.hh | 61 ++++++++++++++++++++++++- 6 files changed, 133 insertions(+), 18 deletions(-) create mode 100644 src/libstore/rust.cc diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs index 28d0abdef..a2003be6f 100644 --- a/nix-rust/src/error.rs +++ b/nix-rust/src/error.rs @@ -1,5 +1,30 @@ #[derive(Debug)] pub enum Error { + IOError(std::io::Error), Misc(String), - Foreign(libc::c_void), // == std::exception_ptr + Foreign(CppException), +} + +impl From for Error { + fn from(err: std::io::Error) -> Self { + Error::IOError(err) + } +} + +impl From for CppException { + fn from(err: Error) -> Self { + match err { + Error::Foreign(ex) => ex, + Error::Misc(s) => unsafe { make_error(&s) }, + Error::IOError(err) => unsafe { make_error(&err.to_string()) }, + } + } +} + +#[repr(C)] +#[derive(Debug)] +pub struct CppException(*const libc::c_void); // == std::exception_ptr* + +extern "C" { + fn make_error(s: &str) -> CppException; } diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index 192ca29e4..b6b0d746d 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -4,7 +4,30 @@ mod tarfile; pub use error::Error; -#[no_mangle] -pub extern "C" fn unpack_tarfile(source: foreign::Source, dest_dir: &str) { - tarfile::unpack_tarfile(source, dest_dir).unwrap(); +pub struct CBox { + ptr: *mut libc::c_void, + phantom: std::marker::PhantomData, +} + +impl CBox { + fn new(t: T) -> Self { + unsafe { + let size = std::mem::size_of::(); + let ptr = libc::malloc(size); + eprintln!("PTR = {:?}, SIZE = {}", ptr, size); + *(ptr as *mut T) = t; // FIXME: probably UB + Self { + ptr, + phantom: std::marker::PhantomData, + } + } + } +} + +#[no_mangle] +pub extern "C" fn unpack_tarfile( + source: foreign::Source, + dest_dir: &str, +) -> CBox> { + CBox::new(tarfile::unpack_tarfile(source, dest_dir).map_err(|err| err.into())) } diff --git a/nix-rust/src/tarfile.rs b/nix-rust/src/tarfile.rs index 696118e4d..797aa5064 100644 --- a/nix-rust/src/tarfile.rs +++ b/nix-rust/src/tarfile.rs @@ -10,19 +10,19 @@ pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> { let mut tar = Archive::new(source); - for file in tar.entries().unwrap() { - let mut file = file.unwrap(); + for file in tar.entries()? { + let mut file = file?; - let dest_file = dest_dir.join(file.path().unwrap()); + let dest_file = dest_dir.join(file.path()?); - fs::create_dir_all(dest_file.parent().unwrap()).unwrap(); + fs::create_dir_all(dest_file.parent().unwrap())?; match file.header().entry_type() { tar::EntryType::Directory => { - fs::create_dir(dest_file).unwrap(); + fs::create_dir(dest_file)?; } tar::EntryType::Regular => { - let mode = if file.header().mode().unwrap() & libc::S_IXUSR == 0 { + let mode = if file.header().mode()? & libc::S_IXUSR == 0 { 0o666 } else { 0o777 @@ -31,13 +31,11 @@ pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> { .create(true) .write(true) .mode(mode) - .open(dest_file) - .unwrap(); - io::copy(&mut file, &mut f).unwrap(); + .open(dest_file)?; + io::copy(&mut file, &mut f)?; } tar::EntryType::Symlink => { - std::os::unix::fs::symlink(file.header().link_name().unwrap().unwrap(), dest_file) - .unwrap(); + std::os::unix::fs::symlink(file.header().link_name()?.unwrap(), dest_file)?; } t => return Err(Error::Misc(format!("unsupported tar entry type '{:?}'", t))), } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index 88202ec6b..2da26d98e 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -27,7 +27,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) decompressor->finish(); }); - unpack_tarfile(*source, out); + unpack_tarfile(*source, out).use()->unwrap(); auto entries = readDirectory(out); if (entries.size() != 1) diff --git a/src/libstore/rust.cc b/src/libstore/rust.cc new file mode 100644 index 000000000..a616d83a6 --- /dev/null +++ b/src/libstore/rust.cc @@ -0,0 +1,12 @@ +#include "logging.hh" +#include "rust.hh" + +namespace nix { + +extern "C" std::exception_ptr * make_error(rust::StringSlice s) +{ + // FIXME: leak + return new std::exception_ptr(std::make_exception_ptr(Error(std::string(s.ptr, s.size)))); +} + +} diff --git a/src/libstore/rust.hh b/src/libstore/rust.hh index 4366e4723..4c7720a44 100644 --- a/src/libstore/rust.hh +++ b/src/libstore/rust.hh @@ -4,7 +4,8 @@ namespace rust { // Depending on the internal representation of Rust slices is slightly // evil... -template struct Slice +template +struct Slice { T * ptr; size_t size; @@ -37,8 +38,64 @@ struct Source } }; +/* C++ representation of Rust's Result. */ +template +struct Result +{ + unsigned int tag; + + union { + T data; + std::exception_ptr * exc; + }; + + /* Rethrow the wrapped exception or return the wrapped value. */ + T unwrap() + { + if (tag == 0) + return data; + else if (tag == 1) + std::rethrow_exception(*exc); + else + abort(); + } +}; + +template +struct CBox +{ + T * ptr; + + T * operator ->() + { + return ptr; + } + + CBox(T * ptr) : ptr(ptr) { } + CBox(const CBox &) = delete; + CBox(CBox &&) = delete; + + ~CBox() + { + free(ptr); + } +}; + +// Grrr, this is only needed because 'extern "C"' functions don't +// support non-POD return types (and CBox has a destructor so it's not +// POD). +template +struct CBox2 +{ + T * ptr; + CBox use() + { + return CBox(ptr); + } +}; + } extern "C" { - void unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); + rust::CBox2>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); } From b7fba16613aed5c2b2a0b4f98e3e0d32b0cddd40 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Sep 2019 12:43:07 +0200 Subject: [PATCH 12/23] Move code around --- src/libstore/rust.hh | 97 +------------------ src/{libstore/rust.cc => libutil/rust-ffi.cc} | 0 src/libutil/rust-ffi.hh | 97 +++++++++++++++++++ 3 files changed, 98 insertions(+), 96 deletions(-) rename src/{libstore/rust.cc => libutil/rust-ffi.cc} (100%) create mode 100644 src/libutil/rust-ffi.hh diff --git a/src/libstore/rust.hh b/src/libstore/rust.hh index 4c7720a44..dccc98b4b 100644 --- a/src/libstore/rust.hh +++ b/src/libstore/rust.hh @@ -1,100 +1,5 @@ #include "serialise.hh" - -namespace rust { - -// Depending on the internal representation of Rust slices is slightly -// evil... -template -struct Slice -{ - T * ptr; - size_t size; - - Slice(T * ptr, size_t size) : ptr(ptr), size(size) - { - assert(ptr); - } -}; - -struct StringSlice : Slice -{ - StringSlice(const std::string & s): Slice((char *) s.data(), s.size()) {} -}; - -struct Source -{ - size_t (*fun)(void * source_this, rust::Slice data); - nix::Source * _this; - - Source(nix::Source & _this) - : fun(sourceWrapper), _this(&_this) - {} - - // FIXME: how to propagate exceptions? - static size_t sourceWrapper(void * _this, rust::Slice data) - { - auto n = ((nix::Source *) _this)->read(data.ptr, data.size); - return n; - } -}; - -/* C++ representation of Rust's Result. */ -template -struct Result -{ - unsigned int tag; - - union { - T data; - std::exception_ptr * exc; - }; - - /* Rethrow the wrapped exception or return the wrapped value. */ - T unwrap() - { - if (tag == 0) - return data; - else if (tag == 1) - std::rethrow_exception(*exc); - else - abort(); - } -}; - -template -struct CBox -{ - T * ptr; - - T * operator ->() - { - return ptr; - } - - CBox(T * ptr) : ptr(ptr) { } - CBox(const CBox &) = delete; - CBox(CBox &&) = delete; - - ~CBox() - { - free(ptr); - } -}; - -// Grrr, this is only needed because 'extern "C"' functions don't -// support non-POD return types (and CBox has a destructor so it's not -// POD). -template -struct CBox2 -{ - T * ptr; - CBox use() - { - return CBox(ptr); - } -}; - -} +#include "rust-ffi.hh" extern "C" { rust::CBox2>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); diff --git a/src/libstore/rust.cc b/src/libutil/rust-ffi.cc similarity index 100% rename from src/libstore/rust.cc rename to src/libutil/rust-ffi.cc diff --git a/src/libutil/rust-ffi.hh b/src/libutil/rust-ffi.hh new file mode 100644 index 000000000..a488b96d6 --- /dev/null +++ b/src/libutil/rust-ffi.hh @@ -0,0 +1,97 @@ +#include "serialise.hh" + +namespace rust { + +// Depending on the internal representation of Rust slices is slightly +// evil... +template +struct Slice +{ + T * ptr; + size_t size; + + Slice(T * ptr, size_t size) : ptr(ptr), size(size) + { + assert(ptr); + } +}; + +struct StringSlice : Slice +{ + StringSlice(const std::string & s): Slice((char *) s.data(), s.size()) {} +}; + +struct Source +{ + size_t (*fun)(void * source_this, rust::Slice data); + nix::Source * _this; + + Source(nix::Source & _this) + : fun(sourceWrapper), _this(&_this) + {} + + // FIXME: how to propagate exceptions? + static size_t sourceWrapper(void * _this, rust::Slice data) + { + auto n = ((nix::Source *) _this)->read(data.ptr, data.size); + return n; + } +}; + +/* C++ representation of Rust's Result. */ +template +struct Result +{ + unsigned int tag; + + union { + T data; + std::exception_ptr * exc; + }; + + /* Rethrow the wrapped exception or return the wrapped value. */ + T unwrap() + { + if (tag == 0) + return data; + else if (tag == 1) + std::rethrow_exception(*exc); + else + abort(); + } +}; + +template +struct CBox +{ + T * ptr; + + T * operator ->() + { + return ptr; + } + + CBox(T * ptr) : ptr(ptr) { } + CBox(const CBox &) = delete; + CBox(CBox &&) = delete; + + ~CBox() + { + free(ptr); + } +}; + +// Grrr, this is only needed because 'extern "C"' functions don't +// support non-POD return types (and CBox has a destructor so it's not +// POD). +template +struct CBox2 +{ + T * ptr; + CBox use() + { + return CBox(ptr); + } +}; + +} From d14b1c261cd3dfb4be2da943d901a394c3f23205 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Sep 2019 12:44:31 +0200 Subject: [PATCH 13/23] Shut up some rust warnings --- nix-rust/src/error.rs | 1 + nix-rust/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs index a2003be6f..519007ea0 100644 --- a/nix-rust/src/error.rs +++ b/nix-rust/src/error.rs @@ -26,5 +26,6 @@ impl From for CppException { pub struct CppException(*const libc::c_void); // == std::exception_ptr* extern "C" { + #[allow(improper_ctypes)] // YOLO fn make_error(s: &str) -> CppException; } diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index b6b0d746d..48952d8b9 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -5,7 +5,7 @@ mod tarfile; pub use error::Error; pub struct CBox { - ptr: *mut libc::c_void, + pub ptr: *mut libc::c_void, phantom: std::marker::PhantomData, } From d33dd6e6c09d87a59989057ad622a6265ddec2e0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Sep 2019 13:10:46 +0200 Subject: [PATCH 14/23] Move code around --- nix-rust/src/lib.rs | 1 - src/libstore/builtins/unpack-channel.cc | 4 ++-- src/libstore/local.mk | 2 +- src/libutil/local.mk | 2 ++ src/libutil/rust-ffi.cc | 2 +- src/{libstore/rust.hh => libutil/tarfile.cc} | 10 +++++++++- src/libutil/tarfile.hh | 7 +++++++ 7 files changed, 22 insertions(+), 6 deletions(-) rename src/{libstore/rust.hh => libutil/tarfile.cc} (53%) create mode 100644 src/libutil/tarfile.hh diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index 48952d8b9..1b88ac8af 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -14,7 +14,6 @@ impl CBox { unsafe { let size = std::mem::size_of::(); let ptr = libc::malloc(size); - eprintln!("PTR = {:?}, SIZE = {}", ptr, size); *(ptr as *mut T) = t; // FIXME: probably UB Self { ptr, diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index 2da26d98e..5fc68cd66 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -1,6 +1,6 @@ -#include "rust.hh" #include "builtins.hh" #include "compression.hh" +#include "tarfile.hh" namespace nix { @@ -27,7 +27,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) decompressor->finish(); }); - unpack_tarfile(*source, out).use()->unwrap(); + unpackTarfile(*source, out); auto entries = readDirectory(out); if (entries.size() != 1) diff --git a/src/libstore/local.mk b/src/libstore/local.mk index d3254554d..d690fea28 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -6,7 +6,7 @@ libstore_DIR := $(d) libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) -libstore_LIBS = libutil libnixrust +libstore_LIBS = libutil libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread ifneq ($(OS), FreeBSD) diff --git a/src/libutil/local.mk b/src/libutil/local.mk index e41a67d1f..35c1f6c13 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -7,3 +7,5 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context + +libutil_LIBS = libnixrust diff --git a/src/libutil/rust-ffi.cc b/src/libutil/rust-ffi.cc index a616d83a6..931d29542 100644 --- a/src/libutil/rust-ffi.cc +++ b/src/libutil/rust-ffi.cc @@ -1,5 +1,5 @@ #include "logging.hh" -#include "rust.hh" +#include "rust-ffi.hh" namespace nix { diff --git a/src/libstore/rust.hh b/src/libutil/tarfile.cc similarity index 53% rename from src/libstore/rust.hh rename to src/libutil/tarfile.cc index dccc98b4b..ae6d512bd 100644 --- a/src/libstore/rust.hh +++ b/src/libutil/tarfile.cc @@ -1,6 +1,14 @@ -#include "serialise.hh" #include "rust-ffi.hh" extern "C" { rust::CBox2>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); } + +namespace nix { + +void unpackTarfile(Source & source, Path destDir) +{ + unpack_tarfile(source, destDir).use()->unwrap(); +} + +} diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh new file mode 100644 index 000000000..c3e95fb0c --- /dev/null +++ b/src/libutil/tarfile.hh @@ -0,0 +1,7 @@ +#include "serialise.hh" + +namespace nix { + +void unpackTarfile(Source & source, Path destDir); + +} From f2bd8470926686361602e545d63a69d4bfc22f90 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Sep 2019 15:03:17 +0200 Subject: [PATCH 15/23] Ignore tar header entries In particular, these are emitted by 'git archive' (in fetchGit). --- nix-rust/src/tarfile.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/nix-rust/src/tarfile.rs b/nix-rust/src/tarfile.rs index 797aa5064..9e32a3f00 100644 --- a/nix-rust/src/tarfile.rs +++ b/nix-rust/src/tarfile.rs @@ -37,6 +37,7 @@ pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> { tar::EntryType::Symlink => { std::os::unix::fs::symlink(file.header().link_name()?.unwrap(), dest_file)?; } + tar::EntryType::XGlobalHeader | tar::EntryType::XHeader => {} t => return Err(Error::Misc(format!("unsupported tar entry type '{:?}'", t))), } } From 8918bae09828133259acb36d6aef60ffbfad252c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Sep 2019 15:25:43 +0200 Subject: [PATCH 16/23] Drop remaining uses of external "tar" Also, fetchGit now runs in O(1) memory since we pipe the output of 'git archive' directly into unpackTarball() (rather than first reading it all into memory). --- src/libexpr/primops/fetchGit.cc | 11 +++++++---- src/libstore/builtins/unpack-channel.cc | 12 +----------- src/libstore/download.cc | 12 ++++++++---- src/libutil/serialise.hh | 1 - src/libutil/tarfile.cc | 22 +++++++++++++++++++++- src/libutil/tarfile.hh | 5 ++++- src/nix-prefetch-url/nix-prefetch-url.cc | 4 ++-- 7 files changed, 43 insertions(+), 24 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 7ef3b3823..9d0c64291 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -4,6 +4,7 @@ #include "store-api.hh" #include "pathlocks.hh" #include "hash.hh" +#include "tarfile.hh" #include @@ -164,14 +165,16 @@ GitInfo exportGit(ref store, const std::string & uri, if (e.errNo != ENOENT) throw; } - // FIXME: should pipe this, or find some better way to extract a - // revision. - auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev }); + auto source = sinkToSource([&](Sink & sink) { + RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); + gitOptions.standardOut = &sink; + runProgram2(gitOptions); + }); Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); - runProgram("tar", true, { "x", "-C", tmpDir }, tar); + unpackTarfile(*source, tmpDir); gitInfo.storePath = store->addToStore(name, tmpDir); diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index 5fc68cd66..d18e3ddaf 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -1,5 +1,4 @@ #include "builtins.hh" -#include "compression.hh" #include "tarfile.hh" namespace nix { @@ -18,16 +17,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) createDirs(out); - auto source = sinkToSource([&](Sink & sink) { - auto decompressor = - hasSuffix(src, ".bz2") ? makeDecompressionSink("bzip2", sink) : - hasSuffix(src, ".xz") ? makeDecompressionSink("xz", sink) : - makeDecompressionSink("none", sink); - readFile(src, *decompressor); - decompressor->finish(); - }); - - unpackTarfile(*source, out); + unpackTarfile(src, out); auto entries = readDirectory(out); if (entries.size() != 1) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index e80663dff..61e88c5c1 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -8,6 +8,7 @@ #include "compression.hh" #include "pathlocks.hh" #include "finally.hh" +#include "tarfile.hh" #ifdef ENABLE_S3 #include @@ -903,12 +904,15 @@ CachedDownloadResult Downloader::downloadCached( unpackedStorePath = ""; } if (unpackedStorePath.empty()) { - printInfo(format("unpacking '%1%'...") % url); + printInfo("unpacking '%s'...", url); Path tmpDir = createTempDir(); AutoDelete autoDelete(tmpDir, true); - // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"}); - unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair); + unpackTarfile(store->toRealPath(storePath), tmpDir, baseNameOf(url)); + auto members = readDirectory(tmpDir); + if (members.size() != 1) + throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); + auto topDir = tmpDir + "/" + members.begin()->name; + unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); } replaceSymlink(unpackedStorePath, unpackedLink); storePath = unpackedStorePath; diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 128e287f3..5780c93a6 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -77,7 +77,6 @@ struct BufferedSource : Source size_t read(unsigned char * data, size_t len) override; - bool hasData(); protected: diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index ae6d512bd..f7d3ad417 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -1,4 +1,5 @@ #include "rust-ffi.hh" +#include "compression.hh" extern "C" { rust::CBox2>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); @@ -6,9 +7,28 @@ extern "C" { namespace nix { -void unpackTarfile(Source & source, Path destDir) +void unpackTarfile(Source & source, const Path & destDir) { unpack_tarfile(source, destDir).use()->unwrap(); } +void unpackTarfile(const Path & tarFile, const Path & destDir, + std::optional baseName) +{ + if (!baseName) baseName = baseNameOf(tarFile); + + auto source = sinkToSource([&](Sink & sink) { + // FIXME: look at first few bytes to determine compression type. + auto decompressor = + // FIXME: add .gz support + hasSuffix(*baseName, ".bz2") ? makeDecompressionSink("bzip2", sink) : + hasSuffix(*baseName, ".xz") ? makeDecompressionSink("xz", sink) : + makeDecompressionSink("none", sink); + readFile(tarFile, *decompressor); + decompressor->finish(); + }); + + unpackTarfile(*source, destDir); +} + } diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index c3e95fb0c..ce0911e2a 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -2,6 +2,9 @@ namespace nix { -void unpackTarfile(Source & source, Path destDir); +void unpackTarfile(Source & source, const Path & destDir); + +void unpackTarfile(const Path & tarFile, const Path & destDir, + std::optional baseName = {}); } diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index f54706a8a..78c883833 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -9,6 +9,7 @@ #include "legacy.hh" #include "finally.hh" #include "progress-bar.hh" +#include "tarfile.hh" #include @@ -192,8 +193,7 @@ static int _main(int argc, char * * argv) if (hasSuffix(baseNameOf(uri), ".zip")) runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked}); else - // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", tmpFile, "-C", unpacked}); + unpackTarfile(tmpFile, unpacked, baseNameOf(uri)); /* If the archive unpacks to a single file/directory, then use that as the top-level. */ From 88f8063917a4261abfabfe94bc33c63f7405d1d5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 26 Nov 2019 22:45:15 +0100 Subject: [PATCH 17/23] -Z offline -> --offline --- nix-rust/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix-rust/local.mk b/nix-rust/local.mk index a5e017cef..fda1c3dee 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -4,6 +4,6 @@ libnixrust_LDFLAGS_USE := -L$(d)/target/release -lnixrust -ldl libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE) $(d)/target/release/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml - $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release $$(if [[ -d vendor ]]; then echo -Z offline; fi) && touch target/release/libnixrust.a + $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release $$(if [[ -d vendor ]]; then echo --offline; fi) && touch target/release/libnixrust.a dist-files += $(d)/vendor From e6c1d1b474b1f14753b27f35b2aad4051ffd9e8d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 26 Nov 2019 22:46:36 +0100 Subject: [PATCH 18/23] Update Cargo.lock --- nix-rust/Cargo.lock | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock index ea7ac72db..0112ed471 100644 --- a/nix-rust/Cargo.lock +++ b/nix-rust/Cargo.lock @@ -1,29 +1,31 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. [[package]] name = "cfg-if" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "filetime" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "libc" -version = "0.2.58" +version = "0.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "nix-rust" version = "0.1.0" dependencies = [ - "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -37,15 +39,15 @@ name = "tar" version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "filetime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "winapi" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -67,16 +69,16 @@ name = "xattr" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", ] [metadata] -"checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33" -"checksum filetime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "450537dc346f0c4d738dda31e790da1da5d4bd12145aad4da0d03d713cb3794f" -"checksum libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)" = "6281b86796ba5e4366000be6e9e18bf35580adf9e63fbe2294aadb587613a319" +"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +"checksum filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1ff6d4dab0aa0c8e6346d46052e93b13a16cf847b54ed357087c35011048cc7d" +"checksum libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)" = "1a31a0627fdf1f6a39ec0dd577e101440b7db22672c0901fe00a9a6fbb5c24e8" "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" "checksum tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)" = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3" -"checksum winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "f10e386af2b13e47c89e7236a7a14a086791a2b88ebad6df9bf42040195cf770" +"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" "checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" From dbc4f9d478814f3ce4ee23531502247d02c85911 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Nov 2019 00:17:24 +0100 Subject: [PATCH 19/23] Fix macOS build https://hydra.nixos.org/build/107466992 --- nix-rust/src/tarfile.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix-rust/src/tarfile.rs b/nix-rust/src/tarfile.rs index 9e32a3f00..379d9098f 100644 --- a/nix-rust/src/tarfile.rs +++ b/nix-rust/src/tarfile.rs @@ -22,7 +22,7 @@ pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> { fs::create_dir(dest_file)?; } tar::EntryType::Regular => { - let mode = if file.header().mode()? & libc::S_IXUSR == 0 { + let mode = if file.header().mode()? & (libc::S_IXUSR as u32) == 0 { 0o666 } else { 0o777 From 949dc848940b68ffd1327a278df4472e98a455dc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Nov 2019 14:17:15 +0100 Subject: [PATCH 20/23] Fix segfault on i686-linux https://hydra.nixos.org/build/107467517 Seems that on i686-linux, gcc and rustc disagree on how to return 1-word structs: gcc has the caller pass a pointer to the result, while rustc has the callee return the result in a register. Work around this by using a bare pointer. --- src/libutil/rust-ffi.hh | 13 ------------- src/libutil/tarfile.cc | 6 ++++-- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/src/libutil/rust-ffi.hh b/src/libutil/rust-ffi.hh index a488b96d6..663758bfc 100644 --- a/src/libutil/rust-ffi.hh +++ b/src/libutil/rust-ffi.hh @@ -81,17 +81,4 @@ struct CBox } }; -// Grrr, this is only needed because 'extern "C"' functions don't -// support non-POD return types (and CBox has a destructor so it's not -// POD). -template -struct CBox2 -{ - T * ptr; - CBox use() - { - return CBox(ptr); - } -}; - } diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index f7d3ad417..2cc7793fd 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -2,14 +2,16 @@ #include "compression.hh" extern "C" { - rust::CBox2>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); + rust::Result> * + unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); } namespace nix { void unpackTarfile(Source & source, const Path & destDir) { - unpack_tarfile(source, destDir).use()->unwrap(); + rust::Source source2(source); + rust::CBox(unpack_tarfile(source2, destDir))->unwrap(); } void unpackTarfile(const Path & tarFile, const Path & destDir, From f553a8bdea96afe18b97dee6eba91c0bbc545b79 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Nov 2019 14:18:57 +0100 Subject: [PATCH 21/23] When OPTIMIZE=0, build rust code in debug mode --- nix-rust/local.mk | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/nix-rust/local.mk b/nix-rust/local.mk index fda1c3dee..c147f9874 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -1,9 +1,19 @@ -libnixrust_PATH := $(d)/target/release/libnixrust.a +ifeq ($(OPTIMIZE), 1) + RUST_MODE = --release + RUST_DIR = release +else + RUST_MODE = + RUST_DIR = debug +endif + +libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.a libnixrust_INSTALL_PATH := $(libnixrust_PATH) -libnixrust_LDFLAGS_USE := -L$(d)/target/release -lnixrust -ldl +libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE) -$(d)/target/release/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml - $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release $$(if [[ -d vendor ]]; then echo --offline; fi) && touch target/release/libnixrust.a +$(d)/target/$(RUST_DIR)/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml + $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) \ + cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \ + && touch target/$(RUST_DIR)/libnixrust.a dist-files += $(d)/vendor From 895ce1bb6cc77324c377baa08eb98666768e5c6d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Nov 2019 17:33:59 +0100 Subject: [PATCH 22/23] make clean: Delete nix-rust/target --- nix-rust/local.mk | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nix-rust/local.mk b/nix-rust/local.mk index c147f9874..ed1e640c9 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -17,3 +17,8 @@ $(d)/target/$(RUST_DIR)/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml && touch target/$(RUST_DIR)/libnixrust.a dist-files += $(d)/vendor + +clean: clean-rust + +clean-rust: + $(suppress) rm -rfv nix-rust/target From 39954a958623431acb8642372f881cbdb7bb789d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 29 Nov 2019 18:27:40 +0100 Subject: [PATCH 23/23] Make libnixrust a dynamic library This is a hack to fix the build on macOS, which was failing because libnixrust.a contains compiler builtins that clash with libclang_rt.osx.a. There's probably a better solution... https://hydra.nixos.org/build/107473280 --- nix-rust/Cargo.toml | 2 +- nix-rust/local.mk | 26 ++++++++++++++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml index 8b56dc89c..c4f4ceb09 100644 --- a/nix-rust/Cargo.toml +++ b/nix-rust/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [lib] name = "nixrust" -crate-type = ["staticlib"] +crate-type = ["cdylib"] [dependencies] tar = "0.4" diff --git a/nix-rust/local.mk b/nix-rust/local.mk index ed1e640c9..7645d5394 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -6,15 +6,29 @@ else RUST_DIR = debug endif -libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.a -libnixrust_INSTALL_PATH := $(libnixrust_PATH) +libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.$(SO_EXT) +libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT) libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl -libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE) +libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ldl -$(d)/target/$(RUST_DIR)/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml +ifeq ($(OS), Darwin) +libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup" +else +libnixrust_LDFLAGS_USE += -Wl,-rpath,$(abspath $(d)/target/$(RUST_DIR)) +libnixrust_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$(libdir) +endif + +$(libnixrust_PATH): $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) \ - cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \ - && touch target/$(RUST_DIR)/libnixrust.a + $(libnixrust_BUILD_FLAGS) \ + cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \ + && touch target/$(RUST_DIR)/libnixrust.$(SO_EXT) + +$(libnixrust_INSTALL_PATH): $(libnixrust_PATH) + $(target-gen) cp $^ $@ +ifeq ($(OS), Darwin) + install_name_tool -id $@ $@ +endif dist-files += $(d)/vendor