Merge pull request #2748 from edolstra/rust

Make nix/unpack-channel.nix a builtin builder
This commit is contained in:
Eelco Dolstra 2019-11-29 19:33:31 +01:00 committed by GitHub
commit f102d793f1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
30 changed files with 514 additions and 86 deletions

3
.gitignore vendored
View file

@ -85,6 +85,7 @@ perl/Makefile.config
/tests/restricted-innocent /tests/restricted-innocent
/tests/shell /tests/shell
/tests/shell.drv /tests/shell.drv
/tests/config.nix
# /tests/lang/ # /tests/lang/
/tests/lang/*.out /tests/lang/*.out
@ -118,3 +119,5 @@ GPATH
GRTAGS GRTAGS
GSYMS GSYMS
GTAGS GTAGS
nix-rust/target

View file

@ -1,6 +1,7 @@
makefiles = \ makefiles = \
mk/precompiled-headers.mk \ mk/precompiled-headers.mk \
local.mk \ local.mk \
nix-rust/local.mk \
src/libutil/local.mk \ src/libutil/local.mk \
src/libstore/local.mk \ src/libstore/local.mk \
src/libmain/local.mk \ src/libmain/local.mk \

View file

@ -117,26 +117,15 @@ fi
]) ])
NEED_PROG(bash, bash) NEED_PROG(bash, bash)
NEED_PROG(patch, patch)
AC_PATH_PROG(xmllint, xmllint, false) AC_PATH_PROG(xmllint, xmllint, false)
AC_PATH_PROG(xsltproc, xsltproc, false) AC_PATH_PROG(xsltproc, xsltproc, false)
AC_PATH_PROG(flex, flex, false) AC_PATH_PROG(flex, flex, false)
AC_PATH_PROG(bison, bison, false) AC_PATH_PROG(bison, bison, false)
NEED_PROG(sed, sed)
NEED_PROG(tar, tar)
NEED_PROG(bzip2, bzip2)
NEED_PROG(gzip, gzip)
NEED_PROG(xz, xz)
AC_PATH_PROG(dot, dot) AC_PATH_PROG(dot, dot)
AC_PATH_PROG(lsof, lsof, lsof) AC_PATH_PROG(lsof, lsof, lsof)
NEED_PROG(cat, cat) AC_SUBST(coreutils, [$(dirname $(type -p cat))])
NEED_PROG(tr, tr)
AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH],
[path of cat, mkdir, etc.]),
coreutils=$withval, coreutils=$(dirname $cat))
AC_SUBST(coreutils)
AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH], AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH],
@ -167,7 +156,8 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then
LIBS="-latomic $LIBS" LIBS="-latomic $LIBS"
fi fi
# Look for OpenSSL, a required dependency. # Look for OpenSSL, a required dependency. FIXME: this is only (maybe)
# used by S3BinaryCacheStore.
PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
@ -177,11 +167,9 @@ AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
AC_CHECK_HEADERS([bzlib.h], [true], AC_CHECK_HEADERS([bzlib.h], [true],
[AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
# Look for SQLite, a required dependency. # Look for SQLite, a required dependency.
PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"]) PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"])
# Look for libcurl, a required dependency. # Look for libcurl, a required dependency.
PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"]) PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"])
@ -204,13 +192,11 @@ PKG_CHECK_MODULES([SODIUM], [libsodium],
have_sodium=1], [have_sodium=]) have_sodium=1], [have_sodium=])
AC_SUBST(HAVE_SODIUM, [$have_sodium]) AC_SUBST(HAVE_SODIUM, [$have_sodium])
# Look for liblzma, a required dependency. # Look for liblzma, a required dependency.
PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"]) PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt], AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt],
[AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])]) [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])])
# Look for libbrotli{enc,dec}. # Look for libbrotli{enc,dec}.
PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]) PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])

View file

@ -1,29 +1,13 @@
# FIXME: remove this file?
let let
fromEnv = var: def: fromEnv = var: def:
let val = builtins.getEnv var; in let val = builtins.getEnv var; in
if val != "" then val else def; if val != "" then val else def;
in rec { in rec {
shell = "@bash@";
coreutils = "@coreutils@";
bzip2 = "@bzip2@";
gzip = "@gzip@";
xz = "@xz@";
tar = "@tar@";
tarFlags = "@tarFlags@";
tr = "@tr@";
nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@"; nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@";
nixPrefix = "@prefix@"; nixPrefix = "@prefix@";
nixLibexecDir = fromEnv "NIX_LIBEXEC_DIR" "@libexecdir@"; nixLibexecDir = fromEnv "NIX_LIBEXEC_DIR" "@libexecdir@";
nixLocalstateDir = "@localstatedir@"; nixLocalstateDir = "@localstatedir@";
nixSysconfDir = "@sysconfdir@"; nixSysconfDir = "@sysconfdir@";
nixStoreDir = fromEnv "NIX_STORE_DIR" "@storedir@"; nixStoreDir = fromEnv "NIX_STORE_DIR" "@storedir@";
# If Nix is installed in the Nix store, then automatically add it as
# a dependency to the core packages. This ensures that they work
# properly in a chroot.
chrootDeps =
if dirOf nixPrefix == builtins.storeDir then
[ (builtins.storePath nixPrefix) ]
else
[ ];
} }

View file

@ -1,39 +1,12 @@
with import <nix/config.nix>;
let
builder = builtins.toFile "unpack-channel.sh"
''
mkdir $out
cd $out
xzpat="\.xz\$"
gzpat="\.gz\$"
if [[ "$src" =~ $xzpat ]]; then
${xz} -d < $src | ${tar} xf - ${tarFlags}
elif [[ "$src" =~ $gzpat ]]; then
${gzip} -d < $src | ${tar} xf - ${tarFlags}
else
${bzip2} -d < $src | ${tar} xf - ${tarFlags}
fi
if [ * != $channelName ]; then
mv * $out/$channelName
fi
'';
in
{ name, channelName, src }: { name, channelName, src }:
derivation { derivation {
system = builtins.currentSystem; builder = "builtin:unpack-channel";
builder = shell;
args = [ "-e" builder ];
inherit name channelName src;
PATH = "${nixBinDir}:${coreutils}"; system = "builtin";
inherit name channelName src;
# No point in doing this remotely. # No point in doing this remotely.
preferLocalBuild = true; preferLocalBuild = true;
inherit chrootDeps;
} }

84
nix-rust/Cargo.lock generated Normal file
View file

@ -0,0 +1,84 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "cfg-if"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "filetime"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
"redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "libc"
version = "0.2.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "nix-rust"
version = "0.1.0"
dependencies = [
"libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
"tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "redox_syscall"
version = "0.1.56"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "tar"
version = "0.4.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
"redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
"xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "winapi"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "xattr"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
]
[metadata]
"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
"checksum filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1ff6d4dab0aa0c8e6346d46052e93b13a16cf847b54ed357087c35011048cc7d"
"checksum libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)" = "1a31a0627fdf1f6a39ec0dd577e101440b7db22672c0901fe00a9a6fbb5c24e8"
"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
"checksum tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)" = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3"
"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6"
"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
"checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c"

13
nix-rust/Cargo.toml Normal file
View file

@ -0,0 +1,13 @@
[package]
name = "nix-rust"
version = "0.1.0"
authors = ["Eelco Dolstra <edolstra@gmail.com>"]
edition = "2018"
[lib]
name = "nixrust"
crate-type = ["cdylib"]
[dependencies]
tar = "0.4"
libc = "0.2"

38
nix-rust/local.mk Normal file
View file

@ -0,0 +1,38 @@
ifeq ($(OPTIMIZE), 1)
RUST_MODE = --release
RUST_DIR = release
else
RUST_MODE =
RUST_DIR = debug
endif
libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.$(SO_EXT)
libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT)
libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl
libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ldl
ifeq ($(OS), Darwin)
libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup"
else
libnixrust_LDFLAGS_USE += -Wl,-rpath,$(abspath $(d)/target/$(RUST_DIR))
libnixrust_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$(libdir)
endif
$(libnixrust_PATH): $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml
$(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) \
$(libnixrust_BUILD_FLAGS) \
cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \
&& touch target/$(RUST_DIR)/libnixrust.$(SO_EXT)
$(libnixrust_INSTALL_PATH): $(libnixrust_PATH)
$(target-gen) cp $^ $@
ifeq ($(OS), Darwin)
install_name_tool -id $@ $@
endif
dist-files += $(d)/vendor
clean: clean-rust
clean-rust:
$(suppress) rm -rfv nix-rust/target

31
nix-rust/src/error.rs Normal file
View file

@ -0,0 +1,31 @@
#[derive(Debug)]
pub enum Error {
IOError(std::io::Error),
Misc(String),
Foreign(CppException),
}
impl From<std::io::Error> for Error {
fn from(err: std::io::Error) -> Self {
Error::IOError(err)
}
}
impl From<Error> for CppException {
fn from(err: Error) -> Self {
match err {
Error::Foreign(ex) => ex,
Error::Misc(s) => unsafe { make_error(&s) },
Error::IOError(err) => unsafe { make_error(&err.to_string()) },
}
}
}
#[repr(C)]
#[derive(Debug)]
pub struct CppException(*const libc::c_void); // == std::exception_ptr*
extern "C" {
#[allow(improper_ctypes)] // YOLO
fn make_error(s: &str) -> CppException;
}

14
nix-rust/src/foreign.rs Normal file
View file

@ -0,0 +1,14 @@
/// A wrapper around Nix's Source class that provides the Read trait.
#[repr(C)]
pub struct Source {
fun: extern "C" fn(this: *mut libc::c_void, data: &mut [u8]) -> usize,
this: *mut libc::c_void,
}
impl std::io::Read for Source {
fn read(&mut self, buf: &mut [u8]) -> std::result::Result<usize, std::io::Error> {
let n = (self.fun)(self.this, buf);
assert!(n <= buf.len());
Ok(n)
}
}

32
nix-rust/src/lib.rs Normal file
View file

@ -0,0 +1,32 @@
mod error;
mod foreign;
mod tarfile;
pub use error::Error;
pub struct CBox<T> {
pub ptr: *mut libc::c_void,
phantom: std::marker::PhantomData<T>,
}
impl<T> CBox<T> {
fn new(t: T) -> Self {
unsafe {
let size = std::mem::size_of::<T>();
let ptr = libc::malloc(size);
*(ptr as *mut T) = t; // FIXME: probably UB
Self {
ptr,
phantom: std::marker::PhantomData,
}
}
}
}
#[no_mangle]
pub extern "C" fn unpack_tarfile(
source: foreign::Source,
dest_dir: &str,
) -> CBox<Result<(), error::CppException>> {
CBox::new(tarfile::unpack_tarfile(source, dest_dir).map_err(|err| err.into()))
}

46
nix-rust/src/tarfile.rs Normal file
View file

@ -0,0 +1,46 @@
use crate::{foreign::Source, Error};
use std::fs;
use std::io;
use std::os::unix::fs::OpenOptionsExt;
use std::path::Path;
use tar::Archive;
pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> {
let dest_dir = Path::new(dest_dir);
let mut tar = Archive::new(source);
for file in tar.entries()? {
let mut file = file?;
let dest_file = dest_dir.join(file.path()?);
fs::create_dir_all(dest_file.parent().unwrap())?;
match file.header().entry_type() {
tar::EntryType::Directory => {
fs::create_dir(dest_file)?;
}
tar::EntryType::Regular => {
let mode = if file.header().mode()? & (libc::S_IXUSR as u32) == 0 {
0o666
} else {
0o777
};
let mut f = fs::OpenOptions::new()
.create(true)
.write(true)
.mode(mode)
.open(dest_file)?;
io::copy(&mut file, &mut f)?;
}
tar::EntryType::Symlink => {
std::os::unix::fs::symlink(file.header().link_name()?.unwrap(), dest_file)?;
}
tar::EntryType::XGlobalHeader | tar::EntryType::XHeader => {}
t => return Err(Error::Misc(format!("unsupported tar entry type '{:?}'", t))),
}
}
Ok(())
}

View file

@ -11,10 +11,6 @@ $logDir = $ENV{"NIX_LOG_DIR"} || "@nixlocalstatedir@/log/nix";
$confDir = $ENV{"NIX_CONF_DIR"} || "@nixsysconfdir@/nix"; $confDir = $ENV{"NIX_CONF_DIR"} || "@nixsysconfdir@/nix";
$storeDir = $ENV{"NIX_STORE_DIR"} || "@nixstoredir@"; $storeDir = $ENV{"NIX_STORE_DIR"} || "@nixstoredir@";
$bzip2 = "@bzip2@";
$xz = "@xz@";
$curl = "@curl@";
$useBindings = 1; $useBindings = 1;
%config = (); %config = ();

View file

@ -51,6 +51,7 @@ rec {
openssl pkgconfig sqlite boehmgc openssl pkgconfig sqlite boehmgc
boost boost
nlohmann_json nlohmann_json
rustc cargo
# Tests # Tests
git git

View file

@ -10,6 +10,50 @@ let
jobs = rec { jobs = rec {
# Create a "vendor" directory that contains the crates listed in
# Cargo.lock, and include it in the Nix tarball. This allows Nix
# to be built without network access.
vendoredCrates =
let
lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock);
files = map (pkg: import <nix/fetchurl.nix> {
url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download";
sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)";
}) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package);
in pkgs.runCommand "cargo-vendor-dir" {}
''
mkdir -p $out/vendor
cat > $out/vendor/config <<EOF
[source.crates-io]
replace-with = "vendored-sources"
[source.vendored-sources]
directory = "vendor"
EOF
${toString (builtins.map (file: ''
mkdir $out/vendor/tmp
tar xvf ${file} -C $out/vendor/tmp
dir=$(echo $out/vendor/tmp/*)
# Add just enough metadata to keep Cargo happy.
printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json"
# Clean up some cruft from the winapi crates. FIXME: find
# a way to remove winapi* from our dependencies.
if [[ $dir =~ /winapi ]]; then
find $dir -name "*.a" -print0 | xargs -0 rm -f --
fi
mv "$dir" $out/vendor/
rm -rf $out/vendor/tmp
'') files)}
'';
tarball = tarball =
with pkgs; with pkgs;
@ -38,6 +82,8 @@ let
distPhase = distPhase =
'' ''
cp -prd ${vendoredCrates}/vendor/ nix-rust/vendor/
runHook preDist runHook preDist
make dist make dist
mkdir -p $out/tarballs mkdir -p $out/tarballs

View file

@ -7,7 +7,7 @@ with import ./release-common.nix { inherit pkgs; };
(if useClang then clangStdenv else stdenv).mkDerivation { (if useClang then clangStdenv else stdenv).mkDerivation {
name = "nix"; name = "nix";
buildInputs = buildDeps ++ tarballDeps ++ perlDeps; buildInputs = buildDeps ++ tarballDeps ++ perlDeps ++ [ pkgs.rustfmt ];
inherit configureFlags; inherit configureFlags;

View file

@ -4,6 +4,7 @@
#include "store-api.hh" #include "store-api.hh"
#include "pathlocks.hh" #include "pathlocks.hh"
#include "hash.hh" #include "hash.hh"
#include "tarfile.hh"
#include <sys/time.h> #include <sys/time.h>
@ -164,14 +165,16 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
if (e.errNo != ENOENT) throw; if (e.errNo != ENOENT) throw;
} }
// FIXME: should pipe this, or find some better way to extract a auto source = sinkToSource([&](Sink & sink) {
// revision. RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev }); gitOptions.standardOut = &sink;
runProgram2(gitOptions);
});
Path tmpDir = createTempDir(); Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true); AutoDelete delTmpDir(tmpDir, true);
runProgram("tar", true, { "x", "-C", tmpDir }, tar); unpackTarfile(*source, tmpDir);
gitInfo.storePath = store->addToStore(name, tmpDir); gitInfo.storePath = store->addToStore(name, tmpDir);

View file

@ -3128,6 +3128,8 @@ void DerivationGoal::runChild()
builtinFetchurl(drv2, netrcData); builtinFetchurl(drv2, netrcData);
else if (drv->builder == "builtin:buildenv") else if (drv->builder == "builtin:buildenv")
builtinBuildenv(drv2); builtinBuildenv(drv2);
else if (drv->builder == "builtin:unpack-channel")
builtinUnpackChannel(drv2);
else else
throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8)); throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8));
_exit(0); _exit(0);

View file

@ -7,5 +7,6 @@ namespace nix {
// TODO: make pluggable. // TODO: make pluggable.
void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData);
void builtinBuildenv(const BasicDerivation & drv); void builtinBuildenv(const BasicDerivation & drv);
void builtinUnpackChannel(const BasicDerivation & drv);
} }

View file

@ -0,0 +1,29 @@
#include "builtins.hh"
#include "tarfile.hh"
namespace nix {
void builtinUnpackChannel(const BasicDerivation & drv)
{
auto getAttr = [&](const string & name) {
auto i = drv.env.find(name);
if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
return i->second;
};
Path out = getAttr("out");
auto channelName = getAttr("channelName");
auto src = getAttr("src");
createDirs(out);
unpackTarfile(src, out);
auto entries = readDirectory(out);
if (entries.size() != 1)
throw Error("channel tarball '%s' contains more than one file", src);
if (rename((out + "/" + entries[0].name).c_str(), (out + "/" + channelName).c_str()) == -1)
throw SysError("renaming channel directory");
}
}

View file

@ -8,6 +8,7 @@
#include "compression.hh" #include "compression.hh"
#include "pathlocks.hh" #include "pathlocks.hh"
#include "finally.hh" #include "finally.hh"
#include "tarfile.hh"
#ifdef ENABLE_S3 #ifdef ENABLE_S3
#include <aws/core/client/ClientConfiguration.h> #include <aws/core/client/ClientConfiguration.h>
@ -903,12 +904,15 @@ CachedDownloadResult Downloader::downloadCached(
unpackedStorePath = ""; unpackedStorePath = "";
} }
if (unpackedStorePath.empty()) { if (unpackedStorePath.empty()) {
printInfo(format("unpacking '%1%'...") % url); printInfo("unpacking '%s'...", url);
Path tmpDir = createTempDir(); Path tmpDir = createTempDir();
AutoDelete autoDelete(tmpDir, true); AutoDelete autoDelete(tmpDir, true);
// FIXME: this requires GNU tar for decompression. unpackTarfile(store->toRealPath(storePath), tmpDir, baseNameOf(url));
runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"}); auto members = readDirectory(tmpDir);
unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair); if (members.size() != 1)
throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
auto topDir = tmpDir + "/" + members.begin()->name;
unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
} }
replaceSymlink(unpackedStorePath, unpackedLink); replaceSymlink(unpackedStorePath, unpackedLink);
storePath = unpackedStorePath; storePath = unpackedStorePath;

View file

@ -7,3 +7,5 @@ libutil_DIR := $(d)
libutil_SOURCES := $(wildcard $(d)/*.cc) libutil_SOURCES := $(wildcard $(d)/*.cc)
libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context
libutil_LIBS = libnixrust

12
src/libutil/rust-ffi.cc Normal file
View file

@ -0,0 +1,12 @@
#include "logging.hh"
#include "rust-ffi.hh"
namespace nix {
extern "C" std::exception_ptr * make_error(rust::StringSlice s)
{
// FIXME: leak
return new std::exception_ptr(std::make_exception_ptr(Error(std::string(s.ptr, s.size))));
}
}

84
src/libutil/rust-ffi.hh Normal file
View file

@ -0,0 +1,84 @@
#include "serialise.hh"
namespace rust {
// Depending on the internal representation of Rust slices is slightly
// evil...
template<typename T>
struct Slice
{
T * ptr;
size_t size;
Slice(T * ptr, size_t size) : ptr(ptr), size(size)
{
assert(ptr);
}
};
struct StringSlice : Slice<char>
{
StringSlice(const std::string & s): Slice((char *) s.data(), s.size()) {}
};
struct Source
{
size_t (*fun)(void * source_this, rust::Slice<uint8_t> data);
nix::Source * _this;
Source(nix::Source & _this)
: fun(sourceWrapper), _this(&_this)
{}
// FIXME: how to propagate exceptions?
static size_t sourceWrapper(void * _this, rust::Slice<uint8_t> data)
{
auto n = ((nix::Source *) _this)->read(data.ptr, data.size);
return n;
}
};
/* C++ representation of Rust's Result<T, CppException>. */
template<typename T>
struct Result
{
unsigned int tag;
union {
T data;
std::exception_ptr * exc;
};
/* Rethrow the wrapped exception or return the wrapped value. */
T unwrap()
{
if (tag == 0)
return data;
else if (tag == 1)
std::rethrow_exception(*exc);
else
abort();
}
};
template<typename T>
struct CBox
{
T * ptr;
T * operator ->()
{
return ptr;
}
CBox(T * ptr) : ptr(ptr) { }
CBox(const CBox &) = delete;
CBox(CBox &&) = delete;
~CBox()
{
free(ptr);
}
};
}

View file

@ -77,7 +77,6 @@ struct BufferedSource : Source
size_t read(unsigned char * data, size_t len) override; size_t read(unsigned char * data, size_t len) override;
bool hasData(); bool hasData();
protected: protected:

36
src/libutil/tarfile.cc Normal file
View file

@ -0,0 +1,36 @@
#include "rust-ffi.hh"
#include "compression.hh"
extern "C" {
rust::Result<std::tuple<>> *
unpack_tarfile(rust::Source source, rust::StringSlice dest_dir);
}
namespace nix {
void unpackTarfile(Source & source, const Path & destDir)
{
rust::Source source2(source);
rust::CBox(unpack_tarfile(source2, destDir))->unwrap();
}
void unpackTarfile(const Path & tarFile, const Path & destDir,
std::optional<std::string> baseName)
{
if (!baseName) baseName = baseNameOf(tarFile);
auto source = sinkToSource([&](Sink & sink) {
// FIXME: look at first few bytes to determine compression type.
auto decompressor =
// FIXME: add .gz support
hasSuffix(*baseName, ".bz2") ? makeDecompressionSink("bzip2", sink) :
hasSuffix(*baseName, ".xz") ? makeDecompressionSink("xz", sink) :
makeDecompressionSink("none", sink);
readFile(tarFile, *decompressor);
decompressor->finish();
});
unpackTarfile(*source, destDir);
}
}

10
src/libutil/tarfile.hh Normal file
View file

@ -0,0 +1,10 @@
#include "serialise.hh"
namespace nix {
void unpackTarfile(Source & source, const Path & destDir);
void unpackTarfile(const Path & tarFile, const Path & destDir,
std::optional<std::string> baseName = {});
}

View file

@ -9,6 +9,7 @@
#include "legacy.hh" #include "legacy.hh"
#include "finally.hh" #include "finally.hh"
#include "progress-bar.hh" #include "progress-bar.hh"
#include "tarfile.hh"
#include <iostream> #include <iostream>
@ -192,8 +193,7 @@ static int _main(int argc, char * * argv)
if (hasSuffix(baseNameOf(uri), ".zip")) if (hasSuffix(baseNameOf(uri), ".zip"))
runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked}); runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked});
else else
// FIXME: this requires GNU tar for decompression. unpackTarfile(tmpFile, unpacked, baseNameOf(uri));
runProgram("tar", true, {"xf", tmpFile, "-C", unpacked});
/* If the archive unpacks to a single file/directory, then use /* If the archive unpacks to a single file/directory, then use
that as the top-level. */ that as the top-level. */

View file

@ -1,9 +1,7 @@
with import <nix/config.nix>;
rec { rec {
inherit shell; shell = "@bash@";
path = coreutils; path = "@coreutils@";
system = builtins.currentSystem; system = builtins.currentSystem;

View file

@ -39,4 +39,4 @@ tests-environment = NIX_REMOTE= $(bash) -e
clean-files += $(d)/common.sh clean-files += $(d)/common.sh
installcheck: $(d)/common.sh $(d)/plugins/libplugintest.$(SO_EXT) installcheck: $(d)/common.sh $(d)/config.nix $(d)/plugins/libplugintest.$(SO_EXT)