From bfa41eb6714a7e7c3956389ee063e898bd1f37ff Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Wed, 10 Aug 2016 10:44:39 -0400 Subject: [PATCH] nix-copy-closure: Implement in C++. Tests fail currently because the database is not given proper hashes in the VM --- .gitignore | 2 + Makefile | 1 + scripts/local.mk | 7 +- scripts/nix-copy-closure.in | 103 ----------------------- src/build-remote/build-remote.cc | 1 - src/build-remote/local.mk | 2 +- src/libstore/ssh-store.cc | 8 +- src/libstore/store-api.cc | 22 ++++- src/libstore/store-api.hh | 2 +- src/nix-copy-closure/local.mk | 7 ++ src/nix-copy-closure/nix-copy-closure.cc | 60 +++++++++++++ 11 files changed, 100 insertions(+), 115 deletions(-) delete mode 100755 scripts/nix-copy-closure.in create mode 100644 src/nix-copy-closure/local.mk create mode 100755 src/nix-copy-closure/nix-copy-closure.cc diff --git a/.gitignore b/.gitignore index 92f95fe1f..a8abc64ad 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,8 @@ Makefile.config # /src/nix-build/ /src/nix-build/nix-build +/src/nix-copy-closure/nix-copy-closure + /src/build-remote/build-remote # /tests/ diff --git a/Makefile b/Makefile index 14be271bb..8390c867d 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,7 @@ makefiles = \ src/nix-env/local.mk \ src/nix-daemon/local.mk \ src/nix-collect-garbage/local.mk \ + src/nix-copy-closure/local.mk \ src/nix-prefetch-url/local.mk \ src/buildenv/local.mk \ src/resolve-system-dependencies/local.mk \ diff --git a/scripts/local.mk b/scripts/local.mk index ee8ae6845..9f666bde4 100644 --- a/scripts/local.mk +++ b/scripts/local.mk @@ -1,8 +1,3 @@ -nix_bin_scripts := \ - $(d)/nix-copy-closure \ - -bin-scripts += $(nix_bin_scripts) - nix_noinst_scripts := \ $(d)/build-remote.pl \ $(d)/nix-http-export.cgi \ @@ -16,4 +11,4 @@ profiledir = $(sysconfdir)/profile.d $(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644)) $(eval $(call install-program-in, $(d)/build-remote.pl, $(libexecdir)/nix)) -clean-files += $(nix_bin_scripts) $(nix_noinst_scripts) +clean-files += $(nix_noinst_scripts) diff --git a/scripts/nix-copy-closure.in b/scripts/nix-copy-closure.in deleted file mode 100755 index af1d30919..000000000 --- a/scripts/nix-copy-closure.in +++ /dev/null @@ -1,103 +0,0 @@ -#! @perl@ -w @perlFlags@ - -use utf8; -use strict; -use Nix::SSH; -use Nix::Config; -use Nix::Store; -use Nix::CopyClosure; -use List::Util qw(sum); - -binmode STDERR, ":encoding(utf8)"; - -if (scalar @ARGV < 1) { - print STDERR < 0) { - print STDERR "copying ", scalar @missing, " missing paths from ‘$sshHost’...\n"; - writeInt(5, $to); # == cmdExportPaths - writeInt(0, $to); # obsolete - writeStrings(\@missing, $to); - importPaths(fileno($from), 1); - } - -} diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 98ccc3ddc..1ac9711a1 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -9,7 +9,6 @@ #include "shared.hh" #include "pathlocks.hh" #include "globals.hh" -#include "serve-protocol.hh" #include "serialise.hh" #include "store-api.hh" #include "derivations.hh" diff --git a/src/build-remote/local.mk b/src/build-remote/local.mk index 05b8cb451..62d5a010c 100644 --- a/src/build-remote/local.mk +++ b/src/build-remote/local.mk @@ -8,4 +8,4 @@ build-remote_LIBS = libmain libutil libformat libstore build-remote_SOURCES := $(d)/build-remote.cc -build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\"" -Isrc/nix-store +build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\"" diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 3d0159400..cce0458c6 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -39,6 +39,8 @@ private: string uri; Path key; + + bool compress; }; SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections) @@ -48,6 +50,7 @@ SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections) , socketPath((Path) tmpDir + "/ssh.sock") , uri(std::move(uri)) , key(get(params, "ssh-key", "")) + , compress(get(params, "compress", "") == "true") { /* open a connection and perform the handshake to verify all is well */ connections->get(); @@ -90,11 +93,12 @@ ref SSHStore::getFSAccessor() ref SSHStore::openConnection() { if ((pid_t) sshMaster == -1) { + auto flags = compress ? "-NMCS" : "-NMS"; sshMaster = startProcess([&]() { if (key.empty()) - execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), uri.c_str(), NULL); + execlp("ssh", "ssh", flags, socketPath.c_str(), uri.c_str(), NULL); else - execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL); + execlp("ssh", "ssh", flags, socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL); throw SysError("starting ssh master"); }); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8fdd62771..c8ca00f00 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -4,6 +4,7 @@ #include "util.hh" #include "nar-info-disk-cache.hh" #include "thread-pool.hh" +#include "derivations.hh" #include @@ -699,8 +700,27 @@ std::list> getDefaultSubstituters() } -void copyPaths(ref from, ref to, const Paths & storePaths) +void copyPaths(ref from, ref to, const Paths & storePaths, bool substitute) { + if (substitute) { + /* Filter out .drv files (we don't want to build anything). */ + PathSet paths2; + for (auto & path : storePaths) + if (!isDerivation(path)) paths2.insert(path); + unsigned long long downloadSize, narSize; + PathSet willBuild, willSubstitute, unknown; + to->queryMissing(PathSet(paths2.begin(), paths2.end()), + willBuild, willSubstitute, unknown, downloadSize, narSize); + /* FIXME: should use ensurePath(), but it only + does one path at a time. */ + if (!willSubstitute.empty()) + try { + to->buildPaths(willSubstitute); + } catch (Error & e) { + printMsg(lvlError, format("warning: %1%") % e.msg()); + } + } + std::string copiedLabel = "copied"; logger->setExpected(copiedLabel, storePaths.size()); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index ec3bf5a6f..30ee433bf 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -625,7 +625,7 @@ void removeTempRoots(); ref openStore(const std::string & uri = getEnv("NIX_REMOTE")); -void copyPaths(ref from, ref to, const Paths & storePaths); +void copyPaths(ref from, ref to, const Paths & storePaths, bool substitute = false); enum StoreType { tDaemon, diff --git a/src/nix-copy-closure/local.mk b/src/nix-copy-closure/local.mk new file mode 100644 index 000000000..42bb34dd8 --- /dev/null +++ b/src/nix-copy-closure/local.mk @@ -0,0 +1,7 @@ +programs += nix-copy-closure + +nix-copy-closure_DIR := $(d) + +nix-copy-closure_LIBS = libmain libutil libformat libstore + +nix-copy-closure_SOURCES := $(d)/nix-copy-closure.cc diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc new file mode 100755 index 000000000..b7e997ca4 --- /dev/null +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -0,0 +1,60 @@ +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +int main(int argc, char ** argv) +{ + return handleExceptions(argv[0], [&]() { + initNix(); + auto gzip = false; + auto toMode = true; + auto includeOutputs = false; + auto dryRun = false; + auto useSubstitutes = false; + auto sshHost = string{}; + auto storePaths = PathSet{}; + parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-copy-closure"); + else if (*arg == "--version") + printVersion("nix-copy-closure"); + else if (*arg == "--gzip" || *arg == "--bzip2" || *arg == "--xz") { + if (*arg != "--gzip") + printMsg(lvlError, format("Warning: ‘%1%’ is not implemented, falling back to gzip") % *arg); + gzip = true; + } else if (*arg == "--from") + toMode = false; + else if (*arg == "--to") + toMode = true; + else if (*arg == "--include-outputs") + includeOutputs = true; + else if (*arg == "--show-progress") + printMsg(lvlError, "Warning: ‘--show-progress’ is not implemented"); + else if (*arg == "--dry-run") + dryRun = true; + else if (*arg == "--use-substitutes" || *arg == "-s") + useSubstitutes = true; + else if (sshHost.empty()) + sshHost = *arg; + else + storePaths.insert(*arg); + return true; + }); + if (sshHost.empty()) + throw UsageError("no host name specified"); + + auto remoteUri = "ssh://" + sshHost + (gzip ? "?compress=true" : ""); + auto to = toMode ? openStore(remoteUri) : openStore(); + auto from = toMode ? openStore() : openStore(remoteUri); + if (includeOutputs) { + auto newPaths = PathSet{}; + for (const auto & p : storePaths) { + auto outputs = from->queryDerivationOutputs(p); + newPaths.insert(outputs.begin(), outputs.end()); + } + storePaths.insert(newPaths.begin(), newPaths.end()); + } + copyPaths(from, to, Paths(storePaths.begin(), storePaths.end()), useSubstitutes); + }); +}