Split up util.{hh,cc}

All OS and IO operations should be moved out, leaving only some misc
portable pure functions.

This is useful to avoid copious CPP when doing things like Windows and
Emscripten ports.

Newly exposed functions to break cycles:

 - `restoreSignals`
 - `updateWindowSize`
This commit is contained in:
John Ericson 2023-10-25 00:43:36 -04:00
parent 2678b51b31
commit ac89bb064a
138 changed files with 3028 additions and 2654 deletions

View file

@ -11,7 +11,6 @@
#include "derivations.hh"
#include "globals.hh"
#include "store-api.hh"
#include "util.hh"
#include "crypto.hh"
#include <sodium.h>

View file

@ -1,3 +1,6 @@
#pragma once
///@file
#include "derived-path.hh"
#include "realisation.hh"

View file

@ -2,7 +2,6 @@
#include "common-eval-args.hh"
#include "shared.hh"
#include "filetransfer.hh"
#include "util.hh"
#include "eval.hh"
#include "fetchers.hh"
#include "registry.hh"

View file

@ -1,5 +1,5 @@
#include "util.hh"
#include "editor-for.hh"
#include "environment-variables.hh"
namespace nix {

View file

@ -4,7 +4,6 @@
#include "globals.hh"
#include "installable-value.hh"
#include "outputs-spec.hh"
#include "util.hh"
#include "command.hh"
#include "attr-path.hh"
#include "common-eval-args.hh"

View file

@ -4,6 +4,7 @@
#include "installable-attr-path.hh"
#include "installable-flake.hh"
#include "outputs-spec.hh"
#include "users.hh"
#include "util.hh"
#include "command.hh"
#include "attr-path.hh"

View file

@ -1,7 +1,6 @@
#pragma once
///@file
#include "util.hh"
#include "path.hh"
#include "outputs-spec.hh"
#include "derived-path.hh"

View file

@ -1,6 +1,7 @@
#include "markdown.hh"
#include "util.hh"
#include "finally.hh"
#include "terminal.hh"
#include <sys/queue.h>
#include <lowdown.h>

View file

@ -22,6 +22,7 @@ extern "C" {
#include "repl.hh"
#include "ansicolor.hh"
#include "signals.hh"
#include "shared.hh"
#include "eval.hh"
#include "eval-cache.hh"
@ -36,6 +37,8 @@ extern "C" {
#include "globals.hh"
#include "flake/flake.hh"
#include "flake/lockfile.hh"
#include "users.hh"
#include "terminal.hh"
#include "editor-for.hh"
#include "finally.hh"
#include "markdown.hh"

View file

@ -1,6 +1,5 @@
#include "attr-path.hh"
#include "eval-inline.hh"
#include "util.hh"
namespace nix {

View file

@ -1,3 +1,4 @@
#include "users.hh"
#include "eval-cache.hh"
#include "sqlite.hh"
#include "eval.hh"

View file

@ -1,3 +1,4 @@
#include "users.hh"
#include "globals.hh"
#include "profiles.hh"
#include "eval.hh"

View file

@ -1,4 +1,6 @@
#pragma once
///@file
#include "config.hh"
namespace nix {

View file

@ -14,6 +14,7 @@
#include "print.hh"
#include "fs-input-accessor.hh"
#include "memory-input-accessor.hh"
#include "signals.hh"
#include <algorithm>
#include <chrono>

View file

@ -1,6 +1,7 @@
#include "flake.hh"
#include "users.hh"
#include "globals.hh"
#include "fetch-settings.hh"
#include "flake.hh"
#include <nlohmann/json.hpp>

View file

@ -1,3 +1,4 @@
#include "terminal.hh"
#include "flake.hh"
#include "eval.hh"
#include "eval-settings.hh"

View file

@ -1,5 +1,4 @@
#include "get-drvs.hh"
#include "util.hh"
#include "eval-inline.hh"
#include "derivations.hh"
#include "store-api.hh"

View file

@ -19,6 +19,7 @@
#include <variant>
#include "util.hh"
#include "users.hh"
#include "nixexpr.hh"
#include "eval.hh"

View file

@ -10,6 +10,7 @@
#include "path-references.hh"
#include "store-api.hh"
#include "util.hh"
#include "processes.hh"
#include "value-to-json.hh"
#include "value-to-xml.hh"
#include "primops.hh"

View file

@ -1,5 +1,4 @@
#include "search-path.hh"
#include "util.hh"
namespace nix {

View file

@ -1,7 +1,7 @@
#include "value-to-json.hh"
#include "eval-inline.hh"
#include "util.hh"
#include "store-api.hh"
#include "signals.hh"
#include <cstdlib>
#include <iomanip>

View file

@ -1,7 +1,7 @@
#include "value-to-xml.hh"
#include "xml-writer.hh"
#include "eval-inline.hh"
#include "util.hh"
#include "signals.hh"
#include <cstdlib>

View file

@ -1,3 +1,4 @@
#include "util.hh"
#include "value/context.hh"
#include <optional>

View file

@ -1,7 +1,6 @@
#pragma once
///@file
#include "util.hh"
#include "comparator.hh"
#include "derived-path.hh"
#include "variant-wrapper.hh"

View file

@ -1,4 +1,5 @@
#include "cache.hh"
#include "users.hh"
#include "sqlite.hh"
#include "sync.hh"
#include "store-api.hh"

View file

@ -3,7 +3,6 @@
#include "types.hh"
#include "config.hh"
#include "util.hh"
#include <map>
#include <limits>

View file

@ -1,11 +1,12 @@
#include "fetchers.hh"
#include "users.hh"
#include "cache.hh"
#include "globals.hh"
#include "tarfile.hh"
#include "store-api.hh"
#include "url-parts.hh"
#include "pathlocks.hh"
#include "util.hh"
#include "processes.hh"
#include "git.hh"
#include "fetch-settings.hh"

View file

@ -1,8 +1,10 @@
#pragma once
///@file
#include "source-accessor.hh"
#include "ref.hh"
#include "types.hh"
#include "file-system.hh"
#include "repair-flag.hh"
#include "content-address.hh"

View file

@ -1,4 +1,6 @@
#include "fetchers.hh"
#include "processes.hh"
#include "users.hh"
#include "cache.hh"
#include "globals.hh"
#include "tarfile.hh"

View file

@ -1,6 +1,6 @@
#include "registry.hh"
#include "tarball.hh"
#include "util.hh"
#include "users.hh"
#include "globals.hh"
#include "store-api.hh"
#include "local-fs-store.hh"

View file

@ -1,7 +1,9 @@
#include "common-args.hh"
#include "args/root.hh"
#include "globals.hh"
#include "logging.hh"
#include "loggers.hh"
#include "util.hh"
namespace nix {

View file

@ -1,6 +1,6 @@
#include "loggers.hh"
#include "environment-variables.hh"
#include "progress-bar.hh"
#include "util.hh"
namespace nix {

View file

@ -1,5 +1,5 @@
#include "progress-bar.hh"
#include "util.hh"
#include "terminal.hh"
#include "sync.hh"
#include "store-api.hh"
#include "names.hh"

View file

@ -1,10 +1,11 @@
#include "globals.hh"
#include "current-process.hh"
#include "shared.hh"
#include "store-api.hh"
#include "gc-store.hh"
#include "util.hh"
#include "loggers.hh"
#include "progress-bar.hh"
#include "signals.hh"
#include <algorithm>
#include <cctype>

View file

@ -1,7 +1,7 @@
#pragma once
///@file
#include "util.hh"
#include "processes.hh"
#include "args.hh"
#include "args/root.hh"
#include "common-args.hh"

View file

@ -11,6 +11,7 @@
#include "nar-accessor.hh"
#include "thread-pool.hh"
#include "callback.hh"
#include "signals.hh"
#include <chrono>
#include <future>

View file

@ -0,0 +1,37 @@
#include "child.hh"
#include "current-process.hh"
#include "logging.hh"
#include <fcntl.h>
#include <unistd.h>
namespace nix {
void commonChildInit()
{
logger = makeSimpleLogger();
const static std::string pathNullDevice = "/dev/null";
restoreProcessContext(false);
/* Put the child in a separate session (and thus a separate
process group) so that it has no controlling terminal (meaning
that e.g. ssh cannot open /dev/tty) and it doesn't receive
terminal signals. */
if (setsid() == -1)
throw SysError("creating a new session");
/* Dup stderr to stdout. */
if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
throw SysError("cannot dup stderr into stdout");
/* Reroute stdin to /dev/null. */
int fdDevNull = open(pathNullDevice.c_str(), O_RDWR);
if (fdDevNull == -1)
throw SysError("cannot open '%1%'", pathNullDevice);
if (dup2(fdDevNull, STDIN_FILENO) == -1)
throw SysError("cannot dup null device into stdin");
close(fdDevNull);
}
}

View file

@ -0,0 +1,11 @@
#pragma once
///@file
namespace nix {
/**
* Common initialisation performed in child processes.
*/
void commonChildInit();
}

View file

@ -1,5 +1,7 @@
#include "globals.hh"
#include "hook-instance.hh"
#include "file-system.hh"
#include "child.hh"
namespace nix {

View file

@ -3,6 +3,7 @@
#include "logging.hh"
#include "serialise.hh"
#include "processes.hh"
namespace nix {

View file

@ -15,7 +15,10 @@
#include "json-utils.hh"
#include "cgroup.hh"
#include "personality.hh"
#include "current-process.hh"
#include "namespaces.hh"
#include "child.hh"
#include "unix-domain-socket.hh"
#include <regex>
#include <queue>

View file

@ -3,6 +3,7 @@
#include "derivation-goal.hh"
#include "local-store.hh"
#include "processes.hh"
namespace nix {

View file

@ -4,6 +4,7 @@
#include "drv-output-substitution-goal.hh"
#include "local-derivation-goal.hh"
#include "hook-instance.hh"
#include "signals.hh"
#include <poll.h>

View file

@ -1,5 +1,4 @@
#include "serialise.hh"
#include "util.hh"
#include "path-with-outputs.hh"
#include "store-api.hh"
#include "build-result.hh"

View file

@ -1,4 +1,5 @@
#include "crypto.hh"
#include "file-system.hh"
#include "util.hh"
#include "globals.hh"

View file

@ -1,4 +1,5 @@
#include "derived-path-map.hh"
#include "util.hh"
namespace nix {

View file

@ -1,4 +1,5 @@
#pragma once
///@file
#include "types.hh"
#include "derived-path.hh"

View file

@ -1,10 +1,10 @@
#pragma once
///@file
#include "util.hh"
#include "path.hh"
#include "outputs-spec.hh"
#include "comparator.hh"
#include "config.hh"
#include <variant>

View file

@ -1,11 +1,12 @@
#include "filetransfer.hh"
#include "util.hh"
#include "namespaces.hh"
#include "globals.hh"
#include "store-api.hh"
#include "s3.hh"
#include "compression.hh"
#include "finally.hh"
#include "callback.hh"
#include "signals.hh"
#if ENABLE_S3
#include <aws/core/client/ClientConfiguration.h>

View file

@ -2,6 +2,13 @@
#include "globals.hh"
#include "local-store.hh"
#include "finally.hh"
#include "unix-domain-socket.hh"
#include "signals.hh"
#if !defined(__linux__)
// For shelling out to lsof
# include "processes.hh"
#endif
#include <functional>
#include <queue>

View file

@ -1,7 +1,8 @@
#include "globals.hh"
#include "util.hh"
#include "current-process.hh"
#include "archive.hh"
#include "args.hh"
#include "users.hh"
#include "abstract-setting-to-json.hh"
#include "compute-levels.hh"
@ -17,9 +18,13 @@
#include <sodium/core.h>
#ifdef __GLIBC__
#include <gnu/lib-names.h>
#include <nss.h>
#include <dlfcn.h>
# include <gnu/lib-names.h>
# include <nss.h>
# include <dlfcn.h>
#endif
#if __APPLE__
# include "processes.hh"
#endif
#include "config-impl.hh"

View file

@ -3,7 +3,7 @@
#include "types.hh"
#include "config.hh"
#include "util.hh"
#include "environment-variables.hh"
#include "experimental-features.hh"
#include <map>

View file

@ -10,6 +10,7 @@
#include "topo-sort.hh"
#include "finally.hh"
#include "compression.hh"
#include "signals.hh"
#include <iostream>
#include <algorithm>

View file

@ -7,7 +7,6 @@
#include "store-api.hh"
#include "indirect-root-store.hh"
#include "sync.hh"
#include "util.hh"
#include <chrono>
#include <future>

View file

@ -1,4 +1,5 @@
#include "lock.hh"
#include "file-system.hh"
#include "globals.hh"
#include "pathlocks.hh"

View file

@ -1,5 +1,4 @@
#include "machines.hh"
#include "util.hh"
#include "globals.hh"
#include "store-api.hh"

View file

@ -1,4 +1,5 @@
#include "nar-info-disk-cache.hh"
#include "users.hh"
#include "sync.hh"
#include "sqlite.hh"
#include "globals.hh"

View file

@ -1,6 +1,6 @@
#include "util.hh"
#include "local-store.hh"
#include "globals.hh"
#include "signals.hh"
#include <cstdlib>
#include <cstring>

View file

@ -1,6 +1,5 @@
#include "path-references.hh"
#include "hash.hh"
#include "util.hh"
#include "archive.hh"
#include <map>

View file

@ -1,4 +1,5 @@
#pragma once
///@file
#include "references.hh"
#include "path.hh"

View file

@ -1,6 +1,7 @@
#include "pathlocks.hh"
#include "util.hh"
#include "sync.hh"
#include "signals.hh"
#include <cerrno>
#include <cstdlib>

View file

@ -1,7 +1,7 @@
#pragma once
///@file
#include "util.hh"
#include "file-descriptor.hh"
namespace nix {

View file

@ -1,7 +1,7 @@
#include "profiles.hh"
#include "store-api.hh"
#include "local-fs-store.hh"
#include "util.hh"
#include "users.hh"
#include <sys/types.h>
#include <sys/stat.h>

View file

@ -1,3 +1,6 @@
#pragma once
///@file
#include "remote-store.hh"
#include "worker-protocol.hh"
#include "pool.hh"

View file

@ -1,5 +1,4 @@
#include "serialise.hh"
#include "util.hh"
#include "path-with-outputs.hh"
#include "store-api.hh"
#include "build-result.hh"

View file

@ -2,6 +2,7 @@
#include "globals.hh"
#include "util.hh"
#include "url.hh"
#include "signals.hh"
#include <sqlite3.h>

View file

@ -1,5 +1,8 @@
#include "ssh.hh"
#include "finally.hh"
#include "current-process.hh"
#include "environment-variables.hh"
#include "util.hh"
namespace nix {

View file

@ -1,8 +1,9 @@
#pragma once
///@file
#include "util.hh"
#include "sync.hh"
#include "processes.hh"
#include "file-system.hh"
namespace nix {

View file

@ -14,6 +14,8 @@
// FIXME this should not be here, see TODO below on
// `addMultipleToStore`.
#include "worker-protocol.hh"
#include "signals.hh"
#include "users.hh"
#include <nlohmann/json.hpp>
#include <regex>

View file

@ -1,5 +1,7 @@
#include "machines.hh"
#include "globals.hh"
#include "file-system.hh"
#include "util.hh"
#include <gmock/gmock-matchers.h>

View file

@ -1,3 +1,6 @@
#pragma once
///@file
#include <nlohmann/json.hpp>
#include <gtest/gtest.h>

View file

@ -1,4 +1,5 @@
#include "uds-remote-store.hh"
#include "unix-domain-socket.hh"
#include "worker-protocol.hh"
#include <sys/types.h>

View file

@ -1,5 +1,4 @@
#include "serialise.hh"
#include "util.hh"
#include "path-with-outputs.hh"
#include "store-api.hh"
#include "build-result.hh"

View file

@ -6,9 +6,10 @@
#include <strings.h> // for strcasecmp
#include "archive.hh"
#include "util.hh"
#include "config.hh"
#include "posix-source-accessor.hh"
#include "file-system.hh"
#include "signals.hh"
namespace nix {

View file

@ -1,6 +1,9 @@
#include "args.hh"
#include "args/root.hh"
#include "hash.hh"
#include "environment-variables.hh"
#include "signals.hh"
#include "users.hh"
#include "json-utils.hh"
#include <glob.h>

View file

@ -2,12 +2,15 @@
///@file
#include <iostream>
#include <functional>
#include <map>
#include <memory>
#include <optional>
#include <nlohmann/json_fwd.hpp>
#include "util.hh"
#include "types.hh"
#include "experimental-features.hh"
namespace nix {

View file

@ -1,5 +1,5 @@
#include "canon-path.hh"
#include "util.hh"
#include "file-system.hh"
namespace nix {

View file

@ -2,6 +2,7 @@
#include "cgroup.hh"
#include "util.hh"
#include "file-system.hh"
#include "finally.hh"
#include <chrono>

View file

@ -1,6 +1,6 @@
#include "compression.hh"
#include "signals.hh"
#include "tarfile.hh"
#include "util.hh"
#include "finally.hh"
#include "logging.hh"

View file

@ -2,6 +2,8 @@
#include "args.hh"
#include "abstract-setting-to-json.hh"
#include "experimental-features.hh"
#include "util.hh"
#include "file-system.hh"
#include "config-impl.hh"

View file

@ -0,0 +1,110 @@
#include "current-process.hh"
#include "namespaces.hh"
#include "util.hh"
#include "finally.hh"
#include "file-system.hh"
#include "processes.hh"
#include "signals.hh"
#ifdef __APPLE__
# include <mach-o/dyld.h>
#endif
#if __linux__
# include <mutex>
# include <sys/resource.h>
# include "cgroup.hh"
#endif
#include <sys/mount.h>
namespace nix {
unsigned int getMaxCPU()
{
#if __linux__
try {
auto cgroupFS = getCgroupFS();
if (!cgroupFS) return 0;
auto cgroups = getCgroups("/proc/self/cgroup");
auto cgroup = cgroups[""];
if (cgroup == "") return 0;
auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max";
auto cpuMax = readFile(cpuFile);
auto cpuMaxParts = tokenizeString<std::vector<std::string>>(cpuMax, " \n");
auto quota = cpuMaxParts[0];
auto period = cpuMaxParts[1];
if (quota != "max")
return std::ceil(std::stoi(quota) / std::stof(period));
} catch (Error &) { ignoreException(lvlDebug); }
#endif
return 0;
}
//////////////////////////////////////////////////////////////////////
#if __linux__
rlim_t savedStackSize = 0;
#endif
void setStackSize(size_t stackSize)
{
#if __linux__
struct rlimit limit;
if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) {
savedStackSize = limit.rlim_cur;
limit.rlim_cur = stackSize;
setrlimit(RLIMIT_STACK, &limit);
}
#endif
}
void restoreProcessContext(bool restoreMounts)
{
restoreSignals();
if (restoreMounts) {
restoreMountNamespace();
}
#if __linux__
if (savedStackSize) {
struct rlimit limit;
if (getrlimit(RLIMIT_STACK, &limit) == 0) {
limit.rlim_cur = savedStackSize;
setrlimit(RLIMIT_STACK, &limit);
}
}
#endif
}
//////////////////////////////////////////////////////////////////////
std::optional<Path> getSelfExe()
{
static auto cached = []() -> std::optional<Path>
{
#if __linux__
return readLink("/proc/self/exe");
#elif __APPLE__
char buf[1024];
uint32_t size = sizeof(buf);
if (_NSGetExecutablePath(buf, &size) == 0)
return buf;
else
return std::nullopt;
#else
return std::nullopt;
#endif
}();
return cached;
}
}

View file

@ -0,0 +1,34 @@
#pragma once
///@file
#include <optional>
#include "types.hh"
namespace nix {
/**
* If cgroups are active, attempt to calculate the number of CPUs available.
* If cgroups are unavailable or if cpu.max is set to "max", return 0.
*/
unsigned int getMaxCPU();
/**
* Change the stack size.
*/
void setStackSize(size_t stackSize);
/**
* Restore the original inherited Unix process context (such as signal
* masks, stack size).
* See startSignalHandlerThread(), saveSignalMask().
*/
void restoreProcessContext(bool restoreMounts = true);
/**
* @return the path of the current executable.
*/
std::optional<Path> getSelfExe();
}

View file

@ -0,0 +1,49 @@
#include "util.hh"
#include "environment-variables.hh"
extern char * * environ __attribute__((weak));
namespace nix {
std::optional<std::string> getEnv(const std::string & key)
{
char * value = getenv(key.c_str());
if (!value) return {};
return std::string(value);
}
std::optional<std::string> getEnvNonEmpty(const std::string & key) {
auto value = getEnv(key);
if (value == "") return {};
return value;
}
std::map<std::string, std::string> getEnv()
{
std::map<std::string, std::string> env;
for (size_t i = 0; environ[i]; ++i) {
auto s = environ[i];
auto eq = strchr(s, '=');
if (!eq)
// invalid env, just keep going
continue;
env.emplace(std::string(s, eq), std::string(eq + 1));
}
return env;
}
void clearEnv()
{
for (auto & name : getEnv())
unsetenv(name.first.c_str());
}
void replaceEnv(const std::map<std::string, std::string> & newEnv)
{
clearEnv();
for (auto & newEnvVar : newEnv)
setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1);
}
}

View file

@ -0,0 +1,41 @@
#pragma once
/**
* @file
*
* Utilities for working with the current process's environment
* variables.
*/
#include <optional>
#include "types.hh"
namespace nix {
/**
* @return an environment variable.
*/
std::optional<std::string> getEnv(const std::string & key);
/**
* @return a non empty environment variable. Returns nullopt if the env
* variable is set to ""
*/
std::optional<std::string> getEnvNonEmpty(const std::string & key);
/**
* Get the entire environment.
*/
std::map<std::string, std::string> getEnv();
/**
* Clear the environment.
*/
void clearEnv();
/**
* Replace the entire environment with the given one.
*/
void replaceEnv(const std::map<std::string, std::string> & newEnv);
}

View file

@ -1,4 +1,7 @@
#include "error.hh"
#include "environment-variables.hh"
#include "signals.hh"
#include "terminal.hh"
#include <iostream>
#include <optional>

View file

@ -0,0 +1,254 @@
#include "file-system.hh"
#include "signals.hh"
#include "finally.hh"
#include "serialise.hh"
#include <fcntl.h>
#include <unistd.h>
namespace nix {
std::string readFile(int fd)
{
struct stat st;
if (fstat(fd, &st) == -1)
throw SysError("statting file");
return drainFD(fd, true, st.st_size);
}
void readFull(int fd, char * buf, size_t count)
{
while (count) {
checkInterrupt();
ssize_t res = read(fd, buf, count);
if (res == -1) {
if (errno == EINTR) continue;
throw SysError("reading from file");
}
if (res == 0) throw EndOfFile("unexpected end-of-file");
count -= res;
buf += res;
}
}
void writeFull(int fd, std::string_view s, bool allowInterrupts)
{
while (!s.empty()) {
if (allowInterrupts) checkInterrupt();
ssize_t res = write(fd, s.data(), s.size());
if (res == -1 && errno != EINTR)
throw SysError("writing to file");
if (res > 0)
s.remove_prefix(res);
}
}
std::string readLine(int fd)
{
std::string s;
while (1) {
checkInterrupt();
char ch;
// FIXME: inefficient
ssize_t rd = read(fd, &ch, 1);
if (rd == -1) {
if (errno != EINTR)
throw SysError("reading a line");
} else if (rd == 0)
throw EndOfFile("unexpected EOF reading a line");
else {
if (ch == '\n') return s;
s += ch;
}
}
}
void writeLine(int fd, std::string s)
{
s += '\n';
writeFull(fd, s);
}
std::string drainFD(int fd, bool block, const size_t reserveSize)
{
// the parser needs two extra bytes to append terminating characters, other users will
// not care very much about the extra memory.
StringSink sink(reserveSize + 2);
drainFD(fd, sink, block);
return std::move(sink.s);
}
void drainFD(int fd, Sink & sink, bool block)
{
// silence GCC maybe-uninitialized warning in finally
int saved = 0;
if (!block) {
saved = fcntl(fd, F_GETFL);
if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1)
throw SysError("making file descriptor non-blocking");
}
Finally finally([&]() {
if (!block) {
if (fcntl(fd, F_SETFL, saved) == -1)
throw SysError("making file descriptor blocking");
}
});
std::vector<unsigned char> buf(64 * 1024);
while (1) {
checkInterrupt();
ssize_t rd = read(fd, buf.data(), buf.size());
if (rd == -1) {
if (!block && (errno == EAGAIN || errno == EWOULDBLOCK))
break;
if (errno != EINTR)
throw SysError("reading from file");
}
else if (rd == 0) break;
else sink({(char *) buf.data(), (size_t) rd});
}
}
//////////////////////////////////////////////////////////////////////
AutoCloseFD::AutoCloseFD() : fd{-1} {}
AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {}
AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd}
{
that.fd = -1;
}
AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that)
{
close();
fd = that.fd;
that.fd = -1;
return *this;
}
AutoCloseFD::~AutoCloseFD()
{
try {
close();
} catch (...) {
ignoreException();
}
}
int AutoCloseFD::get() const
{
return fd;
}
void AutoCloseFD::close()
{
if (fd != -1) {
if (::close(fd) == -1)
/* This should never happen. */
throw SysError("closing file descriptor %1%", fd);
fd = -1;
}
}
void AutoCloseFD::fsync()
{
if (fd != -1) {
int result;
#if __APPLE__
result = ::fcntl(fd, F_FULLFSYNC);
#else
result = ::fsync(fd);
#endif
if (result == -1)
throw SysError("fsync file descriptor %1%", fd);
}
}
AutoCloseFD::operator bool() const
{
return fd != -1;
}
int AutoCloseFD::release()
{
int oldFD = fd;
fd = -1;
return oldFD;
}
void Pipe::create()
{
int fds[2];
#if HAVE_PIPE2
if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe");
#else
if (pipe(fds) != 0) throw SysError("creating pipe");
closeOnExec(fds[0]);
closeOnExec(fds[1]);
#endif
readSide = fds[0];
writeSide = fds[1];
}
void Pipe::close()
{
readSide.close();
writeSide.close();
}
//////////////////////////////////////////////////////////////////////
void closeMostFDs(const std::set<int> & exceptions)
{
#if __linux__
try {
for (auto & s : readDirectory("/proc/self/fd")) {
auto fd = std::stoi(s.name);
if (!exceptions.count(fd)) {
debug("closing leaked FD %d", fd);
close(fd);
}
}
return;
} catch (SysError &) {
}
#endif
int maxFD = 0;
maxFD = sysconf(_SC_OPEN_MAX);
for (int fd = 0; fd < maxFD; ++fd)
if (!exceptions.count(fd))
close(fd); /* ignore result */
}
void closeOnExec(int fd)
{
int prev;
if ((prev = fcntl(fd, F_GETFD, 0)) == -1 ||
fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1)
throw SysError("setting close-on-exec flag");
}
}

View file

@ -0,0 +1,84 @@
#pragma once
///@file
#include "types.hh"
#include "error.hh"
namespace nix {
struct Sink;
struct Source;
/**
* Read the contents of a resource into a string.
*/
std::string readFile(int fd);
/**
* Wrappers arount read()/write() that read/write exactly the
* requested number of bytes.
*/
void readFull(int fd, char * buf, size_t count);
void writeFull(int fd, std::string_view s, bool allowInterrupts = true);
/**
* Read a line from a file descriptor.
*/
std::string readLine(int fd);
/**
* Write a line to a file descriptor.
*/
void writeLine(int fd, std::string s);
/**
* Read a file descriptor until EOF occurs.
*/
std::string drainFD(int fd, bool block = true, const size_t reserveSize=0);
void drainFD(int fd, Sink & sink, bool block = true);
/**
* Automatic cleanup of resources.
*/
class AutoCloseFD
{
int fd;
public:
AutoCloseFD();
AutoCloseFD(int fd);
AutoCloseFD(const AutoCloseFD & fd) = delete;
AutoCloseFD(AutoCloseFD&& fd);
~AutoCloseFD();
AutoCloseFD& operator =(const AutoCloseFD & fd) = delete;
AutoCloseFD& operator =(AutoCloseFD&& fd);
int get() const;
explicit operator bool() const;
int release();
void close();
void fsync();
};
class Pipe
{
public:
AutoCloseFD readSide, writeSide;
void create();
void close();
};
/**
* Close all file descriptors except those listed in the given set.
* Good practice in child processes.
*/
void closeMostFDs(const std::set<int> & exceptions);
/**
* Set the close-on-exec flag for the given file descriptor.
*/
void closeOnExec(int fd);
MakeError(EndOfFile, Error);
}

647
src/libutil/file-system.cc Normal file
View file

@ -0,0 +1,647 @@
#include "environment-variables.hh"
#include "file-system.hh"
#include "signals.hh"
#include "finally.hh"
#include "serialise.hh"
#include <atomic>
#include <cerrno>
#include <climits>
#include <cstdio>
#include <cstdlib>
#include <sstream>
#include <filesystem>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/time.h>
#include <unistd.h>
namespace fs = std::filesystem;
namespace nix {
Path absPath(Path path, std::optional<PathView> dir, bool resolveSymlinks)
{
if (path[0] != '/') {
if (!dir) {
#ifdef __GNU__
/* GNU (aka. GNU/Hurd) doesn't have any limitation on path
lengths and doesn't define `PATH_MAX'. */
char *buf = getcwd(NULL, 0);
if (buf == NULL)
#else
char buf[PATH_MAX];
if (!getcwd(buf, sizeof(buf)))
#endif
throw SysError("cannot get cwd");
path = concatStrings(buf, "/", path);
#ifdef __GNU__
free(buf);
#endif
} else
path = concatStrings(*dir, "/", path);
}
return canonPath(path, resolveSymlinks);
}
Path canonPath(PathView path, bool resolveSymlinks)
{
assert(path != "");
std::string s;
s.reserve(256);
if (path[0] != '/')
throw Error("not an absolute path: '%1%'", path);
std::string temp;
/* Count the number of times we follow a symlink and stop at some
arbitrary (but high) limit to prevent infinite loops. */
unsigned int followCount = 0, maxFollow = 1024;
while (1) {
/* Skip slashes. */
while (!path.empty() && path[0] == '/') path.remove_prefix(1);
if (path.empty()) break;
/* Ignore `.'. */
if (path == "." || path.substr(0, 2) == "./")
path.remove_prefix(1);
/* If `..', delete the last component. */
else if (path == ".." || path.substr(0, 3) == "../")
{
if (!s.empty()) s.erase(s.rfind('/'));
path.remove_prefix(2);
}
/* Normal component; copy it. */
else {
s += '/';
if (const auto slash = path.find('/'); slash == std::string::npos) {
s += path;
path = {};
} else {
s += path.substr(0, slash);
path = path.substr(slash);
}
/* If s points to a symlink, resolve it and continue from there */
if (resolveSymlinks && isLink(s)) {
if (++followCount >= maxFollow)
throw Error("infinite symlink recursion in path '%1%'", path);
temp = concatStrings(readLink(s), path);
path = temp;
if (!temp.empty() && temp[0] == '/') {
s.clear(); /* restart for symlinks pointing to absolute path */
} else {
s = dirOf(s);
if (s == "/") { // we dont want trailing slashes here, which dirOf only produces if s = /
s.clear();
}
}
}
}
}
return s.empty() ? "/" : std::move(s);
}
Path dirOf(const PathView path)
{
Path::size_type pos = path.rfind('/');
if (pos == std::string::npos)
return ".";
return pos == 0 ? "/" : Path(path, 0, pos);
}
std::string_view baseNameOf(std::string_view path)
{
if (path.empty())
return "";
auto last = path.size() - 1;
if (path[last] == '/' && last > 0)
last -= 1;
auto pos = path.rfind('/', last);
if (pos == std::string::npos)
pos = 0;
else
pos += 1;
return path.substr(pos, last - pos + 1);
}
bool isInDir(std::string_view path, std::string_view dir)
{
return path.substr(0, 1) == "/"
&& path.substr(0, dir.size()) == dir
&& path.size() >= dir.size() + 2
&& path[dir.size()] == '/';
}
bool isDirOrInDir(std::string_view path, std::string_view dir)
{
return path == dir || isInDir(path, dir);
}
struct stat stat(const Path & path)
{
struct stat st;
if (stat(path.c_str(), &st))
throw SysError("getting status of '%1%'", path);
return st;
}
struct stat lstat(const Path & path)
{
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError("getting status of '%1%'", path);
return st;
}
bool pathExists(const Path & path)
{
int res;
struct stat st;
res = lstat(path.c_str(), &st);
if (!res) return true;
if (errno != ENOENT && errno != ENOTDIR)
throw SysError("getting status of %1%", path);
return false;
}
bool pathAccessible(const Path & path)
{
try {
return pathExists(path);
} catch (SysError & e) {
// swallow EPERM
if (e.errNo == EPERM) return false;
throw;
}
}
Path readLink(const Path & path)
{
checkInterrupt();
std::vector<char> buf;
for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) {
buf.resize(bufSize);
ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize);
if (rlSize == -1)
if (errno == EINVAL)
throw Error("'%1%' is not a symlink", path);
else
throw SysError("reading symbolic link '%1%'", path);
else if (rlSize < bufSize)
return std::string(buf.data(), rlSize);
}
}
bool isLink(const Path & path)
{
struct stat st = lstat(path);
return S_ISLNK(st.st_mode);
}
DirEntries readDirectory(DIR *dir, const Path & path)
{
DirEntries entries;
entries.reserve(64);
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir)) { /* sic */
checkInterrupt();
std::string name = dirent->d_name;
if (name == "." || name == "..") continue;
entries.emplace_back(name, dirent->d_ino,
#ifdef HAVE_STRUCT_DIRENT_D_TYPE
dirent->d_type
#else
DT_UNKNOWN
#endif
);
}
if (errno) throw SysError("reading directory '%1%'", path);
return entries;
}
DirEntries readDirectory(const Path & path)
{
AutoCloseDir dir(opendir(path.c_str()));
if (!dir) throw SysError("opening directory '%1%'", path);
return readDirectory(dir.get(), path);
}
unsigned char getFileType(const Path & path)
{
struct stat st = lstat(path);
if (S_ISDIR(st.st_mode)) return DT_DIR;
if (S_ISLNK(st.st_mode)) return DT_LNK;
if (S_ISREG(st.st_mode)) return DT_REG;
return DT_UNKNOWN;
}
std::string readFile(const Path & path)
{
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
if (!fd)
throw SysError("opening file '%1%'", path);
return readFile(fd.get());
}
void readFile(const Path & path, Sink & sink)
{
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
if (!fd)
throw SysError("opening file '%s'", path);
drainFD(fd.get(), sink);
}
void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync)
{
AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
if (!fd)
throw SysError("opening file '%1%'", path);
try {
writeFull(fd.get(), s);
} catch (Error & e) {
e.addTrace({}, "writing file '%1%'", path);
throw;
}
if (sync)
fd.fsync();
// Explicitly close to make sure exceptions are propagated.
fd.close();
if (sync)
syncParent(path);
}
void writeFile(const Path & path, Source & source, mode_t mode, bool sync)
{
AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
if (!fd)
throw SysError("opening file '%1%'", path);
std::vector<char> buf(64 * 1024);
try {
while (true) {
try {
auto n = source.read(buf.data(), buf.size());
writeFull(fd.get(), {buf.data(), n});
} catch (EndOfFile &) { break; }
}
} catch (Error & e) {
e.addTrace({}, "writing file '%1%'", path);
throw;
}
if (sync)
fd.fsync();
// Explicitly close to make sure exceptions are propagated.
fd.close();
if (sync)
syncParent(path);
}
void syncParent(const Path & path)
{
AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0);
if (!fd)
throw SysError("opening file '%1%'", path);
fd.fsync();
}
static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed)
{
checkInterrupt();
std::string name(baseNameOf(path));
struct stat st;
if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) {
if (errno == ENOENT) return;
throw SysError("getting status of '%1%'", path);
}
if (!S_ISDIR(st.st_mode)) {
/* We are about to delete a file. Will it likely free space? */
switch (st.st_nlink) {
/* Yes: last link. */
case 1:
bytesFreed += st.st_size;
break;
/* Maybe: yes, if 'auto-optimise-store' or manual optimisation
was performed. Instead of checking for real let's assume
it's an optimised file and space will be freed.
In worst case we will double count on freed space for files
with exactly two hardlinks for unoptimised packages.
*/
case 2:
bytesFreed += st.st_size;
break;
/* No: 3+ links. */
default:
break;
}
}
if (S_ISDIR(st.st_mode)) {
/* Make the directory accessible. */
const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR;
if ((st.st_mode & PERM_MASK) != PERM_MASK) {
if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1)
throw SysError("chmod '%1%'", path);
}
int fd = openat(parentfd, path.c_str(), O_RDONLY);
if (fd == -1)
throw SysError("opening directory '%1%'", path);
AutoCloseDir dir(fdopendir(fd));
if (!dir)
throw SysError("opening directory '%1%'", path);
for (auto & i : readDirectory(dir.get(), path))
_deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed);
}
int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0;
if (unlinkat(parentfd, name.c_str(), flags) == -1) {
if (errno == ENOENT) return;
throw SysError("cannot unlink '%1%'", path);
}
}
static void _deletePath(const Path & path, uint64_t & bytesFreed)
{
Path dir = dirOf(path);
if (dir == "")
dir = "/";
AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)};
if (!dirfd) {
if (errno == ENOENT) return;
throw SysError("opening directory '%1%'", path);
}
_deletePath(dirfd.get(), path, bytesFreed);
}
void deletePath(const Path & path)
{
uint64_t dummy;
deletePath(path, dummy);
}
Paths createDirs(const Path & path)
{
Paths created;
if (path == "/") return created;
struct stat st;
if (lstat(path.c_str(), &st) == -1) {
created = createDirs(dirOf(path));
if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST)
throw SysError("creating directory '%1%'", path);
st = lstat(path);
created.push_back(path);
}
if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1)
throw SysError("statting symlink '%1%'", path);
if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path);
return created;
}
void deletePath(const Path & path, uint64_t & bytesFreed)
{
//Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path);
bytesFreed = 0;
_deletePath(path, bytesFreed);
}
//////////////////////////////////////////////////////////////////////
AutoDelete::AutoDelete() : del{false} {}
AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p)
{
del = true;
this->recursive = recursive;
}
AutoDelete::~AutoDelete()
{
try {
if (del) {
if (recursive)
deletePath(path);
else {
if (remove(path.c_str()) == -1)
throw SysError("cannot unlink '%1%'", path);
}
}
} catch (...) {
ignoreException();
}
}
void AutoDelete::cancel()
{
del = false;
}
void AutoDelete::reset(const Path & p, bool recursive) {
path = p;
this->recursive = recursive;
del = true;
}
//////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////
static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
std::atomic<unsigned int> & counter)
{
tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
if (includePid)
return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++);
else
return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++);
}
Path createTempDir(const Path & tmpRoot, const Path & prefix,
bool includePid, bool useGlobalCounter, mode_t mode)
{
static std::atomic<unsigned int> globalCounter = 0;
std::atomic<unsigned int> localCounter = 0;
auto & counter(useGlobalCounter ? globalCounter : localCounter);
while (1) {
checkInterrupt();
Path tmpDir = tempName(tmpRoot, prefix, includePid, counter);
if (mkdir(tmpDir.c_str(), mode) == 0) {
#if __FreeBSD__
/* Explicitly set the group of the directory. This is to
work around around problems caused by BSD's group
ownership semantics (directories inherit the group of
the parent). For instance, the group of /tmp on
FreeBSD is "wheel", so all directories created in /tmp
will be owned by "wheel"; but if the user is not in
"wheel", then "tar" will fail to unpack archives that
have the setgid bit set on directories. */
if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0)
throw SysError("setting group of directory '%1%'", tmpDir);
#endif
return tmpDir;
}
if (errno != EEXIST)
throw SysError("creating directory '%1%'", tmpDir);
}
}
std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
{
Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
// Strictly speaking, this is UB, but who cares...
// FIXME: use O_TMPFILE.
AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
if (!fd)
throw SysError("creating temporary file '%s'", tmpl);
closeOnExec(fd.get());
return {std::move(fd), tmpl};
}
void createSymlink(const Path & target, const Path & link)
{
if (symlink(target.c_str(), link.c_str()))
throw SysError("creating symlink from '%1%' to '%2%'", link, target);
}
void replaceSymlink(const Path & target, const Path & link)
{
for (unsigned int n = 0; true; n++) {
Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
try {
createSymlink(target, tmp);
} catch (SysError & e) {
if (e.errNo == EEXIST) continue;
throw;
}
renameFile(tmp, link);
break;
}
}
void setWriteTime(const fs::path & p, const struct stat & st)
{
struct timeval times[2];
times[0] = {
.tv_sec = st.st_atime,
.tv_usec = 0,
};
times[1] = {
.tv_sec = st.st_mtime,
.tv_usec = 0,
};
if (lutimes(p.c_str(), times) != 0)
throw SysError("changing modification time of '%s'", p);
}
void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete)
{
// TODO: Rewrite the `is_*` to use `symlink_status()`
auto statOfFrom = lstat(from.path().c_str());
auto fromStatus = from.symlink_status();
// Mark the directory as writable so that we can delete its children
if (andDelete && fs::is_directory(fromStatus)) {
fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
}
if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) {
fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing);
} else if (fs::is_directory(fromStatus)) {
fs::create_directory(to);
for (auto & entry : fs::directory_iterator(from.path())) {
copy(entry, to / entry.path().filename(), andDelete);
}
} else {
throw Error("file '%s' has an unsupported type", from.path());
}
setWriteTime(to, statOfFrom);
if (andDelete) {
if (!fs::is_symlink(fromStatus))
fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
fs::remove(from.path());
}
}
void renameFile(const Path & oldName, const Path & newName)
{
fs::rename(oldName, newName);
}
void moveFile(const Path & oldName, const Path & newName)
{
try {
renameFile(oldName, newName);
} catch (fs::filesystem_error & e) {
auto oldPath = fs::path(oldName);
auto newPath = fs::path(newName);
// For the move to be as atomic as possible, copy to a temporary
// directory
fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp");
Finally removeTemp = [&]() { fs::remove(temp); };
auto tempCopyTarget = temp / "copy-target";
if (e.code().value() == EXDEV) {
fs::remove(newPath);
warn("Cant rename %s as %s, copying instead", oldName, newName);
copy(fs::directory_entry(oldPath), tempCopyTarget, true);
renameFile(tempCopyTarget, newPath);
}
}
}
//////////////////////////////////////////////////////////////////////
}

238
src/libutil/file-system.hh Normal file
View file

@ -0,0 +1,238 @@
#pragma once
/**
* @file
*
* Utiltities for working with the file sytem and file paths.
*/
#include "types.hh"
#include "error.hh"
#include "logging.hh"
#include "file-descriptor.hh"
#include <sys/types.h>
#include <sys/stat.h>
#include <dirent.h>
#include <unistd.h>
#include <signal.h>
#include <boost/lexical_cast.hpp>
#include <atomic>
#include <functional>
#include <map>
#include <sstream>
#include <optional>
#ifndef HAVE_STRUCT_DIRENT_D_TYPE
#define DT_UNKNOWN 0
#define DT_REG 1
#define DT_LNK 2
#define DT_DIR 3
#endif
namespace nix {
struct Sink;
struct Source;
/**
* @return An absolutized path, resolving paths relative to the
* specified directory, or the current directory otherwise. The path
* is also canonicalised.
*/
Path absPath(Path path,
std::optional<PathView> dir = {},
bool resolveSymlinks = false);
/**
* Canonicalise a path by removing all `.` or `..` components and
* double or trailing slashes. Optionally resolves all symlink
* components such that each component of the resulting path is *not*
* a symbolic link.
*/
Path canonPath(PathView path, bool resolveSymlinks = false);
/**
* @return The directory part of the given canonical path, i.e.,
* everything before the final `/`. If the path is the root or an
* immediate child thereof (e.g., `/foo`), this means `/`
* is returned.
*/
Path dirOf(const PathView path);
/**
* @return the base name of the given canonical path, i.e., everything
* following the final `/` (trailing slashes are removed).
*/
std::string_view baseNameOf(std::string_view path);
/**
* Check whether 'path' is a descendant of 'dir'. Both paths must be
* canonicalized.
*/
bool isInDir(std::string_view path, std::string_view dir);
/**
* Check whether 'path' is equal to 'dir' or a descendant of
* 'dir'. Both paths must be canonicalized.
*/
bool isDirOrInDir(std::string_view path, std::string_view dir);
/**
* Get status of `path`.
*/
struct stat stat(const Path & path);
struct stat lstat(const Path & path);
/**
* @return true iff the given path exists.
*/
bool pathExists(const Path & path);
/**
* A version of pathExists that returns false on a permission error.
* Useful for inferring default paths across directories that might not
* be readable.
* @return true iff the given path can be accessed and exists
*/
bool pathAccessible(const Path & path);
/**
* Read the contents (target) of a symbolic link. The result is not
* in any way canonicalised.
*/
Path readLink(const Path & path);
bool isLink(const Path & path);
/**
* Read the contents of a directory. The entries `.` and `..` are
* removed.
*/
struct DirEntry
{
std::string name;
ino_t ino;
/**
* one of DT_*
*/
unsigned char type;
DirEntry(std::string name, ino_t ino, unsigned char type)
: name(std::move(name)), ino(ino), type(type) { }
};
typedef std::vector<DirEntry> DirEntries;
DirEntries readDirectory(const Path & path);
unsigned char getFileType(const Path & path);
/**
* Read the contents of a file into a string.
*/
std::string readFile(const Path & path);
void readFile(const Path & path, Sink & sink);
/**
* Write a string to a file.
*/
void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false);
void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false);
/**
* Flush a file's parent directory to disk
*/
void syncParent(const Path & path);
/**
* Delete a path; i.e., in the case of a directory, it is deleted
* recursively. It's not an error if the path does not exist. The
* second variant returns the number of bytes and blocks freed.
*/
void deletePath(const Path & path);
void deletePath(const Path & path, uint64_t & bytesFreed);
/**
* Create a directory and all its parents, if necessary. Returns the
* list of created directories, in order of creation.
*/
Paths createDirs(const Path & path);
inline Paths createDirs(PathView path)
{
return createDirs(Path(path));
}
/**
* Create a symlink.
*/
void createSymlink(const Path & target, const Path & link);
/**
* Atomically create or replace a symlink.
*/
void replaceSymlink(const Path & target, const Path & link);
void renameFile(const Path & src, const Path & dst);
/**
* Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst`
* are on a different filesystem.
*
* Beware that this might not be atomic because of the copy that happens behind
* the scenes
*/
void moveFile(const Path & src, const Path & dst);
/**
* Automatic cleanup of resources.
*/
class AutoDelete
{
Path path;
bool del;
bool recursive;
public:
AutoDelete();
AutoDelete(const Path & p, bool recursive = true);
~AutoDelete();
void cancel();
void reset(const Path & p, bool recursive = true);
operator Path() const { return path; }
operator PathView() const { return path; }
};
struct DIRDeleter
{
void operator()(DIR * dir) const {
closedir(dir);
}
};
typedef std::unique_ptr<DIR, DIRDeleter> AutoCloseDir;
/**
* Create a temporary directory.
*/
Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755);
/**
* Create a temporary file, returning a file handle and its path.
*/
std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix");
/**
* Used in various places.
*/
typedef std::function<bool(const Path & path)> PathFilter;
extern PathFilter defaultPathFilter;
}

View file

@ -1,162 +0,0 @@
#include <sys/time.h>
#include <filesystem>
#include <atomic>
#include "finally.hh"
#include "util.hh"
#include "types.hh"
namespace fs = std::filesystem;
namespace nix {
static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
std::atomic<unsigned int> & counter)
{
tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
if (includePid)
return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++);
else
return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++);
}
Path createTempDir(const Path & tmpRoot, const Path & prefix,
bool includePid, bool useGlobalCounter, mode_t mode)
{
static std::atomic<unsigned int> globalCounter = 0;
std::atomic<unsigned int> localCounter = 0;
auto & counter(useGlobalCounter ? globalCounter : localCounter);
while (1) {
checkInterrupt();
Path tmpDir = tempName(tmpRoot, prefix, includePid, counter);
if (mkdir(tmpDir.c_str(), mode) == 0) {
#if __FreeBSD__
/* Explicitly set the group of the directory. This is to
work around around problems caused by BSD's group
ownership semantics (directories inherit the group of
the parent). For instance, the group of /tmp on
FreeBSD is "wheel", so all directories created in /tmp
will be owned by "wheel"; but if the user is not in
"wheel", then "tar" will fail to unpack archives that
have the setgid bit set on directories. */
if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0)
throw SysError("setting group of directory '%1%'", tmpDir);
#endif
return tmpDir;
}
if (errno != EEXIST)
throw SysError("creating directory '%1%'", tmpDir);
}
}
std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
{
Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
// Strictly speaking, this is UB, but who cares...
// FIXME: use O_TMPFILE.
AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
if (!fd)
throw SysError("creating temporary file '%s'", tmpl);
closeOnExec(fd.get());
return {std::move(fd), tmpl};
}
void createSymlink(const Path & target, const Path & link)
{
if (symlink(target.c_str(), link.c_str()))
throw SysError("creating symlink from '%1%' to '%2%'", link, target);
}
void replaceSymlink(const Path & target, const Path & link)
{
for (unsigned int n = 0; true; n++) {
Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
try {
createSymlink(target, tmp);
} catch (SysError & e) {
if (e.errNo == EEXIST) continue;
throw;
}
renameFile(tmp, link);
break;
}
}
void setWriteTime(const fs::path & p, const struct stat & st)
{
struct timeval times[2];
times[0] = {
.tv_sec = st.st_atime,
.tv_usec = 0,
};
times[1] = {
.tv_sec = st.st_mtime,
.tv_usec = 0,
};
if (lutimes(p.c_str(), times) != 0)
throw SysError("changing modification time of '%s'", p);
}
void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete)
{
// TODO: Rewrite the `is_*` to use `symlink_status()`
auto statOfFrom = lstat(from.path().c_str());
auto fromStatus = from.symlink_status();
// Mark the directory as writable so that we can delete its children
if (andDelete && fs::is_directory(fromStatus)) {
fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
}
if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) {
fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing);
} else if (fs::is_directory(fromStatus)) {
fs::create_directory(to);
for (auto & entry : fs::directory_iterator(from.path())) {
copy(entry, to / entry.path().filename(), andDelete);
}
} else {
throw Error("file '%s' has an unsupported type", from.path());
}
setWriteTime(to, statOfFrom);
if (andDelete) {
if (!fs::is_symlink(fromStatus))
fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
fs::remove(from.path());
}
}
void renameFile(const Path & oldName, const Path & newName)
{
fs::rename(oldName, newName);
}
void moveFile(const Path & oldName, const Path & newName)
{
try {
renameFile(oldName, newName);
} catch (fs::filesystem_error & e) {
auto oldPath = fs::path(oldName);
auto newPath = fs::path(newName);
// For the move to be as atomic as possible, copy to a temporary
// directory
fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp");
Finally removeTemp = [&]() { fs::remove(temp); };
auto tempCopyTarget = temp / "copy-target";
if (e.code().value() == EXDEV) {
fs::remove(newPath);
warn("Cant rename %s as %s, copying instead", oldName, newName);
copy(fs::directory_entry(oldPath), tempCopyTarget, true);
renameFile(tempCopyTarget, newPath);
}
}
}
}

View file

@ -4,6 +4,7 @@
#include "types.hh"
#include "serialise.hh"
#include "source-accessor.hh"
#include "file-system.hh"
namespace nix {

View file

@ -9,7 +9,6 @@
#include "hash.hh"
#include "archive.hh"
#include "split.hh"
#include "util.hh"
#include <sys/types.h>
#include <sys/stat.h>

View file

@ -3,6 +3,7 @@
#include "types.hh"
#include "serialise.hh"
#include "file-system.hh"
namespace nix {

View file

@ -1,4 +1,7 @@
#include "logging.hh"
#include "file-descriptor.hh"
#include "environment-variables.hh"
#include "terminal.hh"
#include "util.hh"
#include "config.hh"

View file

@ -10,6 +10,8 @@
#include <unistd.h>
#include <signal.h>
#include "signals.hh"
namespace nix {

View file

@ -1,13 +1,22 @@
#if __linux__
#include "namespaces.hh"
#include "current-process.hh"
#include "util.hh"
#include "finally.hh"
#include "file-system.hh"
#include "processes.hh"
#include "signals.hh"
#if __linux__
# include <mutex>
# include <sys/resource.h>
# include "cgroup.hh"
#endif
#include <sys/mount.h>
namespace nix {
#if __linux__
bool userNamespacesSupported()
{
static auto res = [&]() -> bool
@ -92,6 +101,60 @@ bool mountAndPidNamespacesSupported()
return res;
}
#endif
//////////////////////////////////////////////////////////////////////
#if __linux__
static AutoCloseFD fdSavedMountNamespace;
static AutoCloseFD fdSavedRoot;
#endif
void saveMountNamespace()
{
#if __linux__
static std::once_flag done;
std::call_once(done, []() {
fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY);
if (!fdSavedMountNamespace)
throw SysError("saving parent mount namespace");
fdSavedRoot = open("/proc/self/root", O_RDONLY);
});
#endif
}
void restoreMountNamespace()
{
#if __linux__
try {
auto savedCwd = absPath(".");
if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1)
throw SysError("restoring parent mount namespace");
if (fdSavedRoot) {
if (fchdir(fdSavedRoot.get()))
throw SysError("chdir into saved root");
if (chroot("."))
throw SysError("chroot into saved root");
}
if (chdir(savedCwd.c_str()) == -1)
throw SysError("restoring cwd");
} catch (Error & e) {
debug(e.msg());
}
#endif
}
void unshareFilesystem()
{
#ifdef __linux__
if (unshare(CLONE_FS) != 0 && errno != EPERM)
throw SysError("unsharing filesystem state in download thread");
#endif
}
}

View file

@ -1,8 +1,31 @@
#pragma once
///@file
#include <optional>
#include "types.hh"
namespace nix {
/**
* Save the current mount namespace. Ignored if called more than
* once.
*/
void saveMountNamespace();
/**
* Restore the mount namespace saved by saveMountNamespace(). Ignored
* if saveMountNamespace() was never called.
*/
void restoreMountNamespace();
/**
* Cause this thread to not share any FS attributes with the main
* thread, because this causes setns() in restoreMountNamespace() to
* fail.
*/
void unshareFilesystem();
#if __linux__
bool userNamespacesSupported();

View file

@ -1,4 +1,5 @@
#include "posix-source-accessor.hh"
#include "signals.hh"
namespace nix {

421
src/libutil/processes.cc Normal file
View file

@ -0,0 +1,421 @@
#include "current-process.hh"
#include "environment-variables.hh"
#include "signals.hh"
#include "processes.hh"
#include "finally.hh"
#include "serialise.hh"
#include <cerrno>
#include <cstdlib>
#include <cstring>
#include <future>
#include <iostream>
#include <sstream>
#include <thread>
#include <grp.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
#ifdef __APPLE__
# include <sys/syscall.h>
#endif
#ifdef __linux__
# include <sys/prctl.h>
# include <sys/mman.h>
#endif
namespace nix {
Pid::Pid()
{
}
Pid::Pid(pid_t pid)
: pid(pid)
{
}
Pid::~Pid()
{
if (pid != -1) kill();
}
void Pid::operator =(pid_t pid)
{
if (this->pid != -1 && this->pid != pid) kill();
this->pid = pid;
killSignal = SIGKILL; // reset signal to default
}
Pid::operator pid_t()
{
return pid;
}
int Pid::kill()
{
assert(pid != -1);
debug("killing process %1%", pid);
/* Send the requested signal to the child. If it has its own
process group, send the signal to every process in the child
process group (which hopefully includes *all* its children). */
if (::kill(separatePG ? -pid : pid, killSignal) != 0) {
/* On BSDs, killing a process group will return EPERM if all
processes in the group are zombies (or something like
that). So try to detect and ignore that situation. */
#if __FreeBSD__ || __APPLE__
if (errno != EPERM || ::kill(pid, 0) != 0)
#endif
logError(SysError("killing process %d", pid).info());
}
return wait();
}
int Pid::wait()
{
assert(pid != -1);
while (1) {
int status;
int res = waitpid(pid, &status, 0);
if (res == pid) {
pid = -1;
return status;
}
if (errno != EINTR)
throw SysError("cannot get exit status of PID %d", pid);
checkInterrupt();
}
}
void Pid::setSeparatePG(bool separatePG)
{
this->separatePG = separatePG;
}
void Pid::setKillSignal(int signal)
{
this->killSignal = signal;
}
pid_t Pid::release()
{
pid_t p = pid;
pid = -1;
return p;
}
void killUser(uid_t uid)
{
debug("killing all processes running under uid '%1%'", uid);
assert(uid != 0); /* just to be safe... */
/* The system call kill(-1, sig) sends the signal `sig' to all
users to which the current process can send signals. So we
fork a process, switch to uid, and send a mass kill. */
Pid pid = startProcess([&]() {
if (setuid(uid) == -1)
throw SysError("setting uid");
while (true) {
#ifdef __APPLE__
/* OSX's kill syscall takes a third parameter that, among
other things, determines if kill(-1, signo) affects the
calling process. In the OSX libc, it's set to true,
which means "follow POSIX", which we don't want here
*/
if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break;
#else
if (kill(-1, SIGKILL) == 0) break;
#endif
if (errno == ESRCH || errno == EPERM) break; /* no more processes */
if (errno != EINTR)
throw SysError("cannot kill processes for uid '%1%'", uid);
}
_exit(0);
});
int status = pid.wait();
if (status != 0)
throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status));
/* !!! We should really do some check to make sure that there are
no processes left running under `uid', but there is no portable
way to do so (I think). The most reliable way may be `ps -eo
uid | grep -q $uid'. */
}
//////////////////////////////////////////////////////////////////////
/* Wrapper around vfork to prevent the child process from clobbering
the caller's stack frame in the parent. */
static pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline));
static pid_t doFork(bool allowVfork, std::function<void()> fun)
{
#ifdef __linux__
pid_t pid = allowVfork ? vfork() : fork();
#else
pid_t pid = fork();
#endif
if (pid != 0) return pid;
fun();
abort();
}
#if __linux__
static int childEntry(void * arg)
{
auto main = (std::function<void()> *) arg;
(*main)();
return 1;
}
#endif
pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
{
std::function<void()> wrapper = [&]() {
if (!options.allowVfork)
logger = makeSimpleLogger();
try {
#if __linux__
if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1)
throw SysError("setting death signal");
#endif
fun();
} catch (std::exception & e) {
try {
std::cerr << options.errorPrefix << e.what() << "\n";
} catch (...) { }
} catch (...) { }
if (options.runExitHandlers)
exit(1);
else
_exit(1);
};
pid_t pid = -1;
if (options.cloneFlags) {
#ifdef __linux__
// Not supported, since then we don't know when to free the stack.
assert(!(options.cloneFlags & CLONE_VM));
size_t stackSize = 1 * 1024 * 1024;
auto stack = (char *) mmap(0, stackSize,
PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
if (stack == MAP_FAILED) throw SysError("allocating stack");
Finally freeStack([&]() { munmap(stack, stackSize); });
pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper);
#else
throw Error("clone flags are only supported on Linux");
#endif
} else
pid = doFork(options.allowVfork, wrapper);
if (pid == -1) throw SysError("unable to fork");
return pid;
}
std::string runProgram(Path program, bool searchPath, const Strings & args,
const std::optional<std::string> & input, bool isInteractive)
{
auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive});
if (!statusOk(res.first))
throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first));
return res.second;
}
// Output = error code + "standard out" output stream
std::pair<int, std::string> runProgram(RunOptions && options)
{
StringSink sink;
options.standardOut = &sink;
int status = 0;
try {
runProgram2(options);
} catch (ExecError & e) {
status = e.status;
}
return {status, std::move(sink.s)};
}
void runProgram2(const RunOptions & options)
{
checkInterrupt();
assert(!(options.standardIn && options.input));
std::unique_ptr<Source> source_;
Source * source = options.standardIn;
if (options.input) {
source_ = std::make_unique<StringSource>(*options.input);
source = source_.get();
}
/* Create a pipe. */
Pipe out, in;
if (options.standardOut) out.create();
if (source) in.create();
ProcessOptions processOptions;
// vfork implies that the environment of the main process and the fork will
// be shared (technically this is undefined, but in practice that's the
// case), so we can't use it if we alter the environment
processOptions.allowVfork = !options.environment;
std::optional<Finally<std::function<void()>>> resumeLoggerDefer;
if (options.isInteractive) {
logger->pause();
resumeLoggerDefer.emplace(
[]() {
logger->resume();
}
);
}
/* Fork. */
Pid pid = startProcess([&]() {
if (options.environment)
replaceEnv(*options.environment);
if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
throw SysError("dupping stdout");
if (options.mergeStderrToStdout)
if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1)
throw SysError("cannot dup stdout into stderr");
if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1)
throw SysError("dupping stdin");
if (options.chdir && chdir((*options.chdir).c_str()) == -1)
throw SysError("chdir failed");
if (options.gid && setgid(*options.gid) == -1)
throw SysError("setgid failed");
/* Drop all other groups if we're setgid. */
if (options.gid && setgroups(0, 0) == -1)
throw SysError("setgroups failed");
if (options.uid && setuid(*options.uid) == -1)
throw SysError("setuid failed");
Strings args_(options.args);
args_.push_front(options.program);
restoreProcessContext();
if (options.searchPath)
execvp(options.program.c_str(), stringsToCharPtrs(args_).data());
// This allows you to refer to a program with a pathname relative
// to the PATH variable.
else
execv(options.program.c_str(), stringsToCharPtrs(args_).data());
throw SysError("executing '%1%'", options.program);
}, processOptions);
out.writeSide.close();
std::thread writerThread;
std::promise<void> promise;
Finally doJoin([&]() {
if (writerThread.joinable())
writerThread.join();
});
if (source) {
in.readSide.close();
writerThread = std::thread([&]() {
try {
std::vector<char> buf(8 * 1024);
while (true) {
size_t n;
try {
n = source->read(buf.data(), buf.size());
} catch (EndOfFile &) {
break;
}
writeFull(in.writeSide.get(), {buf.data(), n});
}
promise.set_value();
} catch (...) {
promise.set_exception(std::current_exception());
}
in.writeSide.close();
});
}
if (options.standardOut)
drainFD(out.readSide.get(), *options.standardOut);
/* Wait for the child to finish. */
int status = pid.wait();
/* Wait for the writer thread to finish. */
if (source) promise.get_future().get();
if (status)
throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status));
}
//////////////////////////////////////////////////////////////////////
std::string statusToString(int status)
{
if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
if (WIFEXITED(status))
return fmt("failed with exit code %1%", WEXITSTATUS(status));
else if (WIFSIGNALED(status)) {
int sig = WTERMSIG(status);
#if HAVE_STRSIGNAL
const char * description = strsignal(sig);
return fmt("failed due to signal %1% (%2%)", sig, description);
#else
return fmt("failed due to signal %1%", sig);
#endif
}
else
return "died abnormally";
} else return "succeeded";
}
bool statusOk(int status)
{
return WIFEXITED(status) && WEXITSTATUS(status) == 0;
}
}

123
src/libutil/processes.hh Normal file
View file

@ -0,0 +1,123 @@
#pragma once
///@file
#include "types.hh"
#include "error.hh"
#include "logging.hh"
#include "ansicolor.hh"
#include <sys/types.h>
#include <sys/stat.h>
#include <dirent.h>
#include <unistd.h>
#include <signal.h>
#include <boost/lexical_cast.hpp>
#include <atomic>
#include <functional>
#include <map>
#include <sstream>
#include <optional>
namespace nix {
struct Sink;
struct Source;
class Pid
{
pid_t pid = -1;
bool separatePG = false;
int killSignal = SIGKILL;
public:
Pid();
Pid(pid_t pid);
~Pid();
void operator =(pid_t pid);
operator pid_t();
int kill();
int wait();
void setSeparatePG(bool separatePG);
void setKillSignal(int signal);
pid_t release();
};
/**
* Kill all processes running under the specified uid by sending them
* a SIGKILL.
*/
void killUser(uid_t uid);
/**
* Fork a process that runs the given function, and return the child
* pid to the caller.
*/
struct ProcessOptions
{
std::string errorPrefix = "";
bool dieWithParent = true;
bool runExitHandlers = false;
bool allowVfork = false;
/**
* use clone() with the specified flags (Linux only)
*/
int cloneFlags = 0;
};
pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions());
/**
* Run a program and return its stdout in a string (i.e., like the
* shell backtick operator).
*/
std::string runProgram(Path program, bool searchPath = false,
const Strings & args = Strings(),
const std::optional<std::string> & input = {}, bool isInteractive = false);
struct RunOptions
{
Path program;
bool searchPath = true;
Strings args;
std::optional<uid_t> uid;
std::optional<uid_t> gid;
std::optional<Path> chdir;
std::optional<std::map<std::string, std::string>> environment;
std::optional<std::string> input;
Source * standardIn = nullptr;
Sink * standardOut = nullptr;
bool mergeStderrToStdout = false;
bool isInteractive = false;
};
std::pair<int, std::string> runProgram(RunOptions && options);
void runProgram2(const RunOptions & options);
class ExecError : public Error
{
public:
int status;
template<typename... Args>
ExecError(int status, const Args & ... args)
: Error(args...), status(status)
{ }
};
/**
* Convert the exit status of a child as returned by wait() into an
* error string.
*/
std::string statusToString(int status);
bool statusOk(int status);
}

Some files were not shown because too many files have changed in this diff Show more