diff --git a/.github/STALE-BOT.md b/.github/STALE-BOT.md index 5e8f5d929..383717bfc 100644 --- a/.github/STALE-BOT.md +++ b/.github/STALE-BOT.md @@ -3,7 +3,7 @@ - Thanks for your contribution! - To remove the stale label, just leave a new comment. - _How to find the right people to ping?_ → [`git blame`](https://git-scm.com/docs/git-blame) to the rescue! (or GitHub's history and blame buttons.) -- You can always ask for help on [our Discourse Forum](https://discourse.nixos.org/) or on the [#nixos IRC channel](https://webchat.freenode.net/#nixos). +- You can always ask for help on [our Discourse Forum](https://discourse.nixos.org/) or on [Matrix - #nix:nixos.org](https://matrix.to/#/#nix:nixos.org). ## Suggestions for PRs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bde6106e0..17a79dc97 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,52 +8,62 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} - env: - CACHIX_NAME: nix-ci + steps: - uses: actions/checkout@v2.3.4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v12 - - uses: cachix/cachix-action@v8 + - uses: cachix/install-nix-action@v13 + - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV + - uses: cachix/cachix-action@v10 with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' #- run: nix flake check - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi) - installer: - if: github.event_name == 'push' - needs: tests + check_cachix: + name: Cachix secret present for installer tests + runs-on: ubuntu-latest + outputs: + secret: ${{ steps.secret.outputs.secret }} + steps: + - name: Check for Cachix secret + id: secret + env: + _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }} + run: echo "::set-output name=secret::${{ env._CACHIX_SECRETS != '' }}" + installer: + needs: [tests, check_cachix] + if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true' runs-on: ubuntu-latest - env: - CACHIX_NAME: nix-ci outputs: installerURL: ${{ steps.prepare-installer.outputs.installerURL }} steps: - uses: actions/checkout@v2.3.4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v12 - - uses: cachix/cachix-action@v8 + - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV + - uses: cachix/install-nix-action@v13 + - uses: cachix/cachix-action@v10 with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - id: prepare-installer run: scripts/prepare-installer-for-github-actions installer_test: - if: github.event_name == 'push' - needs: installer + needs: [installer, check_cachix] + if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true' strategy: matrix: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} - env: - CACHIX_NAME: nix-ci steps: - uses: actions/checkout@v2.3.4 - - uses: cachix/install-nix-action@master + - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV + - uses: cachix/install-nix-action@v13 with: install_url: '${{needs.installer.outputs.installerURL}}' - install_options: '--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve' + install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" - run: nix-instantiate -E 'builtins.currentTime' --eval - \ No newline at end of file diff --git a/.gitignore b/.gitignore index 37aada307..2e14561fe 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,7 @@ perl/Makefile.config /tests/shell /tests/shell.drv /tests/config.nix +/tests/ca/config.nix # /tests/lang/ /tests/lang/*.out diff --git a/Makefile b/Makefile index 68ec3ab0c..b7f0e79db 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ makefiles = \ src/resolve-system-dependencies/local.mk \ scripts/local.mk \ misc/bash/local.mk \ + misc/zsh/local.mk \ misc/systemd/local.mk \ misc/launchd/local.mk \ misc/upstart/local.mk \ diff --git a/Makefile.config.in b/Makefile.config.in index 3c1f01d1e..fd92365eb 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -15,7 +15,6 @@ LDFLAGS = @LDFLAGS@ LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ LIBCURL_LIBS = @LIBCURL_LIBS@ -LIBLZMA_LIBS = @LIBLZMA_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@ PACKAGE_NAME = @PACKAGE_NAME@ diff --git a/README.md b/README.md index 4686010ef..80d6f128c 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,8 @@ build nix from source with nix-build or how to get a development environment. - [Nix manual](https://nixos.org/nix/manual) - [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) - [NixOS Discourse](https://discourse.nixos.org/) -- [IRC - #nixos on freenode.net](irc://irc.freenode.net/#nixos) +- [Matrix - #nix:nixos.org](https://matrix.to/#/#nix:nixos.org) +- [IRC - #nixos on libera.chat](irc://irc.libera.chat/#nixos) ## License diff --git a/config/config.guess b/config/config.guess index 699b3a10b..1972fda8e 100755 --- a/config/config.guess +++ b/config/config.guess @@ -1,8 +1,8 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright 1992-2020 Free Software Foundation, Inc. +# Copyright 1992-2021 Free Software Foundation, Inc. -timestamp='2020-11-19' +timestamp='2021-01-25' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -50,7 +50,7 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright 1992-2020 Free Software Foundation, Inc. +Copyright 1992-2021 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -188,10 +188,9 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in # # Note: NetBSD doesn't particularly care about the vendor # portion of the name. We always set it to "unknown". - sysctl="sysctl -n hw.machine_arch" UNAME_MACHINE_ARCH=$( (uname -p 2>/dev/null || \ - "/sbin/$sysctl" 2>/dev/null || \ - "/usr/sbin/$sysctl" 2>/dev/null || \ + /sbin/sysctl -n hw.machine_arch 2>/dev/null || \ + /usr/sbin/sysctl -n hw.machine_arch 2>/dev/null || \ echo unknown)) case "$UNAME_MACHINE_ARCH" in aarch64eb) machine=aarch64_be-unknown ;; @@ -996,6 +995,9 @@ EOF k1om:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; + loongarch32:Linux:*:* | loongarch64:Linux:*:* | loongarchx32:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; m32r*:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; @@ -1084,7 +1086,7 @@ EOF ppcle:Linux:*:*) echo powerpcle-unknown-linux-"$LIBC" exit ;; - riscv32:Linux:*:* | riscv64:Linux:*:*) + riscv32:Linux:*:* | riscv32be:Linux:*:* | riscv64:Linux:*:* | riscv64be:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; s390:Linux:*:* | s390x:Linux:*:*) @@ -1480,8 +1482,8 @@ EOF i*86:rdos:*:*) echo "$UNAME_MACHINE"-pc-rdos exit ;; - i*86:AROS:*:*) - echo "$UNAME_MACHINE"-pc-aros + *:AROS:*:*) + echo "$UNAME_MACHINE"-unknown-aros exit ;; x86_64:VMkernel:*:*) echo "$UNAME_MACHINE"-unknown-esx diff --git a/config/config.sub b/config/config.sub index 19c9553b1..63c1f1c8b 100755 --- a/config/config.sub +++ b/config/config.sub @@ -1,8 +1,8 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright 1992-2020 Free Software Foundation, Inc. +# Copyright 1992-2021 Free Software Foundation, Inc. -timestamp='2020-12-02' +timestamp='2021-01-08' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -67,7 +67,7 @@ Report bugs and patches to ." version="\ GNU config.sub ($timestamp) -Copyright 1992-2020 Free Software Foundation, Inc. +Copyright 1992-2021 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -1185,6 +1185,7 @@ case $cpu-$vendor in | k1om \ | le32 | le64 \ | lm32 \ + | loongarch32 | loongarch64 | loongarchx32 \ | m32c | m32r | m32rle \ | m5200 | m68000 | m680[012346]0 | m68360 | m683?2 | m68k \ | m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x \ @@ -1229,7 +1230,7 @@ case $cpu-$vendor in | powerpc | powerpc64 | powerpc64le | powerpcle | powerpcspe \ | pru \ | pyramid \ - | riscv | riscv32 | riscv64 \ + | riscv | riscv32 | riscv32be | riscv64 | riscv64be \ | rl78 | romp | rs6000 | rx \ | s390 | s390x \ | score \ @@ -1682,11 +1683,14 @@ fi # Now, validate our (potentially fixed-up) OS. case $os in - # Sometimes we do "kernel-abi", so those need to count as OSes. + # Sometimes we do "kernel-libc", so those need to count as OSes. musl* | newlib* | uclibc*) ;; - # Likewise for "kernel-libc" - eabi | eabihf | gnueabi | gnueabihf) + # Likewise for "kernel-abi" + eabi* | gnueabi*) + ;; + # VxWorks passes extra cpu info in the 4th filed. + simlinux | simwindows | spe) ;; # Now accept the basic system types. # The portable systems comes first. @@ -1750,6 +1754,8 @@ case $kernel-$os in ;; kfreebsd*-gnu* | kopensolaris*-gnu*) ;; + vxworks-simlinux | vxworks-simwindows | vxworks-spe) + ;; nto-qnx*) ;; os2-emx) diff --git a/configure.ac b/configure.ac index 24a0b3698..6e563eec3 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX"])) +AC_INIT([nix],[m4_esyscmd(bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX")]) AC_CONFIG_MACRO_DIRS([m4]) AC_CONFIG_SRCDIR(README.md) AC_CONFIG_AUX_DIR(config) @@ -9,8 +9,7 @@ AC_PROG_SED AC_CANONICAL_HOST AC_MSG_CHECKING([for the canonical Nix system name]) -AC_ARG_WITH(system, AC_HELP_STRING([--with-system=SYSTEM], - [Platform identifier (e.g., `i686-linux').]), +AC_ARG_WITH(system, AS_HELP_STRING([--with-system=SYSTEM],[Platform identifier (e.g., `i686-linux').]), [system=$withval], [case "$host_cpu" in i*86) @@ -66,7 +65,7 @@ AC_SYS_LARGEFILE AC_STRUCT_DIRENT_D_TYPE if test "$sys_name" = sunos; then # Solaris requires -lsocket -lnsl for network functions - LIBS="-lsocket -lnsl $LIBS" + LDFLAGS="-lsocket -lnsl $LDFLAGS" fi @@ -127,8 +126,7 @@ NEED_PROG(jq, jq) AC_SUBST(coreutils, [$(dirname $(type -p cat))]) -AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH], - [path of the Nix store (defaults to /nix/store)]), +AC_ARG_WITH(store-dir, AS_HELP_STRING([--with-store-dir=PATH],[path of the Nix store (defaults to /nix/store)]), storedir=$withval, storedir='/nix/store') AC_SUBST(storedir) @@ -152,13 +150,12 @@ int main() { }]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes) AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC) if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then - LDFLAGS="$LDFLAGS -latomic" + LDFLAGS="-latomic $LDFLAGS" fi PKG_PROG_PKG_CONFIG -AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared], - [Build shared libraries for Nix [default=yes]]), +AC_ARG_ENABLE(shared, AS_HELP_STRING([--enable-shared],[Build shared libraries for Nix [default=yes]]), shared=$enableval, shared=yes) if test "$shared" = yes; then AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.]) @@ -172,11 +169,6 @@ fi PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) -# Look for libbz2, a required dependency. -AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true], - [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://sourceware.org/bzip2/.])]) -AC_CHECK_HEADERS([bzlib.h], [true], - [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://sourceware.org/bzip2/.])]) # Checks for libarchive PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"]) # Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed @@ -205,16 +197,6 @@ PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLA # Look for libsodium, an optional dependency. PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) -# Look for liblzma, a required dependency. -PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"]) -AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt], - [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])]) - -# Look for zlib, a required dependency. -PKG_CHECK_MODULES([ZLIB], [zlib], [CXXFLAGS="$ZLIB_CFLAGS $CXXFLAGS"]) -AC_CHECK_HEADER([zlib.h],[:],[AC_MSG_ERROR([could not find the zlib.h header])]) -LDFLAGS="-lz $LDFLAGS" - # Look for libbrotli{enc,dec}. PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]) @@ -230,9 +212,8 @@ AC_SUBST(HAVE_LIBCPUID, [$have_libcpuid]) # Look for libseccomp, required for Linux sandboxing. if test "$sys_name" = linux; then AC_ARG_ENABLE([seccomp-sandboxing], - AC_HELP_STRING([--disable-seccomp-sandboxing], - [Don't build support for seccomp sandboxing (only recommended if your arch doesn't support libseccomp yet!)] - )) + AS_HELP_STRING([--disable-seccomp-sandboxing],[Don't build support for seccomp sandboxing (only recommended if your arch doesn't support libseccomp yet!) + ])) if test "x$enable_seccomp_sandboxing" != "xno"; then PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp], [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"]) @@ -250,8 +231,8 @@ AC_SUBST(HAVE_SECCOMP, [$have_seccomp]) # Look for aws-cpp-sdk-s3. AC_LANG_PUSH(C++) AC_CHECK_HEADERS([aws/s3/S3Client.h], - [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-sdk-cpp.]) - enable_s3=1], [enable_s3=]) + [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-sdk-cpp.]) enable_s3=1], + [AC_DEFINE([ENABLE_S3], [0], [Whether to enable S3 support via aws-sdk-cpp.]) enable_s3=]) AC_SUBST(ENABLE_S3, [$enable_s3]) AC_LANG_POP(C++) @@ -264,8 +245,7 @@ fi # Whether to use the Boehm garbage collector. -AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc], - [enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]), +AC_ARG_ENABLE(gc, AS_HELP_STRING([--enable-gc],[enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]), gc=$enableval, gc=yes) if test "$gc" = yes; then PKG_CHECK_MODULES([BDW_GC], [bdw-gc]) @@ -279,8 +259,7 @@ PKG_CHECK_MODULES([GTEST], [gtest_main]) # documentation generation switch -AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen], - [disable documentation generation]), +AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation generation]), doc_generate=$enableval, doc_generate=yes) AC_SUBST(doc_generate) @@ -300,19 +279,7 @@ if test "$(uname)" = "Darwin"; then fi -# Do we have GNU tar? -AC_MSG_CHECKING([if you have a recent GNU tar]) -if $tar --version 2> /dev/null | grep -q GNU && tar cvf /dev/null --warning=no-timestamp ./config.log > /dev/null; then - AC_MSG_RESULT(yes) - tarFlags="--warning=no-timestamp" -else - AC_MSG_RESULT(no) -fi -AC_SUBST(tarFlags) - - -AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH], - [path of a statically-linked shell to use as /bin/sh in sandboxes]), +AC_ARG_WITH(sandbox-shell, AS_HELP_STRING([--with-sandbox-shell=PATH],[path of a statically-linked shell to use as /bin/sh in sandboxes]), sandbox_shell=$withval) AC_SUBST(sandbox_shell) @@ -327,6 +294,6 @@ done rm -f Makefile.config -AC_CONFIG_HEADER([config.h]) +AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([]) AC_OUTPUT diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 81a7755e8..271529b38 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -25,19 +25,19 @@ nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command - $(d)/%.1: $(d)/src/command-ref/%.md @printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp @cat $^ >> $^.tmp - $(trace-gen) lowdown -sT man $^.tmp -o $@ + $(trace-gen) lowdown -sT man -M section=1 $^.tmp -o $@ @rm $^.tmp $(d)/%.8: $(d)/src/command-ref/%.md @printf "Title: %s\n\n" "$$(basename $@ .8)" > $^.tmp @cat $^ >> $^.tmp - $(trace-gen) lowdown -sT man $^.tmp -o $@ + $(trace-gen) lowdown -sT man -M section=8 $^.tmp -o $@ @rm $^.tmp $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md @printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp @cat $^ >> $^.tmp - $(trace-gen) lowdown -sT man $^.tmp -o $@ + $(trace-gen) lowdown -sT man -M section=5 $^.tmp -o $@ @rm $^.tmp $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli @@ -80,7 +80,7 @@ install: $(d)/src/command-ref/new-cli if [[ $$name = SUMMARY ]]; then continue; fi; \ printf "Title: %s\n\n" "$$name" > $$i.tmp; \ cat $$i >> $$i.tmp; \ - lowdown -sT man $$i.tmp -o $(mandir)/man1/$$name.1; \ + lowdown -sT man -M section=1 $$i.tmp -o $(mandir)/man1/$$name.1; \ done $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md diff --git a/doc/manual/src/advanced-topics/cores-vs-jobs.md b/doc/manual/src/advanced-topics/cores-vs-jobs.md index 4a9058ca1..9e91ab9c7 100644 --- a/doc/manual/src/advanced-topics/cores-vs-jobs.md +++ b/doc/manual/src/advanced-topics/cores-vs-jobs.md @@ -4,13 +4,13 @@ Nix has two relevant settings with regards to how your CPU cores will be utilized: `cores` and `max-jobs`. This chapter will talk about what they are, how they interact, and their configuration trade-offs. - - `max-jobs` + - `max-jobs`\ Dictates how many separate derivations will be built at the same time. If you set this to zero, the local machine will do no builds. Nix will still substitute from binary caches, and build remotely if remote builders are configured. - - `cores` + - `cores`\ Suggests how many cores each derivation should use. Similar to `make -j`. diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index c670d82b8..6e2403461 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -2,45 +2,49 @@ Most Nix commands interpret the following environment variables: - - `IN_NIX_SHELL` + - `IN_NIX_SHELL`\ Indicator that tells if the current environment was set up by `nix-shell`. Since Nix 2.0 the values are `"pure"` and `"impure"` - - `NIX_PATH` + - `NIX_PATH`\ A colon-separated list of directories used to look up Nix expressions enclosed in angle brackets (i.e., ``). For instance, the value - + /home/eelco/Dev:/etc/nixos - + will cause Nix to look for paths relative to `/home/eelco/Dev` and `/etc/nixos`, in this order. It is also possible to match paths against a prefix. For example, the value - + nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos - + will cause Nix to search for `` in `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. - + If a path in the Nix search path starts with `http://` or `https://`, it is interpreted as the URL of a tarball that will be downloaded and unpacked to a temporary location. The tarball must consist of a single top-level directory. For example, setting `NIX_PATH` to - - nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz - - tells Nix to download the latest revision in the Nixpkgs/NixOS 15.09 - channel. - - A following shorthand can be used to refer to the official channels: - - nixpkgs=channel:nixos-15.09 - - The search path can be extended using the `-I` option, which takes - precedence over `NIX_PATH`. - - `NIX_IGNORE_SYMLINK_STORE` + nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz + + tells Nix to download and use the current contents of the + `master` branch in the `nixpkgs` repository. + + The URLs of the tarballs from the official nixos.org channels (see + [the manual for `nix-channel`](nix-channel.md)) can be abbreviated + as `channel:`. For instance, the following two + values of `NIX_PATH` are equivalent: + + nixpkgs=channel:nixos-21.05 + nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz + + The Nix search path can also be extended using the `-I` option to + many Nix commands, which takes precedence over `NIX_PATH`. + + - `NIX_IGNORE_SYMLINK_STORE`\ Normally, the Nix store directory (typically `/nix/store`) is not allowed to contain any symlink components. This is to prevent “impure” builds. Builders sometimes “canonicalise” paths by @@ -50,7 +54,7 @@ Most Nix commands interpret the following environment variables: builds are deployed to machines where `/nix/store` resolves differently. If you are sure that you’re not going to do that, you can set `NIX_IGNORE_SYMLINK_STORE` to `1`. - + Note that if you’re symlinking the Nix store so that you can put it on another file system than the root file system, on Linux you’re better off using `bind` mount points, e.g., @@ -59,44 +63,44 @@ Most Nix commands interpret the following environment variables: $ mkdir /nix $ mount -o bind /mnt/otherdisk/nix /nix ``` - + Consult the mount 8 manual page for details. - - `NIX_STORE_DIR` + - `NIX_STORE_DIR`\ Overrides the location of the Nix store (default `prefix/store`). - - `NIX_DATA_DIR` + - `NIX_DATA_DIR`\ Overrides the location of the Nix static data directory (default `prefix/share`). - - `NIX_LOG_DIR` + - `NIX_LOG_DIR`\ Overrides the location of the Nix log directory (default `prefix/var/log/nix`). - - `NIX_STATE_DIR` + - `NIX_STATE_DIR`\ Overrides the location of the Nix state directory (default `prefix/var/nix`). - - `NIX_CONF_DIR` + - `NIX_CONF_DIR`\ Overrides the location of the system Nix configuration directory (default `prefix/etc/nix`). - - `NIX_CONFIG` + - `NIX_CONFIG`\ Applies settings from Nix configuration from the environment. The content is treated as if it was read from a Nix configuration file. Settings are separated by the newline character. - - `NIX_USER_CONF_FILES` + - `NIX_USER_CONF_FILES`\ Overrides the location of the user Nix configuration files to load from (defaults to the XDG spec locations). The variable is treated as a list separated by the `:` token. - - `TMPDIR` + - `TMPDIR`\ Use the specified directory to store temporary files. In particular, this includes temporary build directories; these can take up substantial amounts of disk space. The default is `/tmp`. - - `NIX_REMOTE` + - `NIX_REMOTE`\ This variable should be set to `daemon` if you want to use the Nix daemon to execute Nix operations. This is necessary in [multi-user Nix installations](../installation/multi-user.md). If the Nix @@ -104,16 +108,16 @@ Most Nix commands interpret the following environment variables: should be set to `unix://path/to/socket`. Otherwise, it should be left unset. - - `NIX_SHOW_STATS` + - `NIX_SHOW_STATS`\ If set to `1`, Nix will print some evaluation statistics, such as the number of values allocated. - - `NIX_COUNT_CALLS` + - `NIX_COUNT_CALLS`\ If set to `1`, Nix will print how often functions were called during Nix expression evaluation. This is useful for profiling your Nix expressions. - - `GC_INITIAL_HEAP_SIZE` + - `GC_INITIAL_HEAP_SIZE`\ If Nix has been configured to use the Boehm garbage collector, this variable sets the initial size of the heap in bytes. It defaults to 384 MiB. Setting it to a low value reduces memory consumption, but diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md index 4565bfbc2..43de7a6e6 100644 --- a/doc/manual/src/command-ref/nix-build.md +++ b/doc/manual/src/command-ref/nix-build.md @@ -47,16 +47,16 @@ All options not listed here are passed to `nix-store --realise`, except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`. - - `--no-out-link` + - `--no-out-link`\ Do not create a symlink to the output path. Note that as a result the output does not become a root of the garbage collector, and so might be deleted by `nix-store --gc`. - - `--dry-run` + - `--dry-run`\ Show what store paths would be built or downloaded. - - `--out-link` / `-o` *outlink* + - `--out-link` / `-o` *outlink*\ Change the name of the symlink to the output path created from `result` to *outlink*. diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md index 4ca12d2cc..24353525f 100644 --- a/doc/manual/src/command-ref/nix-channel.md +++ b/doc/manual/src/command-ref/nix-channel.md @@ -17,26 +17,26 @@ To see the list of official NixOS channels, visit This command has the following operations: - - `--add` *url* \[*name*\] + - `--add` *url* \[*name*\]\ Adds a channel named *name* with URL *url* to the list of subscribed channels. If *name* is omitted, it defaults to the last component of *url*, with the suffixes `-stable` or `-unstable` removed. - - `--remove` *name* + - `--remove` *name*\ Removes the channel named *name* from the list of subscribed channels. - - `--list` + - `--list`\ Prints the names and URLs of all subscribed channels on standard output. - - `--update` \[*names*…\] + - `--update` \[*names*…\]\ Downloads the Nix expressions of all subscribed channels (or only those included in *names* if specified) and makes them the default for `nix-env` operations (by symlinking them from the directory `~/.nix-defexpr`). - - `--rollback` \[*generation*\] + - `--rollback` \[*generation*\]\ Reverts the previous call to `nix-channel --update`. Optionally, you can specify a specific channel generation number to restore. @@ -70,14 +70,14 @@ $ nix-instantiate --eval -E '(import {}).lib.version' # Files - - `/nix/var/nix/profiles/per-user/username/channels` + - `/nix/var/nix/profiles/per-user/username/channels`\ `nix-channel` uses a `nix-env` profile to keep track of previous versions of the subscribed channels. Every time you run `nix-channel --update`, a new channel generation (that is, a symlink to the channel Nix expressions in the Nix store) is created. This enables `nix-channel --rollback` to revert to previous versions. - - `~/.nix-defexpr/channels` + - `~/.nix-defexpr/channels`\ This is a symlink to `/nix/var/nix/profiles/per-user/username/channels`. It ensures that `nix-env` can find your channels. In a multi-user installation, you @@ -89,7 +89,7 @@ $ nix-instantiate --eval -E '(import {}).lib.version' A channel URL should point to a directory containing the following files: - - `nixexprs.tar.xz` + - `nixexprs.tar.xz`\ A tarball containing Nix expressions and files referenced by them (such as build scripts and patches). At the top level, the tarball should contain a single directory. That directory must contain a diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md index dcb844a72..7047d3012 100644 --- a/doc/manual/src/command-ref/nix-copy-closure.md +++ b/doc/manual/src/command-ref/nix-copy-closure.md @@ -35,21 +35,21 @@ and second to send the dump of those paths. If this bothers you, use # Options - - `--to` + - `--to`\ Copy the closure of _paths_ from the local Nix store to the Nix store on _machine_. This is the default. - - `--from` + - `--from`\ Copy the closure of _paths_ from the Nix store on _machine_ to the local Nix store. - - `--gzip` + - `--gzip`\ Enable compression of the SSH connection. - - `--include-outputs` + - `--include-outputs`\ Also copy the outputs of store derivations included in the closure. - - `--use-substitutes` / `-s` + - `--use-substitutes` / `-s`\ Attempt to download missing paths on the target machine using Nix’s substitute mechanism. Any paths that cannot be substituted on the target are still copied normally from the source. This is useful, @@ -58,12 +58,12 @@ and second to send the dump of those paths. If this bothers you, use `nixos.org` (the default binary cache server) is fast. - - `-v` + - `-v`\ Show verbose output. # Environment variables - - `NIX_SSHOPTS` + - `NIX_SSHOPTS`\ Additional options to be passed to `ssh` on the command line. diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md index 1c23bb0ad..9138fa05a 100644 --- a/doc/manual/src/command-ref/nix-env.md +++ b/doc/manual/src/command-ref/nix-env.md @@ -36,27 +36,27 @@ case-sensitive. The regular expression can optionally be followed by a dash and a version number; if omitted, any version of the package will match. Here are some examples: - - `firefox` + - `firefox`\ Matches the package name `firefox` and any version. - - `firefox-32.0` + - `firefox-32.0`\ Matches the package name `firefox` and version `32.0`. - - `gtk\\+` + - `gtk\\+`\ Matches the package name `gtk+`. The `+` character must be escaped using a backslash to prevent it from being interpreted as a quantifier, and the backslash must be escaped in turn with another backslash to ensure that the shell passes it on. - - `.\*` + - `.\*`\ Matches any package name. This is the default for most commands. - - `'.*zip.*'` + - `'.*zip.*'`\ Matches any package name containing the string `zip`. Note the dots: `'*zip*'` does not work, because in a regular expression, the character `*` is interpreted as a quantifier. - - `'.*(firefox|chromium).*'` + - `'.*(firefox|chromium).*'`\ Matches any package name containing the strings `firefox` or `chromium`. @@ -66,7 +66,7 @@ This section lists the options that are common to all operations. These options are allowed for every subcommand, though they may not always have an effect. - - `--file` / `-f` *path* + - `--file` / `-f` *path*\ Specifies the Nix expression (designated below as the *active Nix expression*) used by the `--install`, `--upgrade`, and `--query --available` operations to obtain derivations. The default is @@ -77,13 +77,13 @@ have an effect. unpacked to a temporary location. The tarball must include a single top-level directory containing at least a file named `default.nix`. - - `--profile` / `-p` *path* + - `--profile` / `-p` *path*\ Specifies the profile to be used by those operations that operate on a profile (designated below as the *active profile*). A profile is a sequence of user environments called *generations*, one of which is the *current generation*. - - `--dry-run` + - `--dry-run`\ For the `--install`, `--upgrade`, `--uninstall`, `--switch-generation`, `--delete-generations` and `--rollback` operations, this flag will cause `nix-env` to print what *would* be @@ -93,7 +93,7 @@ have an effect. [substituted](../glossary.md) (i.e., downloaded) and which paths will be built from source (because no substitute is available). - - `--system-filter` *system* + - `--system-filter` *system*\ By default, operations such as `--query --available` show derivations matching any platform. This option allows you to use derivations for the specified platform *system*. @@ -102,7 +102,7 @@ have an effect. # Files - - `~/.nix-defexpr` + - `~/.nix-defexpr`\ The source for the default Nix expressions used by the `--install`, `--upgrade`, and `--query --available` operations to obtain derivations. The `--file` option may be used to override @@ -140,7 +140,7 @@ have an effect. The command `nix-channel` places symlinks to the downloaded Nix expressions from each subscribed channel in this directory. - - `~/.nix-profile` + - `~/.nix-profile`\ A symbolic link to the user's current profile. By default, this symlink points to `prefix/var/nix/profiles/default`. The `PATH` environment variable should include `~/.nix-profile/bin` for the @@ -217,13 +217,13 @@ a number of possible ways: ## Flags - - `--prebuilt-only` / `-b` + - `--prebuilt-only` / `-b`\ Use only derivations for which a substitute is registered, i.e., there is a pre-built binary available that can be downloaded in lieu of building the derivation. Thus, no packages will be built from source. - - `--preserve-installed`; `-P` + - `--preserve-installed`; `-P`\ Do not remove derivations with a name matching one of the derivations being installed. Usually, trying to have two versions of the same package installed in the same generation of a profile will @@ -231,7 +231,7 @@ a number of possible ways: clashes between the two versions. However, this is not the case for all packages. - - `--remove-all`; `-r` + - `--remove-all`; `-r`\ Remove all previously installed packages first. This is equivalent to running `nix-env -e '.*'` first, except that everything happens in a single transaction. @@ -346,24 +346,24 @@ version is installed. ## Flags - - `--lt` + - `--lt`\ Only upgrade a derivation to newer versions. This is the default. - - `--leq` + - `--leq`\ In addition to upgrading to newer versions, also “upgrade” to derivations that have the same version. Version are not a unique identification of a derivation, so there may be many derivations that have the same version. This flag may be useful to force “synchronisation” between the installed and available derivations. - - `--eq` + - `--eq`\ *Only* “upgrade” to derivations that have the same version. This may not seem very useful, but it actually is, e.g., when there is a new release of Nixpkgs and you want to replace installed applications with the same versions built against newer dependencies (to reduce the number of dependencies floating around on your system). - - `--always` + - `--always`\ In addition to upgrading to newer versions, also “upgrade” to derivations that have the same or a lower version. I.e., derivations may actually be downgraded depending on what is available in the @@ -578,11 +578,11 @@ The derivations are sorted by their `name` attributes. The following flags specify the set of things on which the query operates. - - `--installed` + - `--installed`\ The query operates on the store paths that are installed in the current generation of the active profile. This is the default. - - `--available`; `-a` + - `--available`; `-a`\ The query operates on the derivations that are available in the active Nix expression. @@ -593,24 +593,24 @@ selected derivations. Multiple flags may be specified, in which case the information is shown in the order given here. Note that the name of the derivation is shown unless `--no-name` is specified. - - `--xml` + - `--xml`\ Print the result in an XML representation suitable for automatic processing by other tools. The root element is called `items`, which contains a `item` element for each available or installed derivation. The fields discussed below are all stored in attributes of the `item` elements. - - `--json` + - `--json`\ Print the result in a JSON representation suitable for automatic processing by other tools. - - `--prebuilt-only` / `-b` + - `--prebuilt-only` / `-b`\ Show only derivations for which a substitute is registered, i.e., there is a pre-built binary available that can be downloaded in lieu of building the derivation. Thus, this shows all packages that probably can be installed quickly. - - `--status`; `-s` + - `--status`; `-s`\ Print the *status* of the derivation. The status consists of three characters. The first is `I` or `-`, indicating whether the derivation is currently installed in the current generation of the @@ -621,49 +621,49 @@ derivation is shown unless `--no-name` is specified. derivation to be built. The third is `S` or `-`, indicating whether a substitute is available for the derivation. - - `--attr-path`; `-P` + - `--attr-path`; `-P`\ Print the *attribute path* of the derivation, which can be used to unambiguously select it using the `--attr` option available in commands that install derivations like `nix-env --install`. This option only works together with `--available` - - `--no-name` + - `--no-name`\ Suppress printing of the `name` attribute of each derivation. - - `--compare-versions` / `-c` + - `--compare-versions` / `-c`\ Compare installed versions to available versions, or vice versa (if `--available` is given). This is useful for quickly seeing whether upgrades for installed packages are available in a Nix expression. A column is added with the following meaning: - - `<` *version* + - `<` *version*\ A newer version of the package is available or installed. - - `=` *version* + - `=` *version*\ At most the same version of the package is available or installed. - - `>` *version* + - `>` *version*\ Only older versions of the package are available or installed. - - `- ?` + - `- ?`\ No version of the package is available or installed. - - `--system` + - `--system`\ Print the `system` attribute of the derivation. - - `--drv-path` + - `--drv-path`\ Print the path of the store derivation. - - `--out-path` + - `--out-path`\ Print the output path of the derivation. - - `--description` + - `--description`\ Print a short (one-line) description of the derivation, if available. The description is taken from the `meta.description` attribute of the derivation. - - `--meta` + - `--meta`\ Print all of the meta-attributes of the derivation. This option is only available with `--xml` or `--json`. @@ -874,7 +874,7 @@ error: no generation older than the current (91) exists # Environment variables - - `NIX_PROFILE` + - `NIX_PROFILE`\ Location of the Nix profile. Defaults to the target of the symlink `~/.nix-profile`, if it exists, or `/nix/var/nix/profiles/default` otherwise. diff --git a/doc/manual/src/command-ref/nix-hash.md b/doc/manual/src/command-ref/nix-hash.md index de0459b9e..45f67f1c5 100644 --- a/doc/manual/src/command-ref/nix-hash.md +++ b/doc/manual/src/command-ref/nix-hash.md @@ -29,29 +29,29 @@ md5sum`. # Options - - `--flat` + - `--flat`\ Print the cryptographic hash of the contents of each regular file *path*. That is, do not compute the hash over the dump of *path*. The result is identical to that produced by the GNU commands `md5sum` and `sha1sum`. - - `--base32` + - `--base32`\ Print the hash in a base-32 representation rather than hexadecimal. This base-32 representation is more compact and can be used in Nix expressions (such as in calls to `fetchurl`). - - `--truncate` + - `--truncate`\ Truncate hashes longer than 160 bits (such as SHA-256) to 160 bits. - - `--type` *hashAlgo* + - `--type` *hashAlgo*\ Use the specified cryptographic hash algorithm, which can be one of `md5`, `sha1`, `sha256`, and `sha512`. - - `--to-base16` + - `--to-base16`\ Don’t hash anything, but convert the base-32 hash representation *hash* to hexadecimal. - - `--to-base32` + - `--to-base32`\ Don’t hash anything, but convert the hexadecimal hash representation *hash* to base-32. diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md index c369397b6..2e198daed 100644 --- a/doc/manual/src/command-ref/nix-instantiate.md +++ b/doc/manual/src/command-ref/nix-instantiate.md @@ -29,26 +29,26 @@ standard input. # Options - - `--add-root` *path* + - `--add-root` *path*\ See the [corresponding option](nix-store.md) in `nix-store`. - - `--parse` + - `--parse`\ Just parse the input files, and print their abstract syntax trees on standard output in ATerm format. - - `--eval` + - `--eval`\ Just parse and evaluate the input files, and print the resulting values on standard output. No instantiation of store derivations takes place. - - `--find-file` + - `--find-file`\ Look up the given files in Nix’s search path (as specified by the `NIX_PATH` environment variable). If found, print the corresponding absolute paths on standard output. For instance, if `NIX_PATH` is `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`. - - `--strict` + - `--strict`\ When used with `--eval`, recursively evaluate list elements and attributes. Normally, such sub-expressions are left unevaluated (since the Nix expression language is lazy). @@ -58,17 +58,17 @@ standard input. > This option can cause non-termination, because lazy data > structures can be infinitely large. - - `--json` + - `--json`\ When used with `--eval`, print the resulting value as an JSON representation of the abstract syntax tree rather than as an ATerm. - - `--xml` + - `--xml`\ When used with `--eval`, print the resulting value as an XML representation of the abstract syntax tree rather than as an ATerm. The schema is the same as that used by the [`toXML` built-in](../expressions/builtins.md). - - `--read-write-mode` + - `--read-write-mode`\ When used with `--eval`, perform evaluation in read/write mode so nix language features that require it will still work (at the cost of needing to do instantiation of every evaluated derivation). If diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 59ab89b29..3bcd209e2 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -37,22 +37,22 @@ Nix store is also printed. # Options - - `--type` *hashAlgo* + - `--type` *hashAlgo*\ Use the specified cryptographic hash algorithm, which can be one of `md5`, `sha1`, `sha256`, and `sha512`. - - `--print-path` + - `--print-path`\ Print the store path of the downloaded file on standard output. - - `--unpack` + - `--unpack`\ Unpack the archive (which must be a tarball or zip file) and add the result to the Nix store. The resulting hash can be used with functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. - - `--executable` + - `--executable`\ Set the executable bit on the downloaded file. - - `--name` *name* + - `--name` *name*\ Override the name of the file in the Nix store. By default, this is `hash-basename`, where *basename* is the last component of *url*. Overriding the name is necessary when *basename* contains characters diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 54812a49f..72f6730f1 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -54,7 +54,7 @@ All options not listed here are passed to `nix-store --realise`, except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`. - - `--command` *cmd* + - `--command` *cmd*\ In the environment of the derivation, run the shell command *cmd*. This command is executed in an interactive shell. (Use `--run` to use a non-interactive shell instead.) However, a call to `exit` is @@ -64,36 +64,34 @@ All options not listed here are passed to `nix-store drop you into the interactive shell. This can be useful for doing any additional initialisation. - - `--run` *cmd* + - `--run` *cmd*\ Like `--command`, but executes the command in a non-interactive shell. This means (among other things) that if you hit Ctrl-C while the command is running, the shell exits. - - `--exclude` *regexp* + - `--exclude` *regexp*\ Do not build any dependencies whose store path matches the regular expression *regexp*. This option may be specified multiple times. - - `--pure` + - `--pure`\ If this flag is specified, the environment is almost entirely cleared before the interactive shell is started, so you get an environment that more closely corresponds to the “real” Nix build. A few variables, in particular `HOME`, `USER` and `DISPLAY`, are - retained. Note that (depending on your Bash - installation) `/etc/bashrc` is still sourced, so any variables set - there will affect the interactive shell. + retained. - - `--packages` / `-p` *packages*… + - `--packages` / `-p` *packages*…\ Set up an environment in which the specified packages are present. The command line arguments are interpreted as attribute names inside the Nix Packages collection. Thus, `nix-shell -p libjpeg openjdk` will start a shell in which the packages denoted by the attribute names `libjpeg` and `openjdk` are present. - - `-i` *interpreter* + - `-i` *interpreter*\ The chained script interpreter to be invoked by `nix-shell`. Only applicable in `#!`-scripts (described below). - - `--keep` *name* + - `--keep` *name*\ When a `--pure` shell is started, keep the listed environment variables. @@ -101,7 +99,7 @@ The following common options are supported: # Environment variables - - `NIX_BUILD_SHELL` + - `NIX_BUILD_SHELL`\ Shell used to start the interactive environment. Defaults to the `bash` found in `PATH`. diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index 361c20cc9..7a131dc02 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -22,7 +22,7 @@ This section lists the options that are common to all operations. These options are allowed for every subcommand, though they may not always have an effect. - - `--add-root` *path* + - `--add-root` *path*\ Causes the result of a realisation (`--realise` and `--force-realise`) to be registered as a root of the garbage collector. *path* will be created as a symlink to the resulting @@ -79,22 +79,22 @@ paths. Realisation is a somewhat overloaded term: system). If the path is already valid, we are done immediately. Otherwise, the path and any missing paths in its closure may be produced through substitutes. If there are no (successful) - subsitutes, realisation fails. + substitutes, realisation fails. The output path of each derivation is printed on standard output. (For non-derivations argument, the argument itself is printed.) The following flags are available: - - `--dry-run` + - `--dry-run`\ Print on standard error a description of what packages would be built or downloaded, without actually performing the operation. - - `--ignore-unknown` + - `--ignore-unknown`\ If a non-derivation path does not have a substitute, then silently ignore it. - - `--check` + - `--check`\ This option allows you to check whether a derivation is deterministic. It rebuilds the specified derivation and checks whether the result is bitwise-identical with the existing outputs, @@ -110,20 +110,20 @@ The following flags are available: Special exit codes: - - `100` + - `100`\ Generic build failure, the builder process returned with a non-zero exit code. - - `101` + - `101`\ Build timeout, the build was aborted because it did not complete within the specified `timeout`. - - `102` + - `102`\ Hash mismatch, the build output was rejected because it does not match the [`outputHash` attribute of the derivation](../expressions/advanced-attributes.md). - - `104` + - `104`\ Not deterministic, the build succeeded in check mode but the resulting output is not binary reproducable. @@ -170,7 +170,7 @@ access to a restricted ssh user. The following flags are available: - - `--write` + - `--write`\ Allow the connected client to request the realization of derivations. In effect, this can be used to make the host act as a remote builder. @@ -200,18 +200,18 @@ reachable via file system references from a set of “roots”, are deleted. The following suboperations may be specified: - - `--print-roots` + - `--print-roots`\ This operation prints on standard output the set of roots used by the garbage collector. - - `--print-live` + - `--print-live`\ This operation prints on standard output the set of “live” store paths, which are all the store paths reachable from the roots. Live paths should never be deleted, since that would break consistency — it would become possible that applications are installed that reference things that are no longer present in the store. - - `--print-dead` + - `--print-dead`\ This operation prints out on standard output the set of “dead” store paths, which is just the opposite of the set of live paths: any path in the store that is not live (with respect to the roots) is dead. @@ -219,7 +219,7 @@ The following suboperations may be specified: By default, all unreachable paths are deleted. The following options control what gets deleted and in what order: - - `--max-freed` *bytes* + - `--max-freed` *bytes*\ Keep deleting paths until at least *bytes* bytes have been deleted, then stop. The argument *bytes* can be followed by the multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB @@ -300,22 +300,22 @@ symlink. ## Common query options - - `--use-output`; `-u` + - `--use-output`; `-u`\ For each argument to the query that is a store derivation, apply the query to the output path of the derivation instead. - - `--force-realise`; `-f` + - `--force-realise`; `-f`\ Realise each argument to the query first (see [`nix-store --realise`](#operation---realise)). ## Queries - - `--outputs` + - `--outputs`\ Prints out the [output paths](../glossary.md) of the store derivations *paths*. These are the paths that will be produced when the derivation is built. - - `--requisites`; `-R` + - `--requisites`; `-R`\ Prints out the [closure](../glossary.md) of the store path *paths*. This query has one option: @@ -332,31 +332,31 @@ symlink. dependencies) is obtained by distributing the closure of a store derivation and specifying the option `--include-outputs`. - - `--references` + - `--references`\ Prints the set of [references](../glossary.md) of the store paths *paths*, that is, their immediate dependencies. (For *all* dependencies, use `--requisites`.) - - `--referrers` + - `--referrers`\ Prints the set of *referrers* of the store paths *paths*, that is, the store paths currently existing in the Nix store that refer to one of *paths*. Note that contrary to the references, the set of referrers is not constant; it can change as store paths are added or removed. - - `--referrers-closure` + - `--referrers-closure`\ Prints the closure of the set of store paths *paths* under the referrers relation; that is, all store paths that directly or indirectly refer to one of *paths*. These are all the path currently in the Nix store that are dependent on *paths*. - - `--deriver`; `-d` + - `--deriver`; `-d`\ Prints the [deriver](../glossary.md) of the store paths *paths*. If the path has no deriver (e.g., if it is a source file), or if the deriver is not known (e.g., in the case of a binary-only deployment), the string `unknown-deriver` is printed. - - `--graph` + - `--graph`\ Prints the references graph of the store paths *paths* in the format of the `dot` tool of AT\&T's [Graphviz package](http://www.graphviz.org/). This can be used to visualise @@ -364,39 +364,39 @@ symlink. this to a store derivation. To obtain a runtime dependency graph, apply it to an output path. - - `--tree` + - `--tree`\ Prints the references graph of the store paths *paths* as a nested ASCII tree. References are ordered by descending closure size; this tends to flatten the tree, making it more readable. The query only recurses into a store path when it is first encountered; this prevents a blowup of the tree representation of the graph. - - `--graphml` + - `--graphml`\ Prints the references graph of the store paths *paths* in the [GraphML](http://graphml.graphdrawing.org/) file format. This can be used to visualise dependency graphs. To obtain a build-time dependency graph, apply this to a store derivation. To obtain a runtime dependency graph, apply it to an output path. - - `--binding` *name*; `-b` *name* + - `--binding` *name*; `-b` *name*\ Prints the value of the attribute *name* (i.e., environment variable) of the store derivations *paths*. It is an error for a derivation to not have the specified attribute. - - `--hash` + - `--hash`\ Prints the SHA-256 hash of the contents of the store paths *paths* (that is, the hash of the output of `nix-store --dump` on the given paths). Since the hash is stored in the Nix database, this is a fast operation. - - `--size` + - `--size`\ Prints the size in bytes of the contents of the store paths *paths* — to be precise, the size of the output of `nix-store --dump` on the given paths. Note that the actual disk space required by the store paths may be higher, especially on filesystems with large cluster sizes. - - `--roots` + - `--roots`\ Prints the garbage collector roots that point, directly or indirectly, at the store paths *paths*. @@ -513,7 +513,7 @@ public url or broke since the download expression was written. This operation has the following options: - - `--recursive` + - `--recursive`\ Use recursive instead of flat hashing mode, used when adding directories to the store. @@ -540,14 +540,14 @@ being modified by non-Nix tools, or of bugs in Nix itself. This operation has the following options: - - `--check-contents` + - `--check-contents`\ Checks that the contents of every valid store path has not been altered by computing a SHA-256 hash of the contents and comparing it with the hash stored in the Nix database at build time. Paths that have been modified are printed out. For large stores, `--check-contents` is obviously quite slow. - - `--repair` + - `--repair`\ If any valid path is missing from the store, or (if `--check-contents` is given) the contents of a valid path has been modified, then try to repair the path by redownloading it. See diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index bc8eb6796..47862bc09 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -2,56 +2,56 @@ Most Nix commands accept the following command-line options: - - `--help` + - `--help`\ Prints out a summary of the command syntax and exits. - - `--version` + - `--version`\ Prints out the Nix version number on standard output and exits. - - `--verbose` / `-v` + - `--verbose` / `-v`\ Increases the level of verbosity of diagnostic messages printed on standard error. For each Nix operation, the information printed on standard output is well-defined; any diagnostic information is printed on standard error, never on standard output. - + This option may be specified repeatedly. Currently, the following verbosity levels exist: - - - 0 + + - 0\ “Errors only”: only print messages explaining why the Nix invocation failed. - - - 1 + + - 1\ “Informational”: print *useful* messages about what Nix is doing. This is the default. - - - 2 + + - 2\ “Talkative”: print more informational messages. - - - 3 + + - 3\ “Chatty”: print even more informational messages. - - - 4 + + - 4\ “Debug”: print debug information. - - - 5 + + - 5\ “Vomit”: print vast amounts of debug information. - - `--quiet` + - `--quiet`\ Decreases the level of verbosity of diagnostic messages printed on standard error. This is the inverse option to `-v` / `--verbose`. - + This option may be specified repeatedly. See the previous verbosity levels list. - - `--log-format` *format* + - `--log-format` *format*\ This option can be used to change the output of the log format, with *format* being one of: - - - raw + + - raw\ This is the raw format, as outputted by nix-build. - - - internal-json + + - internal-json\ Outputs the logs in a structured manner. > **Warning** @@ -60,30 +60,30 @@ Most Nix commands accept the following command-line options: > the error-messages (namely of the `msg`-field) can change > between releases. - - bar + - bar\ Only display a progress bar during the builds. - - - bar-with-logs + + - bar-with-logs\ Display the raw logs, with the progress bar at the bottom. - - `--no-build-output` / `-Q` + - `--no-build-output` / `-Q`\ By default, output written by builders to standard output and standard error is echoed to the Nix command's standard error. This option suppresses this behaviour. Note that the builder's standard output and error are always written to a log file in `prefix/nix/var/log/nix`. - - `--max-jobs` / `-j` *number* + - `--max-jobs` / `-j` *number*\ Sets the maximum number of build jobs that Nix will perform in parallel to the specified number. Specify `auto` to use the number of CPUs in the system. The default is specified by the `max-jobs` configuration setting, which itself defaults to `1`. A higher value is useful on SMP systems or to exploit I/O latency. - + Setting it to `0` disallows building on the local machine, which is useful when you want builds to happen only on remote builders. - - `--cores` + - `--cores`\ Sets the value of the `NIX_BUILD_CORES` environment variable in the invocation of builders. Builders can use this variable at their discretion to control the maximum amount of parallelism. For @@ -94,18 +94,18 @@ Most Nix commands accept the following command-line options: means that the builder should use all available CPU cores in the system. - - `--max-silent-time` + - `--max-silent-time`\ Sets the maximum number of seconds that a builder can go without producing any data on standard output or standard error. The default is specified by the `max-silent-time` configuration setting. `0` means no time-out. - - `--timeout` + - `--timeout`\ Sets the maximum number of seconds that a builder can run. The default is specified by the `timeout` configuration setting. `0` means no timeout. - - `--keep-going` / `-k` + - `--keep-going` / `-k`\ Keep going in case of failed builds, to the greatest extent possible. That is, if building an input of some derivation fails, Nix will still build the other inputs, but not the derivation @@ -113,17 +113,17 @@ Most Nix commands accept the following command-line options: for builds of substitutes), possibly killing builds in progress (in case of parallel or distributed builds). - - `--keep-failed` / `-K` + - `--keep-failed` / `-K`\ Specifies that in case of a build failure, the temporary directory (usually in `/tmp`) in which the build takes place should not be deleted. The path of the build directory is printed as an informational message. - - `--fallback` + - `--fallback`\ Whenever Nix attempts to build a derivation for which substitutes are known for each output path, but realising the output paths through the substitutes fails, fall back on building the derivation. - + The most common scenario in which this is useful is when we have registered substitutes in order to perform binary distribution from, say, a network repository. If the repository is down, the @@ -134,12 +134,12 @@ Most Nix commands accept the following command-line options: failure in obtaining the substitutes to lead to a full build from source (with the related consumption of resources). - - `--readonly-mode` + - `--readonly-mode`\ When this option is used, no attempt is made to open the Nix database. Most Nix operations do need database access, so those operations will fail. - - `--arg` *name* *value* + - `--arg` *name* *value*\ This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell` and `nix-build`. When evaluating Nix expressions, the expression evaluator will automatically try to call functions that @@ -151,7 +151,7 @@ Most Nix commands accept the following command-line options: override a default value). That is, if the evaluator encounters a function with an argument named *name*, it will call it with value *value*. - + For instance, the top-level `default.nix` in Nixpkgs is actually a function: @@ -161,7 +161,7 @@ Most Nix commands accept the following command-line options: ... }: ... ``` - + So if you call this Nix expression (e.g., when you do `nix-env -i pkgname`), the function will be called automatically using the value [`builtins.currentSystem`](../expressions/builtins.md) for @@ -170,13 +170,13 @@ Most Nix commands accept the following command-line options: since the argument is a Nix string literal, you have to escape the quotes.) - - `--argstr` *name* *value* + - `--argstr` *name* *value*\ This option is like `--arg`, only the value is not a Nix expression but a string. So instead of `--arg system \"i686-linux\"` (the outer quotes are to keep the shell happy) you can say `--argstr system i686-linux`. - - `--attr` / `-A` *attrPath* + - `--attr` / `-A` *attrPath*\ Select an attribute from the top-level Nix expression being evaluated. (`nix-env`, `nix-instantiate`, `nix-build` and `nix-shell` only.) The *attribute path* *attrPath* is a sequence @@ -185,34 +185,34 @@ Most Nix commands accept the following command-line options: would cause the expression `e.xorg.xorgserver` to be used. See [`nix-env --install`](nix-env.md#operation---install) for some concrete examples. - + In addition to attribute names, you can also specify array indices. For instance, the attribute path `foo.3.bar` selects the `bar` attribute of the fourth element of the array in the `foo` attribute of the top-level expression. - - `--expr` / `-E` + - `--expr` / `-E`\ Interpret the command line arguments as a list of Nix expressions to be parsed and evaluated, rather than as a list of file names of Nix expressions. (`nix-instantiate`, `nix-build` and `nix-shell` only.) - + For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. If you want to get a shell which contain the *built* packages ready for use, give your expression to the `nix-shell -p` convenience flag instead. - - `-I` *path* + - `-I` *path*\ Add a path to the Nix expression search path. This option may be given multiple times. See the `NIX_PATH` environment variable for information on the semantics of the Nix search path. Paths added through `-I` take precedence over `NIX_PATH`. - - `--option` *name* *value* + - `--option` *name* *value*\ Set the Nix configuration option *name* to *value*. This overrides settings in the Nix configuration file (see nix.conf5). - - `--repair` + - `--repair`\ Fix corrupted or missing store paths by redownloading or rebuilding them. Note that this is slow because it requires computing a cryptographic hash of the contents of every path in the closure of diff --git a/doc/manual/src/expressions/advanced-attributes.md b/doc/manual/src/expressions/advanced-attributes.md index 31ebadda1..5b208df67 100644 --- a/doc/manual/src/expressions/advanced-attributes.md +++ b/doc/manual/src/expressions/advanced-attributes.md @@ -2,14 +2,14 @@ Derivations can declare some infrequently used optional attributes. - - `allowedReferences` + - `allowedReferences`\ The optional attribute `allowedReferences` specifies a list of legal references (dependencies) of the output of the builder. For example, - + ```nix allowedReferences = []; ``` - + enforces that the output of a derivation cannot have any runtime dependencies on its inputs. To allow an output to have a runtime dependency on itself, use `"out"` as a list item. This is used in @@ -17,45 +17,45 @@ Derivations can declare some infrequently used optional attributes. booting Linux don’t have accidental dependencies on other paths in the Nix store. - - `allowedRequisites` + - `allowedRequisites`\ This attribute is similar to `allowedReferences`, but it specifies the legal requisites of the whole closure, so all the dependencies recursively. For example, - + ```nix allowedRequisites = [ foobar ]; ``` - + enforces that the output of a derivation cannot have any other runtime dependency than `foobar`, and in addition it enforces that `foobar` itself doesn't introduce any other dependency itself. - - `disallowedReferences` + - `disallowedReferences`\ The optional attribute `disallowedReferences` specifies a list of illegal references (dependencies) of the output of the builder. For example, - + ```nix disallowedReferences = [ foo ]; ``` - + enforces that the output of a derivation cannot have a direct runtime dependencies on the derivation `foo`. - - `disallowedRequisites` + - `disallowedRequisites`\ This attribute is similar to `disallowedReferences`, but it specifies illegal requisites for the whole closure, so all the dependencies recursively. For example, - + ```nix disallowedRequisites = [ foobar ]; ``` - + enforces that the output of a derivation cannot have any runtime dependency on `foobar` or any other derivation depending recursively on `foobar`. - - `exportReferencesGraph` + - `exportReferencesGraph`\ This attribute allows builders access to the references graph of their inputs. The attribute is a list of inputs in the Nix store whose references graph the builder needs to know. The value of @@ -65,17 +65,17 @@ Derivations can declare some infrequently used optional attributes. files have the format used by `nix-store --register-validity` (with the deriver fields left empty). For example, when the following derivation is built: - + ```nix derivation { ... exportReferencesGraph = [ "libfoo-graph" libfoo ]; }; ``` - + the references graph of `libfoo` is placed in the file `libfoo-graph` in the temporary build directory. - + `exportReferencesGraph` is useful for builders that want to do something with the closure of a store path. Examples include the builders in NixOS that generate the initial ramdisk for booting @@ -84,66 +84,66 @@ Derivations can declare some infrequently used optional attributes. with a Nix store containing the closure of a bootable NixOS configuration). - - `impureEnvVars` + - `impureEnvVars`\ This attribute allows you to specify a list of environment variables that should be passed from the environment of the calling user to the builder. Usually, the environment is cleared completely when the builder is executed, but with this attribute you can allow specific environment variables to be passed unmodified. For example, `fetchurl` in Nixpkgs has the line - + ```nix impureEnvVars = [ "http_proxy" "https_proxy" ... ]; ``` - + to make it use the proxy server configuration specified by the user in the environment variables `http_proxy` and friends. - + This attribute is only allowed in *fixed-output derivations* (see below), where impurities such as these are okay since (the hash of) the output is known in advance. It is ignored for all other derivations. - + > **Warning** - > + > > `impureEnvVars` implementation takes environment variables from > the current builder process. When a daemon is building its > environmental variables are used. Without the daemon, the > environmental variables come from the environment of the > `nix-build`. - - `outputHash`; `outputHashAlgo`; `outputHashMode` + - `outputHash`; `outputHashAlgo`; `outputHashMode`\ These attributes declare that the derivation is a so-called *fixed-output derivation*, which means that a cryptographic hash of the output is already known in advance. When the build of a fixed-output derivation finishes, Nix computes the cryptographic hash of the output and compares it to the hash declared with these attributes. If there is a mismatch, the build fails. - + The rationale for fixed-output derivations is derivations such as those produced by the `fetchurl` function. This function downloads a file from a given URL. To ensure that the downloaded file has not been modified, the caller must also specify a cryptographic hash of the file. For example, - + ```nix fetchurl { url = "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz"; sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; } ``` - + It sometimes happens that the URL of the file changes, e.g., because servers are reorganised or no longer available. We then must update the call to `fetchurl`, e.g., - + ```nix fetchurl { url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz"; sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; } ``` - + If a `fetchurl` derivation was treated like a normal derivation, the output paths of the derivation and *all derivations depending on it* would change. For instance, if we were to change the URL of the @@ -151,16 +151,16 @@ Derivations can declare some infrequently used optional attributes. other packages depend) massive rebuilds would be needed. This is unfortunate for a change which we know cannot have a real effect as it propagates upwards through the dependency graph. - + For fixed-output derivations, on the other hand, the name of the output path only depends on the `outputHash*` and `name` attributes, while all other attributes are ignored for the purpose of computing the output path. (The `name` attribute is included because it is part of the path.) - + As an example, here is the (simplified) Nix expression for `fetchurl`: - + ```nix { stdenv, curl }: # The curl program is used for downloading. @@ -180,43 +180,51 @@ Derivations can declare some infrequently used optional attributes. inherit url; } ``` - + The `outputHashAlgo` attribute specifies the hash algorithm used to compute the hash. It can currently be `"sha1"`, `"sha256"` or `"sha512"`. - + The `outputHashMode` attribute determines how the hash is computed. It must be one of the following two values: - - - `"flat"` + + - `"flat"`\ The output must be a non-executable regular file. If it isn’t, the build fails. The hash is simply computed over the contents of that file (so it’s equal to what Unix commands like `sha256sum` or `sha1sum` produce). - + This is the default. - - - `"recursive"` + + - `"recursive"`\ The hash is computed over the NAR archive dump of the output (i.e., the result of [`nix-store --dump`](../command-ref/nix-store.md#operation---dump)). In this case, the output can be anything, including a directory tree. - + The `outputHash` attribute, finally, must be a string containing the hash in either hexadecimal or base-32 notation. (See the [`nix-hash` command](../command-ref/nix-hash.md) for information about converting to and from base-32 notation.) + + - `__contentAddressed` + If this **experimental** attribute is set to true, then the derivation + outputs will be stored in a content-addressed location rather than the + traditional input-addressed one. + This only has an effect if the `ca-derivation` experimental feature is enabled. + + Setting this attribute also requires setting `outputHashMode` and `outputHashAlgo` like for *fixed-output derivations* (see above). - - `passAsFile` + - `passAsFile`\ A list of names of attributes that should be passed via files rather than environment variables. For example, if you have - + ```nix passAsFile = ["big"]; big = "a very long string"; ``` - + then when the builder runs, the environment variable `bigPath` will contain the absolute path to a temporary file containing `a very long string`. That is, for any attribute *x* listed in @@ -226,7 +234,7 @@ Derivations can declare some infrequently used optional attributes. builder, since most operating systems impose a limit on the size of the environment (typically, a few hundred kilobyte). - - `preferLocalBuild` + - `preferLocalBuild`\ If this attribute is set to `true` and [distributed building is enabled](../advanced-topics/distributed-builds.md), then, if possible, the derivaton will be built locally instead of forwarded @@ -234,14 +242,14 @@ Derivations can declare some infrequently used optional attributes. where the cost of doing a download or remote build would exceed the cost of building locally. - - `allowSubstitutes` + - `allowSubstitutes`\ If this attribute is set to `false`, then Nix will always build this derivation; it will not try to substitute its outputs. This is useful for very trivial derivations (such as `writeText` in Nixpkgs) that are cheaper to build than to substitute from a binary cache. - + > **Note** - > + > > You need to have a builder configured which satisfies the > derivation’s `system` attribute, since the derivation cannot be > substituted. Thus it is usually a good idea to align `system` with diff --git a/doc/manual/src/expressions/builtin-constants.md b/doc/manual/src/expressions/builtin-constants.md index 3345a715b..1404289e5 100644 --- a/doc/manual/src/expressions/builtin-constants.md +++ b/doc/manual/src/expressions/builtin-constants.md @@ -2,7 +2,7 @@ Here are the constants built into the Nix expression evaluator: - - `builtins` + - `builtins`\ The set `builtins` contains all the built-in functions and values. You can use `builtins` to test for the availability of features in the Nix installation, e.g., @@ -14,7 +14,7 @@ Here are the constants built into the Nix expression evaluator: This allows a Nix expression to fall back gracefully on older Nix installations that don’t have the desired built-in function. - - `builtins.currentSystem` + - `builtins.currentSystem`\ The built-in value `currentSystem` evaluates to the Nix platform identifier for the Nix installation on which the expression is being evaluated, such as `"i686-linux"` or `"x86_64-darwin"`. diff --git a/doc/manual/src/expressions/builtins-prefix.md b/doc/manual/src/expressions/builtins-prefix.md index 0f7c3d32f..c16b2805f 100644 --- a/doc/manual/src/expressions/builtins-prefix.md +++ b/doc/manual/src/expressions/builtins-prefix.md @@ -9,7 +9,7 @@ scope. Instead, you can access them through the `builtins` built-in value, which is a set that contains all built-in functions and values. For instance, `derivation` is also available as `builtins.derivation`. - - `derivation` *attrs*; `builtins.derivation` *attrs* + - `derivation` *attrs*; `builtins.derivation` *attrs*\ `derivation` is described in [its own section](derivations.md). diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 56af9cd85..bb350d9de 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -1,48 +1,48 @@ # Glossary - - derivation + - derivation\ A description of a build action. The result of a derivation is a store object. Derivations are typically specified in Nix expressions using the [`derivation` primitive](expressions/derivations.md). These are translated into low-level *store derivations* (implicitly by `nix-env` and `nix-build`, or explicitly by `nix-instantiate`). - - store + - store\ The location in the file system where store objects live. Typically `/nix/store`. - - store path + - store path\ The location in the file system of a store object, i.e., an immediate child of the Nix store directory. - - store object + - store object\ A file that is an immediate child of the Nix store directory. These can be regular files, but also entire directory trees. Store objects can be sources (objects copied from outside of the store), derivation outputs (objects produced by running a build action), or derivations (files describing a build action). - - substitute + - substitute\ A substitute is a command invocation stored in the Nix database that describes how to build a store object, bypassing the normal build mechanism (i.e., derivations). Typically, the substitute builds the store object by downloading a pre-built version of the store object from some server. - - purity + - purity\ The assumption that equal Nix derivations when run always produce the same output. This cannot be guaranteed in general (e.g., a builder can rely on external inputs such as the network or the system time) but the Nix model assumes it. - - Nix expression + - Nix expression\ A high-level description of software packages and compositions thereof. Deploying software using Nix entails writing Nix expressions for your packages. Nix expressions are translated to derivations that are stored in the Nix store. These derivations can then be built. - - reference + - reference\ A store path `P` is said to have a reference to a store path `Q` if the store object at `P` contains the path `Q` somewhere. The *references* of a store path are the set of store paths to which it @@ -52,11 +52,11 @@ output paths), whereas an output path only references other output paths. - - reachable + - reachable\ A store path `Q` is reachable from another store path `P` if `Q` is in the *closure* of the *references* relation. - - closure + - closure\ The closure of a store path is the set of store paths that are directly or indirectly “reachable” from that store path; that is, it’s the closure of the path under the *references* relation. For @@ -71,29 +71,29 @@ to path `Q`, then `Q` is in the closure of `P`. Further, if `Q` references `R` then `R` is also in the closure of `P`. - - output path + - output path\ A store path produced by a derivation. - - deriver + - deriver\ The deriver of an *output path* is the store derivation that built it. - - validity + - validity\ A store path is considered *valid* if it exists in the file system, is listed in the Nix database as being valid, and if all paths in its closure are also valid. - - user environment + - user environment\ An automatically generated store object that consists of a set of symlinks to “active” applications, i.e., other store paths. These are generated automatically by [`nix-env`](command-ref/nix-env.md). See *profiles*. - - profile + - profile\ A symlink to the current *user environment* of a user, e.g., `/nix/var/nix/profiles/default`. - - NAR + - NAR\ A *N*ix *AR*chive. This is a serialisation of a path in the Nix store. It can contain regular files, directories and symbolic links. NARs are generated and unpacked using `nix-store --dump` diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index ae7fd458b..96fa34635 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -1,18 +1,26 @@ # Installing a Binary Distribution -If you are using Linux or macOS versions up to 10.14 (Mojave), the -easiest way to install Nix is to run the following command: +The easiest way to install Nix is to run the following command: ```console $ sh <(curl -L https://nixos.org/nix/install) ``` -If you're using macOS 10.15 (Catalina) or newer, consult [the macOS -installation instructions](#macos-installation) before installing. +This will run the installer interactively (causing it to explain what +it is doing more explicitly), and perform the default "type" of install +for your platform: +- single-user on Linux +- multi-user on macOS -As of Nix 2.1.0, the Nix installer will always default to creating a -single-user installation, however opting in to the multi-user -installation is highly recommended. + > **Notes on read-only filesystem root in macOS 10.15 Catalina +** + > + > - It took some time to support this cleanly. You may see posts, + > examples, and tutorials using obsolete workarounds. + > - Supporting it cleanly made macOS installs too complex to qualify + > as single-user, so this type is no longer supported on macOS. + +We recommend the multi-user install if it supports your platform and +you can authenticate with `sudo`. # Single User Installation @@ -50,9 +58,9 @@ $ rm -rf /nix The multi-user Nix installation creates system users, and a system service for the Nix daemon. - - Linux running systemd, with SELinux disabled - - - macOS +**Supported Systems** +- Linux running systemd, with SELinux disabled +- macOS You can instruct the installer to perform a multi-user installation on your system: @@ -96,165 +104,28 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist There may also be references to Nix in `/etc/profile`, `/etc/bashrc`, and `/etc/zshrc` which you may remove. -# macOS Installation +# macOS Installation + -Starting with macOS 10.15 (Catalina), the root filesystem is read-only. -This means `/nix` can no longer live on your system volume, and that -you'll need a workaround to install Nix. +We believe we have ironed out how to cleanly support the read-only root +on modern macOS. New installs will do this automatically, and you can +also re-run a new installer to convert your existing setup. -The recommended approach, which creates an unencrypted APFS volume for -your Nix store and a "synthetic" empty directory to mount it over at -`/nix`, is least likely to impair Nix or your system. +This section previously detailed the situation, options, and trade-offs, +but it now only outlines what the installer does. You don't need to know +this to run the installer, but it may help if you run into trouble: -> **Note** -> -> With all separate-volume approaches, it's possible something on your -> system (particularly daemons/services and restored apps) may need -> access to your Nix store before the volume is mounted. Adding -> additional encryption makes this more likely. - -If you're using a recent Mac with a [T2 -chip](https://www.apple.com/euro/mac/shared/docs/Apple_T2_Security_Chip_Overview.pdf), -your drive will still be encrypted at rest (in which case "unencrypted" -is a bit of a misnomer). To use this approach, just install Nix with: - -```console -$ sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume -``` - -If you don't like the sound of this, you'll want to weigh the other -approaches and tradeoffs detailed in this section. - -> **Note** -> -> All of the known workarounds have drawbacks, but we hope better -> solutions will be available in the future. Some that we have our eye -> on are: -> -> 1. A true firmlink would enable the Nix store to live on the primary -> data volume without the build problems caused by the symlink -> approach. End users cannot currently create true firmlinks. -> -> 2. If the Nix store volume shared FileVault encryption with the -> primary data volume (probably by using the same volume group and -> role), FileVault encryption could be easily supported by the -> installer without requiring manual setup by each user. - -## Change the Nix store path prefix - -Changing the default prefix for the Nix store is a simple approach which -enables you to leave it on your root volume, where it can take full -advantage of FileVault encryption if enabled. Unfortunately, this -approach also opts your device out of some benefits that are enabled by -using the same prefix across systems: - - - Your system won't be able to take advantage of the binary cache - (unless someone is able to stand up and support duplicate caching - infrastructure), which means you'll spend more time waiting for - builds. - - - It's harder to build and deploy packages to Linux systems. - -It would also possible (and often requested) to just apply this change -ecosystem-wide, but it's an intrusive process that has side effects we -want to avoid for now. - -## Use a separate encrypted volume - -If you like, you can also add encryption to the recommended approach -taken by the installer. You can do this by pre-creating an encrypted -volume before you run the installer--or you can run the installer and -encrypt the volume it creates later. - -In either case, adding encryption to a second volume isn't quite as -simple as enabling FileVault for your boot volume. Before you dive in, -there are a few things to weigh: - -1. The additional volume won't be encrypted with your existing - FileVault key, so you'll need another mechanism to decrypt the - volume. - -2. You can store the password in Keychain to automatically decrypt the - volume on boot--but it'll have to wait on Keychain and may not mount - before your GUI apps restore. If any of your launchd agents or apps - depend on Nix-installed software (for example, if you use a - Nix-installed login shell), the restore may fail or break. - - On a case-by-case basis, you may be able to work around this problem - by using `wait4path` to block execution until your executable is - available. - - It's also possible to decrypt and mount the volume earlier with a - login hook--but this mechanism appears to be deprecated and its - future is unclear. - -3. You can hard-code the password in the clear, so that your store - volume can be decrypted before Keychain is available. - -If you are comfortable navigating these tradeoffs, you can encrypt the -volume with something along the lines of: - -```console -$ diskutil apfs enableFileVault /nix -user disk -``` - -## Symlink the Nix store to a custom location - -Another simple approach is using `/etc/synthetic.conf` to symlink the -Nix store to the data volume. This option also enables your store to -share any configured FileVault encryption. Unfortunately, builds that -resolve the symlink may leak the canonical path or even fail. - -Because of these downsides, we can't recommend this approach. - -## Notes on the recommended approach - -This section goes into a little more detail on the recommended approach. -You don't need to understand it to run the installer, but it can serve -as a helpful reference if you run into trouble. - -1. In order to compose user-writable locations into the new read-only - system root, Apple introduced a new concept called `firmlinks`, - which it describes as a "bi-directional wormhole" between two - filesystems. You can see the current firmlinks in - `/usr/share/firmlinks`. Unfortunately, firmlinks aren't (currently?) - user-configurable. - - For special cases like NFS mount points or package manager roots, - [synthetic.conf(5)](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man5/synthetic.conf.5.html) - supports limited user-controlled file-creation (of symlinks, and - synthetic empty directories) at `/`. To create a synthetic empty - directory for mounting at `/nix`, add the following line to - `/etc/synthetic.conf` (create it if necessary): - - nix - -2. This configuration is applied at boot time, but you can use - `apfs.util` to trigger creation (not deletion) of new entries - without a reboot: - - ```console - $ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B - ``` - -3. Create the new APFS volume with diskutil: - - ```console - $ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix - ``` - -4. Using `vifs`, add the new mount to `/etc/fstab`. If it doesn't - already have other entries, it should look something like: - - # - # Warning - this file should only be modified with vifs(8) - # - # Failure to do so is unsupported and may be destructive. - # - LABEL=Nix\040Store /nix apfs rw,nobrowse - - The nobrowse setting will keep Spotlight from indexing this volume, - and keep it from showing up on your desktop. +- create a new APFS volume for your Nix store +- update `/etc/synthetic.conf` to direct macOS to create a "synthetic" + empty root directory to mount your volume +- specify mount options for the volume in `/etc/fstab` +- if you have FileVault enabled + - generate an encryption password + - put it in your system Keychain + - use it to encrypt the volume +- create a system LaunchDaemon to mount this volume early enough in the + boot process to avoid problems loading or restoring any programs that + need access to your Nix store # Installing a pinned Nix version from a URL diff --git a/doc/manual/src/package-management/s3-substituter.md b/doc/manual/src/package-management/s3-substituter.md index a4f4d561f..30f2b2e11 100644 --- a/doc/manual/src/package-management/s3-substituter.md +++ b/doc/manual/src/package-management/s3-substituter.md @@ -7,17 +7,17 @@ cache mechanism that Nix usually uses to fetch prebuilt binaries from The following options can be specified as URL parameters to the S3 URL: - - `profile` + - `profile`\ The name of the AWS configuration profile to use. By default Nix will use the `default` profile. - - `region` + - `region`\ The region of the S3 bucket. `us–east-1` by default. If your bucket is not in `us–east-1`, you should always explicitly specify the region parameter. - - `endpoint` + - `endpoint`\ The URL to your S3-compatible service, for when not using Amazon S3. Do not specify this value if you're using Amazon S3. @@ -26,7 +26,7 @@ The following options can be specified as URL parameters to the S3 URL: > This endpoint must support HTTPS and will use path-based > addressing instead of virtual host based addressing. - - `scheme` + - `scheme`\ The scheme used for S3 requests, `https` (default) or `http`. This option allows you to disable HTTPS for binary caches which don't support it. diff --git a/flake.lock b/flake.lock index 9867e694b..8aad22957 100644 --- a/flake.lock +++ b/flake.lock @@ -1,22 +1,40 @@ { "nodes": { + "lowdown-src": { + "flake": false, + "locked": { + "lastModified": 1617481909, + "narHash": "sha256-SqnfOFuLuVRRNeVJr1yeEPJue/qWoCp5N6o5Kr///p4=", + "owner": "kristapsdz", + "repo": "lowdown", + "rev": "148f9b2f586c41b7e36e73009db43ea68c7a1a4d", + "type": "github" + }, + "original": { + "owner": "kristapsdz", + "ref": "VERSION_0_8_4", + "repo": "lowdown", + "type": "github" + } + }, "nixpkgs": { "locked": { - "lastModified": 1614309161, - "narHash": "sha256-93kRxDPyEW9QIpxU71kCaV1r+hgOgP6/aVgC7vvO8IU=", + "lastModified": 1622593737, + "narHash": "sha256-9loxFJg85AbzJrSkU4pE/divZ1+zOxDy2FSjlrufCB8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "0e499fde7af3c28d63e9b13636716b86c3162b93", + "rev": "bb8a5e54845012ed1375ffd5f317d2fdf434b20e", "type": "github" }, "original": { "id": "nixpkgs", - "ref": "nixos-20.09-small", + "ref": "nixos-21.05-small", "type": "indirect" } }, "root": { "inputs": { + "lowdown-src": "lowdown-src", "nixpkgs": "nixpkgs" } } diff --git a/flake.nix b/flake.nix index 1fb5911c8..829cd2804 100644 --- a/flake.nix +++ b/flake.nix @@ -1,10 +1,10 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "nixpkgs/nixos-20.09-small"; - #inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; + inputs.nixpkgs.url = "nixpkgs/nixos-21.05-small"; + inputs.lowdown-src = { url = "github:kristapsdz/lowdown/VERSION_0_8_4"; flake = false; }; - outputs = { self, nixpkgs }: + outputs = { self, nixpkgs, lowdown-src }: let @@ -18,7 +18,7 @@ linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ]; linuxSystems = linux64BitSystems ++ [ "i686-linux" ]; - systems = linuxSystems ++ [ "x86_64-darwin" ]; + systems = linuxSystems ++ [ "x86_64-darwin" "aarch64-darwin" ]; crossSystems = [ "armv6l-linux" "armv7l-linux" ]; @@ -80,11 +80,12 @@ buildPackages.git buildPackages.mercurial buildPackages.jq - ] ++ lib.optional stdenv.hostPlatform.isLinux buildPackages.utillinuxMinimal; + ] + ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]; buildDeps = [ curl - bzip2 xz brotli zlib editline + bzip2 xz brotli editline openssl sqlite libarchive boost @@ -92,7 +93,7 @@ lowdown gmock ] - ++ lib.optional stdenv.isLinux libseccomp + ++ lib.optionals stdenv.isLinux [libseccomp] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; @@ -146,12 +147,46 @@ echo "file installer $out/install" >> $out/nix-support/hydra-build-products ''; + testNixVersions = pkgs: client: daemon: with commonDeps pkgs; pkgs.stdenv.mkDerivation { + NIX_DAEMON_PACKAGE = daemon; + NIX_CLIENT_PACKAGE = client; + # Must keep this name short as OSX has a rather strict limit on the + # socket path length, and this name appears in the path of the + # nix-daemon socket used in the tests + name = "nix-tests"; + inherit version; + + src = self; + + VERSION_SUFFIX = versionSuffix; + + nativeBuildInputs = nativeBuildDeps; + buildInputs = buildDeps ++ awsDeps; + propagatedBuildInputs = propagatedDeps; + + enableParallelBuilding = true; + + dontBuild = true; + doInstallCheck = true; + + installPhase = '' + mkdir -p $out + ''; + installCheckPhase = "make installcheck"; + + }; + in { # A Nixpkgs overlay that overrides the 'nix' and # 'nix.perl-bindings' packages. overlay = final: prev: { + # An older version of Nix to test against when using the daemon. + # Currently using `nixUnstable` as the stable one doesn't respect + # `NIX_DAEMON_SOCKET_PATH` which is needed for the tests. + nixStable = prev.nix; + nix = with final; with commonDeps pkgs; stdenv.mkDerivation { name = "nix-${version}"; inherit version; @@ -201,6 +236,8 @@ separateDebugInfo = true; + strictDeps = true; + passthru.perl-bindings = with final; stdenv.mkDerivation { name = "nix-perl-${version}"; @@ -221,7 +258,8 @@ boost nlohmann_json ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium; + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium + ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; configureFlags = '' --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix} @@ -236,21 +274,23 @@ }; lowdown = with final; stdenv.mkDerivation rec { - name = "lowdown-0.8.0"; + name = "lowdown-0.8.4"; + /* src = fetchurl { url = "https://kristaps.bsd.lv/lowdown/snapshots/${name}.tar.gz"; hash = "sha512-U9WeGoInT9vrawwa57t6u9dEdRge4/P+0wLxmQyOL9nhzOEUU2FRz2Be9H0dCjYE7p2v3vCXIYk40M+jjULATw=="; }; + */ - #src = lowdown-src; + src = lowdown-src; outputs = [ "out" "bin" "dev" ]; nativeBuildInputs = [ buildPackages.which ]; - configurePhase = - '' + configurePhase = '' + ${if (stdenv.isDarwin && stdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""} ./configure \ PREFIX=${placeholder "dev"} \ BINDIR=${placeholder "bin"}/bin @@ -353,7 +393,7 @@ # to https://nixos.org/nix/install. It downloads the binary # tarball for the user's system and calls the second half of the # installation script. - installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]; + installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ]; installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" ]; # Line coverage analysis. @@ -439,6 +479,15 @@ checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; perlBindings = self.hydraJobs.perlBindings.${system}; + installTests = + let pkgs = nixpkgsFor.${system}; in + pkgs.runCommand "install-tests" { + againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; + againstCurrentUnstable = testNixVersions pkgs pkgs.nix pkgs.nixUnstable; + # Disabled because the latest stable version doesn't handle + # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work + # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; + } "touch $out"; }); packages = forAllSystems (system: { @@ -479,6 +528,8 @@ installCheckFlags = "sysconfdir=$(out)/etc"; stripAllList = ["bin"]; + + strictDeps = true; }; } // builtins.listToAttrs (map (crossSystem: { name = "nix-${crossSystem}"; diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 6f3882a12..c2933300f 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -133,20 +133,8 @@ for my $fn (glob "$tmpDir/*") { exit if $version =~ /pre/; -# Update Nixpkgs in a very hacky way. +# Update nix-fallback-paths.nix. system("cd $nixpkgsDir && git pull") == 0 or die; -my $oldName = `nix-instantiate --eval $nixpkgsDir -A nix.name`; chomp $oldName; -my $oldHash = `nix-instantiate --eval $nixpkgsDir -A nix.src.outputHash`; chomp $oldHash; -print STDERR "old stable version in Nixpkgs = $oldName / $oldHash\n"; - -my $fn = "$nixpkgsDir/pkgs/tools/package-management/nix/default.nix"; -my $oldFile = read_file($fn); -$oldFile =~ s/$oldName/"$releaseName"/g; -$oldFile =~ s/$oldHash/"$tarballHash"/g; -write_file($fn, $oldFile); - -$oldName =~ s/nix-//g; -$oldName =~ s/"//g; sub getStorePath { my ($jobName) = @_; @@ -167,7 +155,7 @@ write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix", " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . "}\n"); -system("cd $nixpkgsDir && git commit -a -m 'nix: $oldName -> $version'") == 0 or die; +system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die; # Update the "latest" symlink. $channelsBucket->add_key( diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index c334639e2..f1b439840 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -19,7 +19,7 @@ /bin/sh -c - /bin/wait4path /nix/var/nix/profiles/default/bin/nix-daemon && /nix/var/nix/profiles/default/bin/nix-daemon + /bin/wait4path /nix/var/nix/profiles/default/bin/nix-daemon && exec /nix/var/nix/profiles/default/bin/nix-daemon StandardErrorPath /var/log/nix-daemon.log diff --git a/misc/zsh/completion.zsh b/misc/zsh/completion.zsh index d4df6447e..a902e37dc 100644 --- a/misc/zsh/completion.zsh +++ b/misc/zsh/completion.zsh @@ -1,3 +1,5 @@ +#compdef nix + function _nix() { local ifs_bk="$IFS" local input=("${(Q)words[@]}") @@ -18,4 +20,4 @@ function _nix() { _describe 'nix' suggestions } -compdef _nix nix +_nix "$@" diff --git a/misc/zsh/local.mk b/misc/zsh/local.mk new file mode 100644 index 000000000..418fb1377 --- /dev/null +++ b/misc/zsh/local.mk @@ -0,0 +1 @@ +$(eval $(call install-file-as, $(d)/completion.zsh, $(datarootdir)/zsh/site-functions/_nix, 0644)) diff --git a/nix-rust/local.mk b/nix-rust/local.mk index 50db4783c..9650cdf93 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -8,8 +8,13 @@ endif libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.$(SO_EXT) libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT) -libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl -libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ldl +libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust +libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust + +ifeq ($(OS), Linux) +libnixrust_LDFLAGS_USE += -ldl +libnixrust_LDFLAGS_USE_INSTALLED += -ldl +endif ifeq ($(OS), Darwin) libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup" diff --git a/scripts/bigsur-nixbld-user-migration.sh b/scripts/bigsur-nixbld-user-migration.sh new file mode 100755 index 000000000..f1619fd56 --- /dev/null +++ b/scripts/bigsur-nixbld-user-migration.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +((NEW_NIX_FIRST_BUILD_UID=301)) + +id_available(){ + dscl . list /Users UniqueID | grep -E '\b'$1'\b' >/dev/null +} + +change_nixbld_names_and_ids(){ + local name uid next_id + ((next_id=NEW_NIX_FIRST_BUILD_UID)) + echo "Attempting to migrate nixbld users." + echo "Each user should change from nixbld# to _nixbld#" + echo "and their IDs relocated to $next_id+" + while read -r name uid; do + echo " Checking $name (uid: $uid)" + # iterate for a clean ID + while id_available "$next_id"; do + ((next_id++)) + if ((next_id >= 400)); then + echo "We've hit UID 400 without placing all of your users :(" + echo "You should use the commands in this script as a starting" + echo "point to review your UID-space and manually move the" + echo "remaining users (or delete them, if you don't need them)." + exit 1 + fi + done + + if [[ $name == _* ]]; then + echo " It looks like $name has already been renamed--skipping." + else + # first 3 are cleanup, it's OK if they aren't here + sudo dscl . delete /Users/$name dsAttrTypeNative:_writers_passwd &>/dev/null || true + sudo dscl . change /Users/$name NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true + # remove existing user from group + sudo dseditgroup -o edit -t user -d $name nixbld || true + sudo dscl . change /Users/$name UniqueID $uid $next_id + sudo dscl . change /Users/$name RecordName $name _$name + # add renamed user to group + sudo dseditgroup -o edit -t user -a _$name nixbld + echo " $name migrated to _$name (uid: $next_id)" + fi + done < <(dscl . list /Users UniqueID | grep nixbld | sort -n -k2) +} + +change_nixbld_names_and_ids diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index 32fa577a8..8aff03199 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -1,33 +1,262 @@ -#!/bin/sh -set -e +#!/usr/bin/env bash +set -eu +set -o pipefail -root_disk() { - diskutil info -plist / -} +# I'm a little agnostic on the choices, but supporting a wide +# slate of uses for now, including: +# - import-only: `. create-darwin-volume.sh no-main[ ...]` +# - legacy: `./create-darwin-volume.sh` or `. create-darwin-volume.sh` +# (both will run main()) +# - external alt-routine: `./create-darwin-volume.sh no-main func[ ...]` +if [ "${1-}" = "no-main" ]; then + shift + readonly _CREATE_VOLUME_NO_MAIN=1 +else + readonly _CREATE_VOLUME_NO_MAIN=0 + # declare some things we expect to inherit from install-multi-user + # I don't love this (because it's a bit of a kludge). + # + # CAUTION: (Dec 19 2020) + # This is a stopgap. It doesn't cover the full slate of + # identifiers we inherit--just those necessary to: + # - avoid breaking direct invocations of this script (here/now) + # - avoid hard-to-reverse structural changes before the call to rm + # single-user support is verified + # + # In the near-mid term, I (personally) think we should: + # - decide to deprecate the direct call and add a notice + # - fold all of this into install-darwin-multi-user.sh + # - intentionally remove the old direct-invocation form (kill the + # routine, replace this script w/ deprecation notice and a note + # on the remove-after date) + # + readonly NIX_ROOT="${NIX_ROOT:-/nix}" -# i.e., "disk1" + _sudo() { + shift # throw away the 'explanation' + /usr/bin/sudo "$@" + } + failure() { + if [ "$*" = "" ]; then + cat + else + echo "$@" + fi + exit 1 + } + task() { + echo "$@" + } +fi + +# usually "disk1" root_disk_identifier() { - diskutil info -plist / | xmllint --xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" - + # For performance (~10ms vs 280ms) I'm parsing 'diskX' from stat output + # (~diskXsY)--but I'm retaining the more-semantic approach since + # it documents intent better. + # /usr/sbin/diskutil info -plist / | xmllint --xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" - + # + local special_device + special_device="$(/usr/bin/stat -f "%Sd" /)" + echo "${special_device%s[0-9]*}" } -find_nix_volume() { - diskutil apfs list -plist "$1" | xmllint --xpath "(/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='Name']/following-sibling::string[starts-with(translate(text(),'N','n'),'nix')]/text())[1]" - 2>/dev/null || true +# make it easy to play w/ 'Case-sensitive APFS' +readonly NIX_VOLUME_FS="${NIX_VOLUME_FS:-APFS}" +readonly NIX_VOLUME_LABEL="${NIX_VOLUME_LABEL:-Nix Store}" +# Strongly assuming we'll make a volume on the device / is on +# But you can override NIX_VOLUME_USE_DISK to create it on some other device +readonly NIX_VOLUME_USE_DISK="${NIX_VOLUME_USE_DISK:-$(root_disk_identifier)}" +NIX_VOLUME_USE_SPECIAL="${NIX_VOLUME_USE_SPECIAL:-}" +NIX_VOLUME_USE_UUID="${NIX_VOLUME_USE_UUID:-}" +readonly NIX_VOLUME_MOUNTD_DEST="${NIX_VOLUME_MOUNTD_DEST:-/Library/LaunchDaemons/org.nixos.darwin-store.plist}" + +if /usr/bin/fdesetup isactive >/dev/null; then + test_filevault_in_use() { return 0; } + # no readonly; we may modify if user refuses from cure_volume + NIX_VOLUME_DO_ENCRYPT="${NIX_VOLUME_DO_ENCRYPT:-1}" +else + test_filevault_in_use() { return 1; } + NIX_VOLUME_DO_ENCRYPT="${NIX_VOLUME_DO_ENCRYPT:-0}" +fi + +should_encrypt_volume() { + test_filevault_in_use && (( NIX_VOLUME_DO_ENCRYPT == 1 )) +} + +substep() { + printf " %s\n" "" "- $1" "" "${@:2}" +} + + +volumes_labeled() { + local label="$1" + xsltproc --novalid --stringparam label "$label" - <(/usr/sbin/ioreg -ra -c "AppleAPFSVolume") <<'EOF' + + + + + + + + = + + + + +EOF + # I cut label out of the extracted values, but here it is for reference: + # + # = +} + +right_disk() { + local volume_special="$1" # (i.e., disk1s7) + [[ "$volume_special" == "$NIX_VOLUME_USE_DISK"s* ]] +} + +right_volume() { + local volume_special="$1" # (i.e., disk1s7) + # if set, it must match; otherwise ensure it's on the right disk + if [ -z "$NIX_VOLUME_USE_SPECIAL" ]; then + if right_disk "$volume_special"; then + NIX_VOLUME_USE_SPECIAL="$volume_special" # latch on + return 0 + else + return 1 + fi + else + [ "$volume_special" = "$NIX_VOLUME_USE_SPECIAL" ] + fi +} + +right_uuid() { + local volume_uuid="$1" + # if set, it must match; otherwise allow + if [ -z "$NIX_VOLUME_USE_UUID" ]; then + NIX_VOLUME_USE_UUID="$volume_uuid" # latch on + return 0 + else + [ "$volume_uuid" = "$NIX_VOLUME_USE_UUID" ] + fi +} + +cure_volumes() { + local found volume special uuid + # loop just in case they have more than one volume + # (nothing stops you from doing this) + for volume in $(volumes_labeled "$NIX_VOLUME_LABEL"); do + # CAUTION: this could (maybe) be a more normal read + # loop like: + # while IFS== read -r special uuid; do + # # ... + # done <<<"$(volumes_labeled "$NIX_VOLUME_LABEL")" + # + # I did it with for to skirt a problem with the obvious + # pattern replacing stdin and causing user prompts + # inside (which also use read and access stdin) to skip + # + # If there's an existing encrypted volume we can't find + # in keychain, the user never gets prompted to delete + # the volume, and the install fails. + # + # If you change this, a human needs to test a very + # specific scenario: you already have an encrypted + # Nix Store volume, and have deleted its credential + # from keychain. Ensure the script asks you if it can + # delete the volume, and then prompts for your sudo + # password to confirm. + # + # shellcheck disable=SC1097 + IFS== read -r special uuid <<< "$volume" + # take the first one that's on the right disk + if [ -z "${found:-}" ]; then + if right_volume "$special" && right_uuid "$uuid"; then + cure_volume "$special" "$uuid" + found="${special} (${uuid})" + else + warning < + # Cryptographic user for (1 found) + # Cryptographic users for (2 found) + /usr/sbin/diskutil apfs listCryptoUsers -plist "$volume_special" | /usr/bin/grep -q APFSCryptoUserUUID } test_fstab() { - grep -q "/nix apfs rw" /etc/fstab 2>/dev/null + /usr/bin/grep -q "$NIX_ROOT apfs rw" /etc/fstab 2>/dev/null } -test_nix_symlink() { - [ -L "/nix" ] || grep -q "^nix." /etc/synthetic.conf 2>/dev/null +test_nix_root_is_symlink() { + [ -L "$NIX_ROOT" ] } -test_synthetic_conf() { - grep -q "^nix$" /etc/synthetic.conf 2>/dev/null +test_synthetic_conf_either(){ + /usr/bin/grep -qE "^${NIX_ROOT:1}($|\t.{3,}$)" /etc/synthetic.conf 2>/dev/null +} + +test_synthetic_conf_mountable() { + /usr/bin/grep -q "^${NIX_ROOT:1}$" /etc/synthetic.conf 2>/dev/null +} + +test_synthetic_conf_symlinked() { + /usr/bin/grep -qE "^${NIX_ROOT:1}\t.{3,}$" /etc/synthetic.conf 2>/dev/null +} + +test_nix_volume_mountd_installed() { + test -e "$NIX_VOLUME_MOUNTD_DEST" +} + +# current volume password +test_keychain_by_uuid() { + local volume_uuid="$1" + # Note: doesn't need sudo just to check; doesn't output pw + security find-generic-password -s "$volume_uuid" &>/dev/null +} + +get_volume_pass() { + local volume_uuid="$1" + _sudo \ + "to confirm keychain has a password that unlocks this volume" \ + security find-generic-password -s "$volume_uuid" -w +} + +verify_volume_pass() { + local volume_special="$1" # (i.e., disk1s7) + local volume_uuid="$2" + /usr/sbin/diskutil apfs unlockVolume "$volume_special" -verify -stdinpassphrase -user "$volume_uuid" +} + +volume_pass_works() { + local volume_special="$1" # (i.e., disk1s7) + local volume_uuid="$2" + get_volume_pass "$volume_uuid" | verify_volume_pass "$volume_special" "$volume_uuid" } # Create the paths defined in synthetic.conf, saving us a reboot. -create_synthetic_objects(){ +create_synthetic_objects() { # Big Sur takes away the -B flag we were using and replaces it # with a -t flag that appears to do the same thing (but they # don't behave exactly the same way in terms of return values). @@ -41,129 +270,570 @@ create_synthetic_objects(){ } test_nix() { - test -d "/nix" + test -d "$NIX_ROOT" } -test_t2_chip_present(){ - # Use xartutil to see if system has a t2 chip. - # - # This isn't well-documented on its own; until it is, - # let's keep track of knowledge/assumptions. - # - # Warnings: - # - Don't search "xart" if porn will cause you trouble :) - # - Other xartutil flags do dangerous things. Don't run them - # naively. If you must, search "xartutil" first. - # - # Assumptions: - # - the "xART session seeds recovery utility" - # appears to interact with xartstorageremoted - # - `sudo xartutil --list` lists xART sessions - # and their seeds and exits 0 if successful. If - # not, it exits 1 and prints an error such as: - # xartutil: ERROR: No supported link to the SEP present - # - xART sessions/seeds are present when a T2 chip is - # (and not, otherwise) - # - the presence of a T2 chip means a newly-created - # volume on the primary drive will be - # encrypted at rest - # - all together: `sudo xartutil --list` - # should exit 0 if a new Nix Store volume will - # be encrypted at rest, and exit 1 if not. - sudo xartutil --list >/dev/null 2>/dev/null +test_voldaemon() { + test -f "$NIX_VOLUME_MOUNTD_DEST" } -test_filevault_in_use() { - fdesetup isactive >/dev/null +generate_mount_command() { + local cmd_type="$1" # encrypted|unencrypted + local volume_uuid mountpoint cmd=() + printf -v volume_uuid "%q" "$2" + printf -v mountpoint "%q" "$NIX_ROOT" + + case "$cmd_type" in + encrypted) + cmd=(/bin/sh -c "/usr/bin/security find-generic-password -s '$volume_uuid' -w | /usr/sbin/diskutil apfs unlockVolume '$volume_uuid' -mountpoint '$mountpoint' -stdinpassphrase");; + unencrypted) + cmd=(/usr/sbin/diskutil mount -mountPoint "$mountpoint" "$volume_uuid");; + *) + failure "Invalid first arg $cmd_type to generate_mount_command";; + esac + + printf " %s\n" "${cmd[@]}" } -# use after error msg for conditions we don't understand -suggest_report_error(){ - # ex "error: something sad happened :(" >&2 - echo " please report this @ https://github.com/nixos/nix/issues" >&2 +generate_mount_daemon() { + local cmd_type="$1" # encrypted|unencrypted + local volume_uuid="$2" + cat < + + + + RunAtLoad + + Label + org.nixos.darwin-store + ProgramArguments + +$(generate_mount_command "$cmd_type" "$volume_uuid") + + + +EOF } -main() { - ( - echo "" - echo " ------------------------------------------------------------------ " - echo " | This installer will create a volume for the nix store and |" - echo " | configure it to mount at /nix. Follow these steps to uninstall. |" - echo " ------------------------------------------------------------------ " - echo "" - echo " 1. Remove the entry from fstab using 'sudo vifs'" - echo " 2. Destroy the data volume using 'diskutil apfs deleteVolume'" - echo " 3. Remove the 'nix' line from /etc/synthetic.conf or the file" - echo "" - ) >&2 +_eat_bootout_err() { + /usr/bin/grep -v "Boot-out failed: 36: Operation now in progress" +} - if test_nix_symlink; then - echo "error: /nix is a symlink, please remove it and make sure it's not in synthetic.conf (in which case a reboot is required)" >&2 - echo " /nix -> $(readlink "/nix")" >&2 - exit 2 +# TODO: remove with --uninstall? +uninstall_launch_daemon_directions() { + local daemon_label="$1" # i.e., org.nixos.blah-blah + local daemon_plist="$2" # abspath + substep "Uninstall LaunchDaemon $daemon_label" \ + " sudo launchctl bootout system/$daemon_label" \ + " sudo rm $daemon_plist" +} + +uninstall_launch_daemon_prompt() { + local daemon_label="$1" # i.e., org.nixos.blah-blah + local daemon_plist="$2" # abspath + local reason_for_daemon="$3" + cat < >(_eat_bootout_err >&2) || true + # this can "fail" with a message like: + # Boot-out failed: 36: Operation now in progress + _sudo "to remove the daemon definition" rm "$daemon_plist" + fi +} + +nix_volume_mountd_uninstall_directions() { + uninstall_launch_daemon_directions "org.nixos.darwin-store" \ + "$NIX_VOLUME_MOUNTD_DEST" +} + +nix_volume_mountd_uninstall_prompt() { + uninstall_launch_daemon_prompt "org.nixos.darwin-store" \ + "$NIX_VOLUME_MOUNTD_DEST" \ + "mount your Nix volume" +} + +# TODO: move nix_daemon to install-darwin-multi-user if/when uninstall_launch_daemon_prompt moves up to install-multi-user +nix_daemon_uninstall_prompt() { + uninstall_launch_daemon_prompt "org.nixos.nix-daemon" \ + "$NIX_DAEMON_DEST" \ + "run the nix-daemon" +} + +# TODO: remove with --uninstall? +nix_daemon_uninstall_directions() { + uninstall_launch_daemon_directions "org.nixos.nix-daemon" \ + "$NIX_DAEMON_DEST" +} + + +# TODO: remove with --uninstall? +synthetic_conf_uninstall_directions() { + # :1 to strip leading slash + substep "Remove ${NIX_ROOT:1} from /etc/synthetic.conf" \ + " If nix is the only entry: sudo rm /etc/synthetic.conf" \ + " Otherwise: sudo /usr/bin/sed -i '' -e '/^${NIX_ROOT:1}$/d' /etc/synthetic.conf" +} + +synthetic_conf_uninstall_prompt() { + cat < "$SCRATCH/synthetic.conf.edit" + + if test_synthetic_conf_symlinked; then + warning <&2 - echo nix | sudo tee -a /etc/synthetic.conf - if ! test_synthetic_conf; then - echo "error: failed to configure synthetic.conf;" >&2 - suggest_report_error - exit 1 + # ask to rm if this left the file empty aside from comments, else edit + if /usr/bin/diff -q <(:) <(/usr/bin/grep -v "^#" "$SCRATCH/synthetic.conf.edit") &>/dev/null; then + if confirm_rm "/etc/synthetic.conf"; then + if test_nix_root_is_symlink; then + failure >&2 < $(readlink "$NIX_ROOT")). The system should remove it when you reboot. +Once you've rebooted, run the installer again. +EOF + fi + return 0 + fi + else + if confirm_edit "$SCRATCH/synthetic.conf.edit" "/etc/synthetic.conf"; then + if test_nix_root_is_symlink; then + failure >&2 < $(readlink "$NIX_ROOT")). The system should remove it when you reboot. +Once you've rebooted, run the installer again. +EOF + fi + return 0 fi fi + # fallback instructions + echo "Manually remove nix from /etc/synthetic.conf" + return 1 +} - if ! test_nix; then - echo "Creating mountpoint for /nix..." >&2 - create_synthetic_objects # the ones we defined in synthetic.conf - if ! test_nix; then - sudo mkdir -p /nix 2>/dev/null || true +add_nix_vol_fstab_line() { + local uuid="$1" + # shellcheck disable=SC1003,SC2026 + local escaped_mountpoint="${NIX_ROOT/ /'\\\'040}" + shift + EDITOR="/usr/bin/ex" _sudo "to add nix to fstab" "$@" <multi-user reinstalls, which may cover this) + # + # I'm not sure if it's safe to approach this way? + # + # I think I think the most-proper way to test for it is: + # diskutil info -plist "$NIX_VOLUME_LABEL" | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1][name()='true']" -; echo $? + # + # There's also `sudo /usr/sbin/vsdbutil -c /path` (which is much faster, but is also + # deprecated and needs minor parsing). + # + # If no one finds a problem with doing so, I think the simplest approach + # is to just eagerly set this. I found a few imperative approaches: + # (diskutil enableOwnership, ~100ms), a cheap one (/usr/sbin/vsdbutil -a, ~40-50ms), + # a very cheap one (append the internal format to /var/db/volinfo.database). + # + # But vsdbutil's deprecation notice suggests using fstab, so I want to + # give that a whirl first. + # + # TODO: when this is workable, poke infinisil about reproducing the issue + # and confirming this fix? +} + +delete_nix_vol_fstab_line() { + # TODO: I'm scaffolding this to handle the new nix volumes + # but it might be nice to generalize a smidge further to + # go ahead and set up a pattern for curing "old" things + # we no longer do? + EDITOR="/usr/bin/patch" _sudo "to cut nix from fstab" "$@" < <(/usr/bin/diff /etc/fstab <(/usr/bin/grep -v "$NIX_ROOT apfs rw" /etc/fstab)) + # leaving some parts out of the grep; people may fiddle this a little? +} + +# TODO: hope to remove with --uninstall +fstab_uninstall_directions() { + substep "Remove ${NIX_ROOT} from /etc/fstab" \ + " If nix is the only entry: sudo rm /etc/fstab" \ + " Otherwise, run 'sudo /usr/sbin/vifs' to remove the nix line" +} + +fstab_uninstall_prompt() { + cat </dev/null + + # if the patch test edit, minus comment lines, is equal to empty (:) + if /usr/bin/diff -q <(:) <(/usr/bin/grep -v "^#" "$SCRATCH/fstab.edit") &>/dev/null; then + # this edit would leave it empty; propose deleting it + if confirm_rm "/etc/fstab"; then + return 0 + else + echo "Remove nix from /etc/fstab (or remove the file)" fi - if ! test_nix; then - echo "error: failed to bootstrap /nix; if a reboot doesn't help," >&2 - suggest_report_error - exit 1 + else + echo "I might be able to help you make this edit. Here's the diff:" + if ! _diff "/etc/fstab" "$SCRATCH/fstab.edit" && ui_confirm "Does the change above look right?"; then + delete_nix_vol_fstab_line /usr/sbin/vifs + else + echo "Remove nix from /etc/fstab (or remove the file)" fi fi +} - disk="$(root_disk_identifier)" - volume=$(find_nix_volume "$disk") - if [ -z "$volume" ]; then - echo "Creating a Nix Store volume..." >&2 +remove_volume() { + local volume_special="$1" # (i.e., disk1s7) + _sudo "to unmount the Nix volume" \ + /usr/sbin/diskutil unmount force "$volume_special" || true # might not be mounted + _sudo "to delete the Nix volume" \ + /usr/sbin/diskutil apfs deleteVolume "$volume_special" +} - if test_filevault_in_use; then - # TODO: Not sure if it's in-scope now, but `diskutil apfs list` - # shows both filevault and encrypted at rest status, and it - # may be the more semantic way to test for this? It'll show - # `FileVault: No (Encrypted at rest)` - # `FileVault: No` - # `FileVault: Yes (Unlocked)` - # and so on. - if test_t2_chip_present; then - echo "warning: boot volume is FileVault-encrypted, but the Nix store volume" >&2 - echo " is only encrypted at rest." >&2 - echo " See https://nixos.org/nix/manual/#sect-macos-installation" >&2 +# aspiration: robust enough to both fix problems +# *and* update older darwin volumes +cure_volume() { + local volume_special="$1" # (i.e., disk1s7) + local volume_uuid="$2" + header "Found existing Nix volume" + row " special" "$volume_special" + row " uuid" "$volume_uuid" + + if volume_encrypted "$volume_special"; then + row "encrypted" "yes" + if volume_pass_works "$volume_special" "$volume_uuid"; then + NIX_VOLUME_DO_ENCRYPT=0 + ok "Found a working decryption password in keychain :)" + echo "" + else + # - this is a volume we made, and + # - the user encrypted it on their own + # - something deleted the credential + # - this is an old or BYO volume and the pw + # just isn't somewhere we can find it. + # + # We're going to explain why we're freaking out + # and prompt them to either delete the volume + # (requiring a sudo auth), or abort to fix + warning <&2 - echo " FileVault encrypted, but encryption-at-rest is not available." >&2 - echo " Manually create a volume for the store and re-run this script." >&2 - echo " See https://nixos.org/nix/manual/#sect-macos-installation" >&2 - exit 1 + # TODO: this is a good design case for a warn-and + # remind idiom... + failure <&2 - fi - - if ! test_fstab; then - echo "Configuring /etc/fstab..." >&2 - label=$(echo "$volume" | sed 's/ /\\040/g') - # shellcheck disable=SC2209 - printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs + row "encrypted" "no" fi } -main "$@" +remove_volume_artifacts() { + if test_synthetic_conf_either; then + # NIX_ROOT is in synthetic.conf + if synthetic_conf_uninstall_prompt; then + # TODO: moot until we tackle uninstall, but when we're + # actually uninstalling, we should issue: + # reminder "macOS will clean up the empty mount-point directory at $NIX_ROOT on reboot." + : + fi + fi + if test_fstab; then + fstab_uninstall_prompt + fi + + if test_nix_volume_mountd_installed; then + nix_volume_mountd_uninstall_prompt + fi +} + +setup_synthetic_conf() { + if test_nix_root_is_symlink; then + if ! test_synthetic_conf_symlinked; then + failure >&2 < $(readlink "$NIX_ROOT")). +Please remove it. If nix is in /etc/synthetic.conf, remove it and reboot. +EOF + fi + fi + if ! test_synthetic_conf_mountable; then + task "Configuring /etc/synthetic.conf to make a mount-point at $NIX_ROOT" >&2 + # technically /etc/synthetic.d/nix is supported in Big Sur+ + # but handling both takes even more code... + _sudo "to add Nix to /etc/synthetic.conf" \ + /usr/bin/ex /etc/synthetic.conf <&2 + fi + create_synthetic_objects + if ! test_nix; then + failure >&2 <&2 + add_nix_vol_fstab_line "$volume_uuid" /usr/sbin/vifs + fi +} + +encrypt_volume() { + local volume_uuid="$1" + local volume_label="$2" + local password + # Note: mount/unmount are late additions to support the right order + # of operations for creating the volume and then baking its uuid into + # other artifacts; not as well-trod wrt to potential errors, race + # conditions, etc. + + /usr/sbin/diskutil mount "$volume_label" + + password="$(/usr/bin/xxd -l 32 -p -c 256 /dev/random)" + _sudo "to add your Nix volume's password to Keychain" \ + /usr/bin/security -i </dev/null; do + : + done +} + +setup_volume() { + local use_special use_uuid profile_packages + task "Creating a Nix volume" >&2 + # DOING: I'm tempted to wrap this call in a grep to get the new disk special without doing anything too complex, but this sudo wrapper *is* a little complex, so it'll be a PITA unless maybe we can skip sudo on this. Let's just try it without. + + use_special="${NIX_VOLUME_USE_SPECIAL:-$(create_volume)}" + + use_uuid=${NIX_VOLUME_USE_UUID:-$(volume_uuid_from_special "$use_special")} + + setup_fstab "$use_uuid" + + if should_encrypt_volume; then + encrypt_volume "$use_uuid" "$NIX_VOLUME_LABEL" + setup_volume_daemon "encrypted" "$use_uuid" + # TODO: might be able to save ~60ms by caching or setting + # this somewhere rather than re-checking here. + elif volume_encrypted "$use_special"; then + setup_volume_daemon "encrypted" "$use_uuid" + else + setup_volume_daemon "unencrypted" "$use_uuid" + fi + + await_volume + + # TODO: below is a vague kludge for now; I just don't know + # what if any safe action there is to take here. Also, the + # reminder isn't very helpful. + # I'm less sure where this belongs, but it also wants mounted, pre-install + if type -p nix-env; then + profile_packages="$(nix-env --query --installed)" + # TODO: can probably do below faster w/ read + # intentionally unquoted string to eat whitespace in wc output + # shellcheck disable=SC2046,SC2059 + if ! [ $(printf "$profile_packages" | /usr/bin/wc -l) = "0" ]; then + reminder <&2 + _sudo "to install the Nix volume mounter" /usr/bin/ex "$NIX_VOLUME_MOUNTD_DEST" <&2 + + setup_darwin_volume + } + + main "$@" +fi diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index a27be2a43..f8d6c5e8f 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -3,57 +3,99 @@ set -eu set -o pipefail -readonly PLIST_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist +readonly NIX_DAEMON_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist +# create by default; set 0 to DIY, use a symlink, etc. +readonly NIX_VOLUME_CREATE=${NIX_VOLUME_CREATE:-1} # now default +NIX_FIRST_BUILD_UID="301" +NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d" + +# caution: may update times on / if not run as normal non-root user +read_only_root() { + # this touch command ~should~ always produce an error + # as of this change I confirmed /usr/bin/touch emits: + # "touch: /: Read-only file system" Catalina+ and Big Sur + # "touch: /: Permission denied" Mojave + # (not matching prefix for compat w/ coreutils touch in case using + # an explicit path causes problems; its prefix differs) + [[ "$(/usr/bin/touch / 2>&1)" = *"Read-only file system" ]] + + # Avoiding the slow semantic way to get this information (~330ms vs ~8ms) + # unless using touch causes problems. Just in case, that approach is: + # diskutil info -plist / | , i.e. + # diskutil info -plist / | xmllint --xpath "name(/plist/dict/key[text()='Writable']/following-sibling::*[1])" - +} + +if read_only_root && [ "$NIX_VOLUME_CREATE" = 1 ]; then + should_create_volume() { return 0; } +else + should_create_volume() { return 1; } +fi + +# shellcheck source=./create-darwin-volume.sh +. "$EXTRACTED_NIX_PATH/create-darwin-volume.sh" "no-main" dsclattr() { /usr/bin/dscl . -read "$1" \ - | awk "/$2/ { print \$2 }" + | /usr/bin/awk "/$2/ { print \$2 }" } -poly_validate_assumptions() { - if [ "$(uname -s)" != "Darwin" ]; then - failure "This script is for use with macOS!" +test_nix_daemon_installed() { + test -e "$NIX_DAEMON_DEST" +} + +poly_cure_artifacts() { + if should_create_volume; then + task "Fixing any leftover Nix volume state" + cat < /dev/null 2>&1 + /usr/sbin/dseditgroup -o checkmember -m "$username" "$group" > /dev/null 2>&1 } poly_user_in_group_set() { @@ -149,3 +193,17 @@ poly_create_build_user() { /usr/bin/dscl . create "/Users/$username" \ UniqueID "${uid}" } + +poly_prepare_to_install() { + if should_create_volume; then + header "Preparing a Nix volume" + # intentional indent below to match task indent + cat < 1 )); then + header "Reminders" + for line in "${_reminders[@]}"; do + echo "$line" + if ! headless && [ "${#line}" = 0 ]; then + if read -r -p "Press enter/return to acknowledge."; then + printf $'\033[A\33[2K\r' + fi + fi + done + fi +} + +reminder() { + printf -v label "${BLUE}[ %d ]${ESC}" "$_remind_num" + _reminders+=("$label") + if [[ "$*" = "" ]]; then + while read -r line; do + _reminders+=("$line") + done + else + # this expands each arg to an array entry (and each entry will + # ultimately be a separate line in the output) + _reminders+=("$@") + fi + _reminders+=("") + ((_remind_num++)) +} + __sudo() { local expl="$1" local cmd="$2" @@ -219,18 +314,18 @@ _sudo() { local expl="$1" shift if ! headless; then - __sudo "$expl" "$*" + __sudo "$expl" "$*" >&2 fi sudo "$@" } -readonly SCRATCH=$(mktemp -d -t tmp.XXXXXXXXXX) -function finish_cleanup { +readonly SCRATCH=$(mktemp -d "${TMPDIR:-/tmp/}tmp.XXXXXXXXXX") +finish_cleanup() { rm -rf "$SCRATCH" } -function finish_fail { +finish_fail() { finish_cleanup failure < /dev/null >&2; then warning <&2 -elif [ "$(uname -s)" = "Linux" ]; then +if [ "$(uname -s)" = "Linux" ]; then echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2 fi -INSTALL_MODE=no-daemon -CREATE_DARWIN_VOLUME=0 +case "$(uname -s)" in + "Darwin") + INSTALL_MODE=daemon;; + *) + INSTALL_MODE=no-daemon;; +esac + +# space-separated string +ACTIONS= + # handle the command line flags while [ $# -gt 0 ]; do case $1 in --daemon) - INSTALL_MODE=daemon;; + INSTALL_MODE=daemon + ACTIONS="${ACTIONS}install " + ;; --no-daemon) - INSTALL_MODE=no-daemon;; + if [ "$(uname -s)" = "Darwin" ]; then + printf '\e[1;31mError: --no-daemon installs are no-longer supported on Darwin/macOS!\e[0m\n' >&2 + exit 1 + fi + INSTALL_MODE=no-daemon + # intentional tail space + ACTIONS="${ACTIONS}install " + ;; + # --uninstall) + # # intentional tail space + # ACTIONS="${ACTIONS}uninstall " + # ;; --no-channel-add) export NIX_INSTALLER_NO_CHANNEL_ADD=1;; --daemon-user-count) @@ -69,13 +79,18 @@ while [ $# -gt 0 ]; do --no-modify-profile) NIX_INSTALLER_NO_MODIFY_PROFILE=1;; --darwin-use-unencrypted-nix-store-volume) - CREATE_DARWIN_VOLUME=1;; + { + echo "Warning: the flag --darwin-use-unencrypted-nix-store-volume" + echo " is no longer needed and will be removed in the future." + echo "" + } >&2;; --nix-extra-conf-file) - export NIX_EXTRA_CONF="$(cat $2)" + # shellcheck disable=SC2155 + export NIX_EXTRA_CONF="$(cat "$2")" shift;; *) - ( - echo "Nix Installer [--daemon|--no-daemon] [--daemon-user-count INT] [--no-channel-add] [--no-modify-profile] [--darwin-use-unencrypted-nix-store-volume] [--nix-extra-conf-file FILE]" + { + echo "Nix Installer [--daemon|--no-daemon] [--daemon-user-count INT] [--no-channel-add] [--no-modify-profile] [--nix-extra-conf-file FILE]" echo "Choose installation method." echo "" @@ -101,45 +116,16 @@ while [ $# -gt 0 ]; do if [ -n "${INVOKED_FROM_INSTALL_IN:-}" ]; then echo " --tarball-url-prefix URL: Base URL to download the Nix tarball from." fi - ) >&2 + } >&2 - # darwin and Catalina+ - if [ "$(uname -s)" = "Darwin" ] && { [ "$macos_major" -gt 10 ] || { [ "$macos_major" -eq 10 ] && [ "$macos_minor" -gt 14 ]; }; }; then - ( - echo " --darwin-use-unencrypted-nix-store-volume: Create an APFS volume for the Nix" - echo " store and mount it at /nix. This is the recommended way to create" - echo " /nix with a read-only / on macOS >=10.15." - echo " See: https://nixos.org/nix/manual/#sect-macos-installation" - echo "" - ) >&2 - fi exit;; esac shift done -if [ "$(uname -s)" = "Darwin" ]; then - if [ "$CREATE_DARWIN_VOLUME" = 1 ]; then - printf '\e[1;31mCreating volume and mountpoint /nix.\e[0m\n' - "$self/create-darwin-volume.sh" - fi - - writable="$(diskutil info -plist / | xmllint --xpath "name(/plist/dict/key[text()='Writable']/following-sibling::*[1])" -)" - if ! [ -e $dest ] && [ "$writable" = "false" ]; then - ( - echo "" - echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume." - echo "Use sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume or run the preparation steps manually." - echo "See https://nixos.org/nix/manual/#sect-macos-installation" - echo "" - ) >&2 - exit 1 - fi -fi - if [ "$INSTALL_MODE" = "daemon" ]; then printf '\e[1;31mSwitching to the Multi-user Installer\e[0m\n' - exec "$self/install-multi-user" + exec "$self/install-multi-user" $ACTIONS # let ACTIONS split exit 0 fi @@ -194,6 +180,7 @@ if ! "$nix/bin/nix-store" --load-db < "$self/.reginfo"; then exit 1 fi +# shellcheck source=./nix-profile.sh.in . "$nix/etc/profile.d/nix.sh" if ! "$nix/bin/nix-env" -i "$nix"; then diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index fda5ef600..81c61b2a0 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -41,10 +41,8 @@ handle_network_proxy() { fi } -poly_validate_assumptions() { - if [ "$(uname -s)" != "Linux" ]; then - failure "This script is for use with Linux!" - fi +poly_cure_artifacts() { + : } poly_service_installed_check() { @@ -72,7 +70,7 @@ poly_service_setup_note() { EOF } -poly_extra_try_me_commands(){ +poly_extra_try_me_commands() { if [ -e /run/systemd/system ]; then : else @@ -81,19 +79,10 @@ poly_extra_try_me_commands(){ EOF fi } -poly_extra_setup_instructions(){ - if [ -e /run/systemd/system ]; then - : - else - cat <readDerivation(*drvPath); auto outputHashes = staticOutputHashes(*store, drv); - drv.inputSrcs = store->parseStorePathSet(inputs); + + // Hijack the inputs paths of the derivation to include all the paths + // that come from the `inputDrvs` set. + // We don’t do that for the derivations whose `inputDrvs` is empty + // because + // 1. It’s not needed + // 2. Changing the `inputSrcs` set changes the associated output ids, + // which break CA derivations + if (!drv.inputDrvs.empty()) + drv.inputSrcs = store->parseStorePathSet(inputs); auto result = sshStore->buildDerivation(*drvPath, drv); diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index d29954f67..569c4b9e4 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -54,7 +54,7 @@ void StoreCommand::run() run(getStore()); } -RealisedPathsCommand::RealisedPathsCommand(bool recursive) +BuiltPathsCommand::BuiltPathsCommand(bool recursive) : recursive(recursive) { if (recursive) @@ -81,39 +81,45 @@ RealisedPathsCommand::RealisedPathsCommand(bool recursive) }); } -void RealisedPathsCommand::run(ref store) +void BuiltPathsCommand::run(ref store) { - std::vector paths; + BuiltPaths paths; if (all) { if (installables.size()) throw UsageError("'--all' does not expect arguments"); // XXX: Only uses opaque paths, ignores all the realisations for (auto & p : store->queryAllValidPaths()) - paths.push_back(p); + paths.push_back(BuiltPath::Opaque{p}); } else { - auto pathSet = toRealisedPaths(store, realiseMode, operateOn, installables); + paths = toBuiltPaths(store, realiseMode, operateOn, installables); if (recursive) { - auto roots = std::move(pathSet); - pathSet = {}; - RealisedPath::closure(*store, roots, pathSet); + // XXX: This only computes the store path closure, ignoring + // intermediate realisations + StorePathSet pathsRoots, pathsClosure; + for (auto & root: paths) { + auto rootFromThis = root.outPaths(); + pathsRoots.insert(rootFromThis.begin(), rootFromThis.end()); + } + store->computeFSClosure(pathsRoots, pathsClosure); + for (auto & path : pathsClosure) + paths.push_back(BuiltPath::Opaque{path}); } - for (auto & path : pathSet) - paths.push_back(path); } run(store, std::move(paths)); } StorePathsCommand::StorePathsCommand(bool recursive) - : RealisedPathsCommand(recursive) + : BuiltPathsCommand(recursive) { } -void StorePathsCommand::run(ref store, std::vector paths) +void StorePathsCommand::run(ref store, BuiltPaths paths) { StorePaths storePaths; - for (auto & p : paths) - storePaths.push_back(p.path()); + for (auto& builtPath : paths) + for (auto& p : builtPath.outPaths()) + storePaths.push_back(p); run(store, std::move(storePaths)); } @@ -162,7 +168,7 @@ void MixProfile::updateProfile(const StorePath & storePath) profile2, storePath)); } -void MixProfile::updateProfile(const Buildables & buildables) +void MixProfile::updateProfile(const BuiltPaths & buildables) { if (!profile) return; @@ -170,18 +176,15 @@ void MixProfile::updateProfile(const Buildables & buildables) for (auto & buildable : buildables) { std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](BuiltPath::Opaque bo) { result.push_back(bo.path); }, - [&](BuildableFromDrv bfd) { + [&](BuiltPath::Built bfd) { for (auto & output : bfd.outputs) { - /* Output path should be known because we just tried to - build it. */ - assert(output.second); - result.push_back(*output.second); + result.push_back(output.second); } }, - }, buildable); + }, buildable.raw()); } if (result.size() != 1) diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index e66c697eb..35b3a384b 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -143,7 +143,7 @@ private: }; /* A command that operates on zero or more store paths. */ -struct RealisedPathsCommand : public InstallablesCommand +struct BuiltPathsCommand : public InstallablesCommand { private: @@ -156,26 +156,26 @@ protected: public: - RealisedPathsCommand(bool recursive = false); + BuiltPathsCommand(bool recursive = false); using StoreCommand::run; - virtual void run(ref store, std::vector paths) = 0; + virtual void run(ref store, BuiltPaths paths) = 0; void run(ref store) override; bool useDefaultInstallables() override { return !all; } }; -struct StorePathsCommand : public RealisedPathsCommand +struct StorePathsCommand : public BuiltPathsCommand { StorePathsCommand(bool recursive = false); - using RealisedPathsCommand::run; + using BuiltPathsCommand::run; virtual void run(ref store, std::vector storePaths) = 0; - void run(ref store, std::vector paths) override; + void run(ref store, BuiltPaths paths) override; }; /* A command that operates on exactly one store path. */ @@ -216,7 +216,7 @@ static RegisterCommand registerCommand2(std::vector && name) return RegisterCommand(std::move(name), [](){ return make_ref(); }); } -Buildables build(ref store, Realise mode, +BuiltPaths build(ref store, Realise mode, std::vector> installables, BuildMode bMode = bmNormal); std::set toStorePaths(ref store, @@ -231,7 +231,7 @@ std::set toDerivations(ref store, std::vector> installables, bool useDeriver = false); -std::set toRealisedPaths( +BuiltPaths toBuiltPaths( ref store, Realise mode, OperateOn operateOn, @@ -252,7 +252,7 @@ struct MixProfile : virtual StoreCommand /* If 'profile' is set, make it point at the store path produced by 'buildables'. */ - void updateProfile(const Buildables & buildables); + void updateProfile(const BuiltPaths & buildables); }; struct MixDefaultProfile : MixProfile diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 7102f5a1a..fe52912cf 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -20,31 +20,6 @@ namespace nix { -nlohmann::json BuildableOpaque::toJSON(ref store) const { - nlohmann::json res; - res["path"] = store->printStorePath(path); - return res; -} - -nlohmann::json BuildableFromDrv::toJSON(ref store) const { - nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); - for (const auto& [output, path] : outputs) { - res["outputs"][output] = path ? store->printStorePath(*path) : ""; - } - return res; -} - -nlohmann::json buildablesToJSON(const Buildables & buildables, ref store) { - auto res = nlohmann::json::array(); - for (const Buildable & buildable : buildables) { - std::visit([&res, store](const auto & buildable) { - res.push_back(buildable.toJSON(store)); - }, buildable); - } - return res; -} - void completeFlakeInputPath( ref evalState, const FlakeRef & flakeRef, @@ -111,10 +86,11 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "override-input", - .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`).", + .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.", .category = category, .labels = {"input-path", "flake-url"}, .handler = {[&](std::string inputPath, std::string flakeRef) { + lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), parseFlakeRef(flakeRef, absPath("."))); @@ -309,9 +285,9 @@ void completeFlakeRef(ref store, std::string_view prefix) } } -Buildable Installable::toBuildable() +DerivedPath Installable::toDerivedPath() { - auto buildables = toBuildables(); + auto buildables = toDerivedPaths(); if (buildables.size() != 1) throw Error("installable '%s' evaluates to %d derivations, where only one is expected", what(), buildables.size()); return std::move(buildables[0]); @@ -345,22 +321,19 @@ struct InstallableStorePath : Installable std::string what() override { return store->printStorePath(storePath); } - Buildables toBuildables() override + DerivedPaths toDerivedPaths() override { if (storePath.isDerivation()) { - std::map> outputs; auto drv = store->readDerivation(storePath); - for (auto & [name, output] : drv.outputsAndOptPaths(*store)) - outputs.emplace(name, output.second); return { - BuildableFromDrv { + DerivedPath::Built { .drvPath = storePath, - .outputs = std::move(outputs) + .outputs = drv.outputNames(), } }; } else { return { - BuildableOpaque { + DerivedPath::Opaque { .path = storePath, } }; @@ -373,22 +346,22 @@ struct InstallableStorePath : Installable } }; -Buildables InstallableValue::toBuildables() +DerivedPaths InstallableValue::toDerivedPaths() { - Buildables res; + DerivedPaths res; - std::map>> drvsToOutputs; + std::map> drvsToOutputs; // Group by derivation, helps with .all in particular for (auto & drv : toDerivations()) { auto outputName = drv.outputName; if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath)); - drvsToOutputs[drv.drvPath].insert_or_assign(outputName, drv.outPath); + drvsToOutputs[drv.drvPath].insert(outputName); } for (auto & i : drvsToOutputs) - res.push_back(BuildableFromDrv { i.first, i.second }); + res.push_back(DerivedPath::Built { i.first, i.second }); return res; } @@ -527,7 +500,11 @@ std::tuple InstallableF auto root = cache->getRoot(); for (auto & attrPath : getActualAttrPaths()) { - auto attr = root->findAlongAttrPath(parseAttrPath(*state, attrPath)); + auto attr = root->findAlongAttrPath( + parseAttrPath(*state, attrPath), + true + ); + if (!attr) continue; if (!attr->isDerivation()) @@ -695,31 +672,67 @@ std::shared_ptr SourceExprCommand::parseInstallable( return installables.front(); } -Buildables build(ref store, Realise mode, +BuiltPaths getBuiltPaths(ref store, DerivedPaths hopefullyBuiltPaths) +{ + BuiltPaths res; + for (auto& b : hopefullyBuiltPaths) + std::visit( + overloaded{ + [&](DerivedPath::Opaque bo) { + res.push_back(BuiltPath::Opaque{bo.path}); + }, + [&](DerivedPath::Built bfd) { + OutputPathMap outputs; + auto drv = store->readDerivation(bfd.drvPath); + auto outputHashes = staticOutputHashes(*store, drv); + auto drvOutputs = drv.outputsAndOptPaths(*store); + for (auto& output : bfd.outputs) { + if (!outputHashes.count(output)) + throw Error( + "the derivation '%s' doesn't have an output " + "named '%s'", + store->printStorePath(bfd.drvPath), output); + if (settings.isExperimentalFeatureEnabled( + "ca-derivations")) { + auto outputId = + DrvOutput{outputHashes.at(output), output}; + auto realisation = + store->queryRealisation(outputId); + if (!realisation) + throw Error( + "cannot operate on an output of unbuilt " + "content-addresed derivation '%s'", + outputId.to_string()); + outputs.insert_or_assign( + output, realisation->outPath); + } else { + // If ca-derivations isn't enabled, assume that + // the output path is statically known. + assert(drvOutputs.count(output)); + assert(drvOutputs.at(output).second); + outputs.insert_or_assign( + output, *drvOutputs.at(output).second); + } + } + res.push_back(BuiltPath::Built{bfd.drvPath, outputs}); + }, + }, + b.raw()); + + return res; +} + +BuiltPaths build(ref store, Realise mode, std::vector> installables, BuildMode bMode) { if (mode == Realise::Nothing) settings.readOnlyMode = true; - Buildables buildables; - - std::vector pathsToBuild; + std::vector pathsToBuild; for (auto & i : installables) { - for (auto & b : i->toBuildables()) { - std::visit(overloaded { - [&](BuildableOpaque bo) { - pathsToBuild.push_back({bo.path}); - }, - [&](BuildableFromDrv bfd) { - StringSet outputNames; - for (auto & output : bfd.outputs) - outputNames.insert(output.first); - pathsToBuild.push_back({bfd.drvPath, outputNames}); - }, - }, b); - buildables.push_back(std::move(b)); - } + auto b = i->toDerivedPaths(); + pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end()); } if (mode == Realise::Nothing) @@ -727,59 +740,26 @@ Buildables build(ref store, Realise mode, else if (mode == Realise::Outputs) store->buildPaths(pathsToBuild, bMode); - return buildables; + return getBuiltPaths(store, pathsToBuild); } -std::set toRealisedPaths( +BuiltPaths toBuiltPaths( ref store, Realise mode, OperateOn operateOn, std::vector> installables) { - std::set res; if (operateOn == OperateOn::Output) { - for (auto & b : build(store, mode, installables)) - std::visit(overloaded { - [&](BuildableOpaque bo) { - res.insert(bo.path); - }, - [&](BuildableFromDrv bfd) { - auto drv = store->readDerivation(bfd.drvPath); - auto outputHashes = staticOutputHashes(*store, drv); - for (auto & output : bfd.outputs) { - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - if (!outputHashes.count(output.first)) - throw Error( - "the derivation '%s' doesn't have an output named '%s'", - store->printStorePath(bfd.drvPath), - output.first); - auto outputId = DrvOutput{outputHashes.at(output.first), output.first}; - auto realisation = store->queryRealisation(outputId); - if (!realisation) - throw Error("cannot operate on an output of unbuilt content-addresed derivation '%s'", outputId.to_string()); - res.insert(RealisedPath{*realisation}); - } - else { - // If ca-derivations isn't enabled, behave as if - // all the paths are opaque to keep the default - // behavior - assert(output.second); - res.insert(*output.second); - } - } - }, - }, b); + return build(store, mode, installables); } else { if (mode == Realise::Nothing) settings.readOnlyMode = true; - for (auto & i : installables) - for (auto & b : i->toBuildables()) - if (auto bfd = std::get_if(&b)) - res.insert(bfd->drvPath); + BuiltPaths res; + for (auto & drvPath : toDerivations(store, installables, true)) + res.push_back(BuiltPath::Opaque{drvPath}); + return res; } - - return res; } StorePathSet toStorePaths(ref store, @@ -787,8 +767,10 @@ StorePathSet toStorePaths(ref store, std::vector> installables) { StorePathSet outPaths; - for (auto & path : toRealisedPaths(store, mode, operateOn, installables)) - outPaths.insert(path.path()); + for (auto & path : toBuiltPaths(store, mode, operateOn, installables)) { + auto thisOutPaths = path.outPaths(); + outPaths.insert(thisOutPaths.begin(), thisOutPaths.end()); + } return outPaths; } @@ -810,9 +792,9 @@ StorePathSet toDerivations(ref store, StorePathSet drvPaths; for (auto & i : installables) - for (auto & b : i->toBuildables()) + for (auto & b : i->toDerivedPaths()) std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPath::Opaque bo) { if (!useDeriver) throw Error("argument '%s' did not evaluate to a derivation", i->what()); auto derivers = store->queryValidDerivers(bo.path); @@ -821,10 +803,10 @@ StorePathSet toDerivations(ref store, // FIXME: use all derivers? drvPaths.insert(*derivers.begin()); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPath::Built bfd) { drvPaths.insert(bfd.drvPath); }, - }, b); + }, b.raw()); return drvPaths; } diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index b714f097b..298fd48f8 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -2,13 +2,13 @@ #include "util.hh" #include "path.hh" +#include "path-with-outputs.hh" +#include "derived-path.hh" #include "eval.hh" #include "flake/flake.hh" #include -#include - namespace nix { struct DrvInfo; @@ -16,25 +16,6 @@ struct SourceExprCommand; namespace eval_cache { class EvalCache; class AttrCursor; } -struct BuildableOpaque { - StorePath path; - nlohmann::json toJSON(ref store) const; -}; - -struct BuildableFromDrv { - StorePath drvPath; - std::map> outputs; - nlohmann::json toJSON(ref store) const; -}; - -typedef std::variant< - BuildableOpaque, - BuildableFromDrv -> Buildable; - -typedef std::vector Buildables; -nlohmann::json buildablesToJSON(const Buildables & buildables, ref store); - struct App { std::vector context; @@ -42,17 +23,23 @@ struct App // FIXME: add args, sandbox settings, metadata, ... }; +struct UnresolvedApp +{ + App unresolved; + App resolve(ref); +}; + struct Installable { virtual ~Installable() { } virtual std::string what() = 0; - virtual Buildables toBuildables() = 0; + virtual DerivedPaths toDerivedPaths() = 0; - Buildable toBuildable(); + DerivedPath toDerivedPath(); - App toApp(EvalState & state); + UnresolvedApp toApp(EvalState & state); virtual std::pair toValue(EvalState & state) { @@ -93,7 +80,7 @@ struct InstallableValue : Installable virtual std::vector toDerivations() = 0; - Buildables toBuildables() override; + DerivedPaths toDerivedPaths() override; }; struct InstallableFlake : InstallableValue diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index 6d68e5df3..1da8d91df 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -35,6 +35,7 @@ class Bindings { public: typedef uint32_t size_t; + Pos *pos; private: size_t size_, capacity_; diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 98d91c905..d7e21783d 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -486,11 +486,11 @@ std::shared_ptr AttrCursor::getAttr(std::string_view name) return getAttr(root->state.symbols.create(name)); } -std::shared_ptr AttrCursor::findAlongAttrPath(const std::vector & attrPath) +std::shared_ptr AttrCursor::findAlongAttrPath(const std::vector & attrPath, bool force) { auto res = shared_from_this(); for (auto & attr : attrPath) { - res = res->maybeGetAttr(attr); + res = res->maybeGetAttr(attr, force); if (!res) return {}; } return res; diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index e23e45c94..43b34ebcb 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -102,7 +102,7 @@ public: std::shared_ptr getAttr(std::string_view name); - std::shared_ptr findAlongAttrPath(const std::vector & attrPath); + std::shared_ptr findAlongAttrPath(const std::vector & attrPath, bool force = false); std::string getString(); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 3afe2e47b..ef9f8efca 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -201,6 +201,15 @@ string showType(const Value & v) } } +Pos Value::determinePos(const Pos &pos) const +{ + switch (internalType) { + case tAttrs: return *attrs->pos; + case tLambda: return lambda.fun->pos; + case tApp: return app.left->determinePos(pos); + default: return pos; + } +} bool Value::isTrivial() const { @@ -1060,6 +1069,8 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), &i.pos)); v.attrs->sort(); // FIXME: inefficient } + + v.attrs->pos = &pos; } @@ -2091,9 +2102,12 @@ Strings EvalSettings::getDefaultNixPath() } }; - add(getHome() + "/.nix-defexpr/channels"); - add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs"); - add(settings.nixStateDir + "/profiles/per-user/root/channels"); + if (!evalSettings.restrictEval && !evalSettings.pureEval) { + add(getHome() + "/.nix-defexpr/channels"); + add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs"); + add(settings.nixStateDir + "/profiles/per-user/root/channels"); + } + return res; } diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index 63566131e..c8a5a319f 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -22,7 +22,9 @@ static TrustedList readTrustedList() static void writeTrustedList(const TrustedList & trustedList) { - writeFile(trustedListPath(), nlohmann::json(trustedList).dump()); + auto path = trustedListPath(); + createDirs(dirOf(path)); + writeFile(path, nlohmann::json(trustedList).dump()); } void ConfigFile::apply() diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 65ed1ad0a..d17d5e183 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -113,7 +113,7 @@ struct LockFlags /* Whether to commit changes to flake.lock. */ bool commitLockFile = false; - /* Flake inputs to be overriden. */ + /* Flake inputs to be overridden. */ std::map inputOverrides; /* Flake inputs to be updated. This means that any existing lock diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 1a3990ea1..f774e6493 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "eval-inline.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include #include @@ -19,7 +20,7 @@ DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs) DrvInfo::DrvInfo(EvalState & state, ref store, const std::string & drvPathWithOutputs) : state(&state), attrs(nullptr), attrPath("") { - auto [drvPath, selectedOutputs] = store->parsePathWithOutputs(drvPathWithOutputs); + auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs); this->drvPath = store->printStorePath(drvPath); diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 26c53d301..c40abfb78 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib libexpr_LIBS = libutil libstore libfetchers libexpr_LDFLAGS = -lboost_context -ifneq ($(OS), FreeBSD) +ifeq ($(OS), Linux) libexpr_LDFLAGS += -ldl endif diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 8df8055b3..51a14cd59 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -180,6 +180,7 @@ struct ExprOpHasAttr : Expr struct ExprAttrs : Expr { bool recursive; + Pos pos; struct AttrDef { bool inherited; Expr * e; @@ -199,7 +200,8 @@ struct ExprAttrs : Expr }; typedef std::vector DynamicAttrDefs; DynamicAttrDefs dynamicAttrs; - ExprAttrs() : recursive(false) { }; + ExprAttrs(const Pos &pos) : recursive(false), pos(pos) { }; + ExprAttrs() : recursive(false), pos(noPos) { }; COMMON_METHODS }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 49d995bb9..f948dde47 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -478,7 +478,7 @@ binds $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data)); } } - | { $$ = new ExprAttrs; } + | { $$ = new ExprAttrs(makeCurPos(@0, data)); } ; attrs diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 1d1afa768..e8569b654 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -21,6 +21,8 @@ #include #include +#include + namespace nix { @@ -35,7 +37,7 @@ InvalidPathError::InvalidPathError(const Path & path) : void EvalState::realiseContext(const PathSet & context) { - std::vector drvs; + std::vector drvs; for (auto & i : context) { auto [ctxS, outputName] = decodeContext(i); @@ -43,7 +45,7 @@ void EvalState::realiseContext(const PathSet & context) if (!store->isValidPath(ctx)) throw InvalidPathError(store->printStorePath(ctx)); if (!outputName.empty() && ctx.isDerivation()) { - drvs.push_back(StorePathWithOutputs{ctx, {outputName}}); + drvs.push_back({ctx, {outputName}}); } } @@ -51,14 +53,16 @@ void EvalState::realiseContext(const PathSet & context) if (!evalSettings.enableImportFromDerivation) throw EvalError("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false", - store->printStorePath(drvs.begin()->path)); + store->printStorePath(drvs.begin()->drvPath)); /* For performance, prefetch all substitute info. */ StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; - store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); + std::vector buildReqs; + for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d }); + store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize); - store->buildPaths(drvs); + store->buildPaths(buildReqs); /* Add the output of this derivations to the allowed paths. */ @@ -545,18 +549,56 @@ typedef list ValueList; #endif +static Bindings::iterator getAttr( + EvalState & state, + string funcName, + string attrName, + Bindings * attrSet, + const Pos & pos) +{ + Bindings::iterator value = attrSet->find(state.symbols.create(attrName)); + if (value == attrSet->end()) { + hintformat errorMsg = hintfmt( + "attribute '%s' missing for call to '%s'", + attrName, + funcName + ); + + Pos aPos = *attrSet->pos; + if (aPos == noPos) { + throw TypeError({ + .msg = errorMsg, + .errPos = pos, + }); + } else { + auto e = TypeError({ + .msg = errorMsg, + .errPos = aPos, + }); + + // Adding another trace for the function name to make it clear + // which call received wrong arguments. + e.addTrace(pos, hintfmt("while invoking '%s'", funcName)); + throw e; + } + } + + return value; +} + static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceAttrs(*args[0], pos); /* Get the start set. */ - Bindings::iterator startSet = - args[0]->attrs->find(state.symbols.create("startSet")); - if (startSet == args[0]->attrs->end()) - throw EvalError({ - .msg = hintfmt("attribute 'startSet' required"), - .errPos = pos - }); + Bindings::iterator startSet = getAttr( + state, + "genericClosure", + "startSet", + args[0]->attrs, + pos + ); + state.forceList(*startSet->value, pos); ValueList workSet; @@ -564,13 +606,14 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar workSet.push_back(startSet->value->listElems()[n]); /* Get the operator. */ - Bindings::iterator op = - args[0]->attrs->find(state.symbols.create("operator")); - if (op == args[0]->attrs->end()) - throw EvalError({ - .msg = hintfmt("attribute 'operator' required"), - .errPos = pos - }); + Bindings::iterator op = getAttr( + state, + "genericClosure", + "operator", + args[0]->attrs, + pos + ); + state.forceValue(*op->value, pos); /* Construct the closure by applying the operator to element of @@ -673,6 +716,44 @@ static RegisterPrimOp primop_addErrorContext(RegisterPrimOp::Info { .fun = prim_addErrorContext, }); +static void prim_ceil(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto value = state.forceFloat(*args[0], args[0]->determinePos(pos)); + mkInt(v, ceil(value)); +} + +static RegisterPrimOp primop_ceil({ + .name = "__ceil", + .args = {"double"}, + .doc = R"( + Converts an IEEE-754 double-precision floating-point number (*double*) to + the next higher integer. + + If the datatype is neither an integer nor a "float", an evaluation error will be + thrown. + )", + .fun = prim_ceil, +}); + +static void prim_floor(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto value = state.forceFloat(*args[0], args[0]->determinePos(pos)); + mkInt(v, floor(value)); +} + +static RegisterPrimOp primop_floor({ + .name = "__floor", + .args = {"double"}, + .doc = R"( + Converts an IEEE-754 double-precision floating-point number (*double*) to + the next lower integer. + + If the datatype is neither an integer nor a "float", an evaluation error will be + thrown. + )", + .fun = prim_floor, +}); + /* Try evaluating the argument. Success => {success=true; value=something;}, * else => {success=false; value=false;} */ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -814,12 +895,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * state.forceAttrs(*args[0], pos); /* Figure out the name first (for stack backtraces). */ - Bindings::iterator attr = args[0]->attrs->find(state.sName); - if (attr == args[0]->attrs->end()) - throw EvalError({ - .msg = hintfmt("required attribute 'name' missing"), - .errPos = pos - }); + Bindings::iterator attr = getAttr( + state, + "derivationStrict", + state.sName, + args[0]->attrs, + pos + ); + string drvName; Pos & posDrvName(*attr->pos); try { @@ -951,7 +1034,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * } } else { - auto s = state.coerceToString(posDrvName, *i->value, context, true); + auto s = state.coerceToString(*i->pos, *i->value, context, true); drv.env.emplace(key, s); if (i->name == state.sBuilder) drv.builder = s; else if (i->name == state.sSystem) drv.platform = s; @@ -1208,7 +1291,10 @@ static RegisterPrimOp primop_toPath({ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, Value & v) { if (evalSettings.pureEval) - throw EvalError("builtins.storePath' is not allowed in pure evaluation mode"); + throw EvalError({ + .msg = hintfmt("'%s' is not allowed in pure evaluation mode", "builtins.storePath"), + .errPos = pos + }); PathSet context; Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context)); @@ -1367,12 +1453,13 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va if (i != v2.attrs->end()) prefix = state.forceStringNoCtx(*i->value, pos); - i = v2.attrs->find(state.symbols.create("path")); - if (i == v2.attrs->end()) - throw EvalError({ - .msg = hintfmt("attribute 'path' missing"), - .errPos = pos - }); + i = getAttr( + state, + "findFile", + "path", + v2.attrs, + pos + ); PathSet context; string path = state.coerceToString(pos, *i->value, context, false, false); @@ -1918,26 +2005,26 @@ static RegisterPrimOp primop_path({ An enrichment of the built-in path type, based on the attributes present in *args*. All are optional except `path`: - - path + - path\ The underlying path. - - name + - name\ The name of the path when added to the store. This can used to reference paths that have nix-illegal characters in their names, like `@`. - - filter + - filter\ A function of the type expected by `builtins.filterSource`, with the same semantics. - - recursive + - recursive\ When `false`, when `path` is added to the store it is with a flat hash, rather than a hash of the NAR serialization of the file. Thus, `path` must refer to a regular file, not a directory. This allows similar behavior to `fetchurl`. Defaults to `true`. - - sha256 + - sha256\ When provided, this is the expected hash of the file at the path. Evaluation will fail if the hash is incorrect, and providing a hash allows `builtins.path` to be used even when the @@ -2014,12 +2101,13 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) string attr = state.forceStringNoCtx(*args[0], pos); state.forceAttrs(*args[1], pos); // !!! Should we create a symbol here or just do a lookup? - Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); - if (i == args[1]->attrs->end()) - throw EvalError({ - .msg = hintfmt("attribute '%1%' missing", attr), - .errPos = pos - }); + Bindings::iterator i = getAttr( + state, + "getAttr", + attr, + args[1]->attrs, + pos + ); // !!! add to stack trace? if (state.countCalls && i->pos) state.attrSelects[*i->pos]++; state.forceValue(*i->value, pos); @@ -2146,22 +2234,25 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v2(*args[0]->listElems()[i]); state.forceAttrs(v2, pos); - Bindings::iterator j = v2.attrs->find(state.sName); - if (j == v2.attrs->end()) - throw TypeError({ - .msg = hintfmt("'name' attribute missing in a call to 'listToAttrs'"), - .errPos = pos - }); - string name = state.forceStringNoCtx(*j->value, pos); + Bindings::iterator j = getAttr( + state, + "listToAttrs", + state.sName, + v2.attrs, + pos + ); + + string name = state.forceStringNoCtx(*j->value, *j->pos); Symbol sym = state.symbols.create(name); if (seen.insert(sym).second) { - Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue)); - if (j2 == v2.attrs->end()) - throw TypeError({ - .msg = hintfmt("'value' attribute missing in a call to 'listToAttrs'"), - .errPos = pos - }); + Bindings::iterator j2 = getAttr( + state, + "listToAttrs", + state.sValue, + v2.attrs, + pos + ); v.attrs->push_back(Attr(sym, j2->value, j2->pos)); } } @@ -2802,7 +2893,12 @@ static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, V for (unsigned int n = 0; n < nrLists; ++n) { Value * vElem = args[1]->listElems()[n]; state.callFunction(*args[0], *vElem, lists[n], pos); - state.forceList(lists[n], pos); + try { + state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos))); + } catch (TypeError &e) { + e.addTrace(pos, hintfmt("while invoking '%s'", "concatMap")); + throw e; + } len += lists[n].listSize(); } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 27d8ddf35..b8b99d4fa 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -303,17 +303,17 @@ static RegisterPrimOp primop_fetchGit({ of the repo at that URL is fetched. Otherwise, it can be an attribute with the following attributes (all except `url` optional): - - url + - url\ The URL of the repo. - - name + - name\ The name of the directory the repo should be exported to in the store. Defaults to the basename of the URL. - - rev + - rev\ The git revision to fetch. Defaults to the tip of `ref`. - - ref + - ref\ The git ref to look for the requested revision under. This is often a branch or tag name. Defaults to `HEAD`. @@ -321,11 +321,11 @@ static RegisterPrimOp primop_fetchGit({ of Nix 2.3.0 Nix will not prefix `refs/heads/` if `ref` starts with `refs/`. - - submodules + - submodules\ A Boolean parameter that specifies whether submodules should be checked out. Defaults to `false`. - - allRefs + - allRefs\ Whether to fetch all refs of the repository. With this argument being true, it's possible to load a `rev` from *any* `ref` (by default only `rev`s from the specified `ref` are supported). diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index b317c1898..a1f131f9e 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -341,6 +341,8 @@ public: return internalType == tList1 ? 1 : internalType == tList2 ? 2 : bigList.size; } + Pos determinePos(const Pos &pos) const; + /* Check whether forcing this value requires a trivial amount of computation. In particular, function applications are non-trivial. */ diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index a2d53a7bf..e41037633 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -6,6 +6,8 @@ #include +#include + namespace nix::fetchers { typedef std::variant> Attr; diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index c6b219c02..a72cfafa4 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -145,13 +145,7 @@ DownloadFileResult downloadFile( bool immutable, const Headers & headers = {}); -struct DownloadTarballMeta -{ - time_t lastModified; - std::string effectiveUrl; -}; - -std::pair downloadTarball( +std::pair downloadTarball( ref store, const std::string & url, const std::string & name, diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 81c647f89..d8e0dbe0a 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -6,6 +6,7 @@ #include "url-parts.hh" #include +#include using namespace std::string_literals; @@ -153,12 +154,14 @@ struct GitInputScheme : InputScheme std::pair getActualUrl(const Input & input) const { - // Don't clone file:// URIs (but otherwise treat them the - // same as remote URIs, i.e. don't use the working tree or - // HEAD). + // file:// URIs are normally not cloned (but otherwise treated the + // same as remote URIs, i.e. we don't use the working tree or + // HEAD). Exception: If _NIX_FORCE_HTTP is set, or the repo is a bare git + // repo, treat as a remote URI to force a clone. static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing auto url = parseURL(getStrAttr(input.attrs, "url")); - bool isLocal = url.scheme == "file" && !forceHttp; + bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git"); + bool isLocal = url.scheme == "file" && !forceHttp && !isBareRepository; return {isLocal, isLocal ? url.path : url.base}; } @@ -363,7 +366,9 @@ struct GitInputScheme : InputScheme ? "refs/*" : ref->compare(0, 5, "refs/") == 0 ? *ref - : "refs/heads/" + *ref; + : ref == "HEAD" + ? *ref + : "refs/heads/" + *ref; runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }); } catch (Error & e) { if (!pathExists(localRefFile)) throw; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 3e5ad75a8..8352ef02d 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -207,16 +207,16 @@ struct GitArchiveInputScheme : InputScheme auto url = getDownloadUrl(input); - auto [tree, meta] = downloadTarball(store, url.url, "source", true, url.headers); + auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers); - input.attrs.insert_or_assign("lastModified", uint64_t(meta.lastModified)); + input.attrs.insert_or_assign("lastModified", uint64_t(lastModified)); getCache()->add( store, immutableAttrs, { {"rev", rev->gitRev()}, - {"lastModified", uint64_t(meta.lastModified)} + {"lastModified", uint64_t(lastModified)} }, tree.storePath, true); diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 81b2227de..74376adc0 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -114,7 +114,7 @@ static std::shared_ptr getSystemRegistry() Path getUserRegistryPath() { - return getHome() + "/.config/nix/registry.json"; + return getConfigDir() + "/nix/registry.json"; } std::shared_ptr getUserRegistry() diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index bd05bb2f1..257465bae 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -109,7 +109,7 @@ DownloadFileResult downloadFile( }; } -std::pair downloadTarball( +std::pair downloadTarball( ref store, const std::string & url, const std::string & name, @@ -127,10 +127,7 @@ std::pair downloadTarball( if (cached && !cached->expired) return { Tree(store->toRealPath(cached->storePath), std::move(cached->storePath)), - { - .lastModified = time_t(getIntAttr(cached->infoAttrs, "lastModified")), - .effectiveUrl = maybeGetStrAttr(cached->infoAttrs, "effectiveUrl").value_or(url), - }, + getIntAttr(cached->infoAttrs, "lastModified") }; auto res = downloadFile(store, url, name, immutable, headers); @@ -155,7 +152,6 @@ std::pair downloadTarball( Attrs infoAttrs({ {"lastModified", uint64_t(lastModified)}, - {"effectiveUrl", res.effectiveUrl}, {"etag", res.etag}, }); @@ -168,10 +164,7 @@ std::pair downloadTarball( return { Tree(store->toRealPath(*unpackedStorePath), std::move(*unpackedStorePath)), - { - .lastModified = lastModified, - .effectiveUrl = res.effectiveUrl, - }, + lastModified, }; } @@ -185,7 +178,8 @@ struct TarballInputScheme : InputScheme && !hasSuffix(url.path, ".tar") && !hasSuffix(url.path, ".tar.gz") && !hasSuffix(url.path, ".tar.xz") - && !hasSuffix(url.path, ".tar.bz2")) + && !hasSuffix(url.path, ".tar.bz2") + && !hasSuffix(url.path, ".tar.zst")) return {}; Input input; @@ -230,11 +224,9 @@ struct TarballInputScheme : InputScheme return true; } - std::pair fetch(ref store, const Input & _input) override + std::pair fetch(ref store, const Input & input) override { - Input input(_input); - auto [tree, meta] = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false); - input.attrs.insert_or_assign("url", meta.effectiveUrl); + auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first; return {std::move(tree), input}; } }; diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 0e5432fca..15354549a 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -122,6 +122,7 @@ public: void log(Verbosity lvl, const FormatOrString & fs) override { + if (lvl > verbosity) return; auto state(state_.lock()); log(*state, lvl, fs.s); } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 5baaff3e9..86930c2e3 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -36,7 +36,7 @@ void printGCWarning() } -void printMissing(ref store, const std::vector & paths, Verbosity lvl) +void printMissing(ref store, const std::vector & paths, Verbosity lvl) { uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; @@ -310,7 +310,7 @@ void printVersion(const string & programName) void showManPage(const string & name) { - restoreSignals(); + restoreProcessContext(); setenv("MANPATH", settings.nixManDir.c_str(), 1); execlp("man", "man", name.c_str(), nullptr); throw SysError("command 'man %1%' failed", name.c_str()); @@ -373,7 +373,7 @@ RunPager::RunPager() throw SysError("dupping stdin"); if (!getenv("LESS")) setenv("LESS", "FRSXMK", 1); - restoreSignals(); + restoreProcessContext(); if (pager) execl("/bin/sh", "sh", "-c", pager, nullptr); execlp("pager", "pager", nullptr); diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index edc7b5efa..05277d90a 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -4,6 +4,7 @@ #include "args.hh" #include "common-args.hh" #include "path.hh" +#include "derived-path.hh" #include @@ -42,7 +43,7 @@ struct StorePathWithOutputs; void printMissing( ref store, - const std::vector & paths, + const std::vector & paths, Verbosity lvl = lvlInfo); void printMissing(ref store, const StorePathSet & willBuild, diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 4f5f8607d..df401e6f4 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -179,6 +179,9 @@ ref BinaryCacheStore::addToStoreCommon( narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : + compression == "zstd" ? ".zst" : + compression == "lzip" ? ".lzip" : + compression == "lz4" ? ".lz4" : compression == "br" ? ".br" : ""); @@ -447,18 +450,43 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s std::optional BinaryCacheStore::queryRealisation(const DrvOutput & id) { + if (diskCache) { + auto [cacheOutcome, maybeCachedRealisation] = + diskCache->lookupRealisation(getUri(), id); + switch (cacheOutcome) { + case NarInfoDiskCache::oValid: + debug("Returning a cached realisation for %s", id.to_string()); + return *maybeCachedRealisation; + case NarInfoDiskCache::oInvalid: + debug("Returning a cached missing realisation for %s", id.to_string()); + return {}; + case NarInfoDiskCache::oUnknown: + break; + } + } + auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi"; auto rawOutputInfo = getFile(outputInfoFilePath); if (rawOutputInfo) { - return {Realisation::fromJSON( - nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath)}; + auto realisation = Realisation::fromJSON( + nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath); + + if (diskCache) + diskCache->upsertRealisation( + getUri(), realisation); + + return {realisation}; } else { + if (diskCache) + diskCache->upsertAbsentRealisation(getUri(), id); return std::nullopt; } } void BinaryCacheStore::registerDrvOutput(const Realisation& info) { + if (diskCache) + diskCache->upsertRealisation(getUri(), info); auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi"; upsertFile(filePath, info.toJSON().dump(), "application/json"); } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index c2163166c..657be2fcf 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -34,7 +34,7 @@ private: protected: // The prefix under which realisation infos will be stored - const std::string realisationsPrefix = "/realisations"; + const std::string realisationsPrefix = "realisations"; BinaryCacheStore(const Params & params); diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index c29237f5c..8c9ef0101 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -73,7 +74,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", - StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store)); + DerivedPath::Built { drvPath, wantedOutputs }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -94,7 +95,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", - StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store)); + DerivedPath::Built { drvPath, drv.outputNames() }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -170,7 +171,7 @@ void DerivationGoal::getDerivation() return; } - addWaitee(upcast_goal(worker.makeSubstitutionGoal(drvPath))); + addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath))); state = &DerivationGoal::loadDerivation; } @@ -246,17 +247,22 @@ void DerivationGoal::haveDerivation() through substitutes. If that doesn't work, we'll build them. */ if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) - for (auto & [_, status] : initialOutputs) { + for (auto & [outputName, status] : initialOutputs) { if (!status.wanted) continue; - if (!status.known) { - warn("do not know how to query for unknown floating content-addressed derivation output yet"); - /* Nothing to wait for; tail call */ - return DerivationGoal::gaveUpOnSubstitution(); - } - addWaitee(upcast_goal(worker.makeSubstitutionGoal( - status.known->path, - buildMode == bmRepair ? Repair : NoRepair, - getDerivationCA(*drv)))); + if (!status.known) + addWaitee( + upcast_goal( + worker.makeDrvOutputSubstitutionGoal( + DrvOutput{status.outputHash, outputName}, + buildMode == bmRepair ? Repair : NoRepair + ) + ) + ); + else + addWaitee(upcast_goal(worker.makePathSubstitutionGoal( + status.known->path, + buildMode == bmRepair ? Repair : NoRepair, + getDerivationCA(*drv)))); } if (waitees.empty()) /* to prevent hang (no wake-up event) */ @@ -337,7 +343,7 @@ void DerivationGoal::gaveUpOnSubstitution() if (!settings.useSubstitutes) throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - addWaitee(upcast_goal(worker.makeSubstitutionGoal(i))); + addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i))); } if (waitees.empty()) /* to prevent hang (no wake-up event) */ @@ -388,7 +394,7 @@ void DerivationGoal::repairClosure() worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); auto drvPath2 = outputsToDrv.find(i); if (drvPath2 == outputsToDrv.end()) - addWaitee(upcast_goal(worker.makeSubstitutionGoal(i, Repair))); + addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair))); else addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); } @@ -920,6 +926,9 @@ void DerivationGoal::resolvedFinished() { if (realisation) { auto newRealisation = *realisation; newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput}; + newRealisation.signatures.clear(); + newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath); + signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); } else { // If we don't have a realisation, then it must mean that something @@ -1243,9 +1252,12 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap() void DerivationGoal::checkPathValidity() { bool checkHash = buildMode == bmRepair; + auto wantedOutputsLeft = wantedOutputs; for (auto & i : queryPartialDerivationOutputMap()) { InitialOutput & info = initialOutputs.at(i.first); info.wanted = wantOutput(i.first, wantedOutputs); + if (info.wanted) + wantedOutputsLeft.erase(i.first); if (i.second) { auto outputPath = *i.second; info.known = { @@ -1258,15 +1270,33 @@ void DerivationGoal::checkPathValidity() }; } if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - if (auto real = worker.store.queryRealisation( - DrvOutput{initialOutputs.at(i.first).outputHash, i.first})) { + auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first}; + if (auto real = worker.store.queryRealisation(drvOutput)) { info.known = { .path = real->outPath, .status = PathStatus::Valid, }; + } else if (info.known && info.known->status == PathStatus::Valid) { + // We know the output because it' a static output of the + // derivation, and the output path is valid, but we don't have + // its realisation stored (probably because it has been built + // without the `ca-derivations` experimental flag) + worker.store.registerDrvOutput( + Realisation{ + drvOutput, + info.known->path, + } + ); } } } + // If we requested all the outputs via the empty set, we are always fine. + // If we requested specific elements, the loop above removes all the valid + // ones, so any that are left must be invalid. + if (!wantedOutputsLeft.empty()) + throw Error("derivation '%s' does not have wanted outputs %s", + worker.store.printStorePath(drvPath), + concatStringsSep(", ", quoteStrings(wantedOutputsLeft))); } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index c85bcd84f..704b77caf 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -180,6 +180,9 @@ struct DerivationGoal : public Goal /* Open a log file and a pipe to it. */ Path openLogFile(); + /* Sign the newly built realisation if the store allows it */ + virtual void signRealisation(Realisation&) {} + /* Close the log file. */ void closeLogFile(); diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc new file mode 100644 index 000000000..be270d079 --- /dev/null +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -0,0 +1,122 @@ +#include "drv-output-substitution-goal.hh" +#include "worker.hh" +#include "substitution-goal.hh" + +namespace nix { + +DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair, std::optional ca) + : Goal(worker) + , id(id) +{ + state = &DrvOutputSubstitutionGoal::init; + name = fmt("substitution of '%s'", id.to_string()); + trace("created"); +} + + +void DrvOutputSubstitutionGoal::init() +{ + trace("init"); + + /* If the derivation already exists, we’re done */ + if (worker.store.queryRealisation(id)) { + amDone(ecSuccess); + return; + } + + subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); + tryNext(); +} + +void DrvOutputSubstitutionGoal::tryNext() +{ + trace("Trying next substituter"); + + if (subs.size() == 0) { + /* None left. Terminate this goal and let someone else deal + with it. */ + debug("drv output '%s' is required, but there is no substituter that can provide it", id.to_string()); + + /* Hack: don't indicate failure if there were no substituters. + In that case the calling derivation should just do a + build. */ + amDone(substituterFailed ? ecFailed : ecNoSubstituters); + + if (substituterFailed) { + worker.failedSubstitutions++; + worker.updateProgress(); + } + + return; + } + + auto sub = subs.front(); + subs.pop_front(); + + // FIXME: Make async + outputInfo = sub->queryRealisation(id); + if (!outputInfo) { + tryNext(); + return; + } + + for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { + if (depId != id) { + if (auto localOutputInfo = worker.store.queryRealisation(depId); + localOutputInfo && localOutputInfo->outPath != depPath) { + warn( + "substituter '%s' has an incompatible realisation for '%s', ignoring.\n" + "Local: %s\n" + "Remote: %s", + sub->getUri(), + depId.to_string(), + worker.store.printStorePath(localOutputInfo->outPath), + worker.store.printStorePath(depPath) + ); + tryNext(); + return; + } + addWaitee(worker.makeDrvOutputSubstitutionGoal(depId)); + } + } + + addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath)); + + if (waitees.empty()) outPathValid(); + else state = &DrvOutputSubstitutionGoal::outPathValid; +} + +void DrvOutputSubstitutionGoal::outPathValid() +{ + assert(outputInfo); + trace("Output path substituted"); + + if (nrFailed > 0) { + debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); + amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); + return; + } + + worker.store.registerDrvOutput(*outputInfo); + finished(); +} + +void DrvOutputSubstitutionGoal::finished() +{ + trace("finished"); + amDone(ecSuccess); +} + +string DrvOutputSubstitutionGoal::key() +{ + /* "a$" ensures substitution goals happen before derivation + goals. */ + return "a$" + std::string(id.to_string()); +} + +void DrvOutputSubstitutionGoal::work() +{ + (this->*state)(); +} + +} diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh new file mode 100644 index 000000000..63ab53d89 --- /dev/null +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -0,0 +1,50 @@ +#pragma once + +#include "store-api.hh" +#include "goal.hh" +#include "realisation.hh" + +namespace nix { + +class Worker; + +// Substitution of a derivation output. +// This is done in three steps: +// 1. Fetch the output info from a substituter +// 2. Substitute the corresponding output path +// 3. Register the output info +class DrvOutputSubstitutionGoal : public Goal { +private: + // The drv output we're trying to substitue + DrvOutput id; + + // The realisation corresponding to the given output id. + // Will be filled once we can get it. + std::optional outputInfo; + + /* The remaining substituters. */ + std::list> subs; + + /* Whether a substituter failed. */ + bool substituterFailed = false; + +public: + DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + + typedef void (DrvOutputSubstitutionGoal::*GoalState)(); + GoalState state; + + void init(); + void tryNext(); + void outPathValid(); + void finished(); + + void timedOut(Error && ex) override { abort(); }; + + string key() override; + + void work() override; + +}; + +} diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 01a564aba..732d4785d 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -6,16 +6,20 @@ namespace nix { -void Store::buildPaths(const std::vector & drvPaths, BuildMode buildMode) +void Store::buildPaths(const std::vector & reqs, BuildMode buildMode) { Worker worker(*this); Goals goals; - for (auto & path : drvPaths) { - if (path.path.isDerivation()) - goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode)); - else - goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); + for (auto & br : reqs) { + std::visit(overloaded { + [&](DerivedPath::Built bfd) { + goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); + }, + [&](DerivedPath::Opaque bo) { + goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); + }, + }, br.raw()); } worker.run(goals); @@ -31,7 +35,7 @@ void Store::buildPaths(const std::vector & drvPaths, Build } if (i->exitCode != Goal::ecSuccess) { if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->drvPath); - else if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->storePath); + else if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->storePath); } } @@ -90,7 +94,7 @@ void Store::ensurePath(const StorePath & path) if (isValidPath(path)) return; Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path); + GoalPtr goal = worker.makePathSubstitutionGoal(path); Goals goals = {goal}; worker.run(goals); @@ -108,7 +112,7 @@ void Store::ensurePath(const StorePath & path) void LocalStore::repairPath(const StorePath & path) { Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); + GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair); Goals goals = {goal}; worker.run(goals); diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 2dd7a4d37..9de40bdf2 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -78,6 +78,8 @@ void Goal::amDone(ExitCode result, std::optional ex) } waiters.clear(); worker.removeGoal(shared_from_this()); + + cleanup(); } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index fca4f2d00..e6bf628cb 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -100,6 +100,8 @@ struct Goal : public std::enable_shared_from_this virtual string key() = 0; void amDone(ExitCode result, std::optional ex = {}); + + virtual void cleanup() { } }; void addToWeakGoals(WeakGoals & goals, GoalPtr p); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9c2f1dda6..ba0aca29c 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -153,6 +153,7 @@ void LocalDerivationGoal::killChild() void LocalDerivationGoal::tryLocalBuild() { unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { + state = &DerivationGoal::tryToBuild; worker.waitForBuildSlot(shared_from_this()); outputLocks.unlock(); return; @@ -287,17 +288,17 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() So instead, check if the disk is (nearly) full now. If so, we don't mark this build as a permanent failure. */ #if HAVE_STATVFS - { + { auto & localStore = getLocalStore(); uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable struct statvfs st; - if (statvfs(localStore.realStoreDir.c_str(), &st) == 0 && + if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 && (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; if (statvfs(tmpDir.c_str(), &st) == 0 && (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; - } + } #endif deleteTmpDir(false); @@ -416,7 +417,7 @@ void LocalDerivationGoal::startBuilder() } auto & localStore = getLocalStore(); - if (localStore.storeDir != localStore.realStoreDir) { + if (localStore.storeDir != localStore.realStoreDir.get()) { #if __linux__ useChroot = true; #else @@ -581,7 +582,9 @@ void LocalDerivationGoal::startBuilder() throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", worker.store.printStorePath(drvPath), i); - dirsInChroot[i] = i; + /* Allow files in __impureHostDeps to be missing; e.g. + macOS 11+ has no /usr/lib/libSystem*.dylib */ + dirsInChroot[i] = {i, true}; } #if __linux__ @@ -1190,6 +1193,26 @@ void LocalDerivationGoal::writeStructuredAttrs() chownToBuilder(tmpDir + "/.attrs.sh"); } + +static StorePath pathPartOfReq(const DerivedPath & req) +{ + return std::visit(overloaded { + [&](DerivedPath::Opaque bo) { + return bo.path; + }, + [&](DerivedPath::Built bfd) { + return bfd.drvPath; + }, + }, req.raw()); +} + + +bool LocalDerivationGoal::isAllowed(const DerivedPath & req) +{ + return this->isAllowed(pathPartOfReq(req)); +} + + struct RestrictedStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; @@ -1310,33 +1333,52 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo std::optional queryRealisation(const DrvOutput & id) override // XXX: This should probably be allowed if the realisation corresponds to // an allowed derivation - { throw Error("queryRealisation"); } + { + if (!goal.isAllowed(id)) + throw InvalidPath("cannot query an unknown output id '%s' in recursive Nix", id.to_string()); + return next->queryRealisation(id); + } - void buildPaths(const std::vector & paths, BuildMode buildMode) override + void buildPaths(const std::vector & paths, BuildMode buildMode) override { if (buildMode != bmNormal) throw Error("unsupported build mode"); StorePathSet newPaths; + std::set newRealisations; - for (auto & path : paths) { - if (!goal.isAllowed(path.path)) - throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); + for (auto & req : paths) { + if (!goal.isAllowed(req)) + throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next)); } next->buildPaths(paths, buildMode); for (auto & path : paths) { - if (!path.path.isDerivation()) continue; - auto outputs = next->queryDerivationOutputMap(path.path); - for (auto & output : outputs) - if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second); + auto p = std::get_if(&path); + if (!p) continue; + auto & bfd = *p; + auto drv = readDerivation(bfd.drvPath); + auto drvHashes = staticOutputHashes(*this, drv); + auto outputs = next->queryDerivationOutputMap(bfd.drvPath); + for (auto & [outputName, outputPath] : outputs) + if (wantOutput(outputName, bfd.outputs)) { + newPaths.insert(outputPath); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + auto thisRealisation = next->queryRealisation( + DrvOutput{drvHashes.at(outputName), outputName} + ); + assert(thisRealisation); + newRealisations.insert(*thisRealisation); + } + } } StorePathSet closure; next->computeFSClosure(newPaths, closure); for (auto & path : closure) goal.addDependency(path); + for (auto & real : Realisation::closure(*next, newRealisations)) + goal.addedDrvOutputs.insert(real.id); } BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, @@ -1358,7 +1400,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo void addSignatures(const StorePath & storePath, const StringSet & sigs) override { unsupported("addSignatures"); } - void queryMissing(const std::vector & targets, + void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) override { @@ -1366,12 +1408,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo client about what paths will be built/substituted or are already present. Probably not a big deal. */ - std::vector allowed; - for (auto & path : targets) { - if (goal.isAllowed(path.path)) - allowed.emplace_back(path); + std::vector allowed; + for (auto & req : targets) { + if (goal.isAllowed(req)) + allowed.emplace_back(req); else - unknown.insert(path.path); + unknown.insert(pathPartOfReq(req)); } next->queryMissing(allowed, willBuild, willSubstitute, @@ -1703,18 +1745,18 @@ void LocalDerivationGoal::runChild() network, so give them access to /etc/resolv.conf and so on. */ if (derivationIsImpure(derivationType)) { - ss.push_back("/etc/resolv.conf"); - // Only use nss functions to resolve hosts and // services. Don’t use it for anything else that may // be configured for this system. This limits the // potential impurities introduced in fixed-outputs. writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); - ss.push_back("/etc/services"); - ss.push_back("/etc/hosts"); - if (pathExists("/var/run/nscd/socket")) - ss.push_back("/var/run/nscd/socket"); + /* N.B. it is realistic that these paths might not exist. It + happens when testing Nix building fixed-output derivations + within a pure derivation. */ + for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts", "/var/run/nscd/socket" }) + if (pathExists(path)) + ss.push_back(path); } for (auto & i : ss) dirsInChroot.emplace(i, i); @@ -2276,10 +2318,6 @@ void LocalDerivationGoal::registerOutputs() sink.s = make_ref(rewriteStrings(*sink.s, outputRewrites)); StringSource source(*sink.s); restorePath(actualPath, source); - - /* FIXME: set proper permissions in restorePath() so - we don't have to do another traversal. */ - canonicalisePathMetaData(actualPath, -1, inodesSeen); } }; @@ -2333,32 +2371,19 @@ void LocalDerivationGoal::registerOutputs() } auto got = caSink.finish().first; auto refs = rewriteRefs(); - HashModuloSink narSink { htSHA256, oldHashPart }; - dumpPath(actualPath, narSink); - auto narHashAndSize = narSink.finish(); - ValidPathInfo newInfo0 { - worker.store.makeFixedOutputPath( + + auto finalPath = worker.store.makeFixedOutputPath( outputHash.method, got, outputPathName(drv->name, outputName), refs.second, - refs.first), - narHashAndSize.first, - }; - newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - if (scratchPath != newInfo0.path) { + refs.first); + if (scratchPath != finalPath) { // Also rewrite the output path auto source = sinkToSource([&](Sink & nextSink) { StringSink sink; dumpPath(actualPath, sink); - RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); + RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink); rsink2(*sink.s); rsink2.flush(); }); @@ -2368,6 +2393,21 @@ void LocalDerivationGoal::registerOutputs() movePath(tmpPath, actualPath); } + HashResult narHashAndSize = hashPath(htSHA256, actualPath); + ValidPathInfo newInfo0 { + finalPath, + narHashAndSize.first, + }; + + newInfo0.narSize = narHashAndSize.second; + newInfo0.ca = FixedOutputHash { + .method = outputHash.method, + .hash = got, + }; + newInfo0.references = refs.second; + if (refs.first) + newInfo0.references.insert(newInfo0.path); + assert(newInfo0.ca); return newInfo0; }; @@ -2428,6 +2468,10 @@ void LocalDerivationGoal::registerOutputs() }, }, output.output); + /* FIXME: set proper permissions in restorePath() so + we don't have to do another traversal. */ + canonicalisePathMetaData(actualPath, -1, inodesSeen); + /* Calculate where we'll move the output files. In the checking case we will leave leave them where they are, for now, rather than move to their usual "final destination" */ @@ -2460,6 +2504,7 @@ void LocalDerivationGoal::registerOutputs() assert(newInfo.ca); } else { auto destPath = worker.store.toRealPath(finalDestPath); + deletePath(destPath); movePath(actualPath, destPath); actualPath = destPath; } @@ -2615,13 +2660,22 @@ void LocalDerivationGoal::registerOutputs() but it's fine to do in all cases. */ if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - for (auto& [outputName, newInfo] : infos) - worker.store.registerDrvOutput(Realisation{ - .id = DrvOutput{initialOutputs.at(outputName).outputHash, outputName}, - .outPath = newInfo.path}); + for (auto& [outputName, newInfo] : infos) { + auto thisRealisation = Realisation{ + .id = DrvOutput{initialOutputs.at(outputName).outputHash, + outputName}, + .outPath = newInfo.path}; + signRealisation(thisRealisation); + worker.store.registerDrvOutput(thisRealisation); + } } } +void LocalDerivationGoal::signRealisation(Realisation & realisation) +{ + getLocalStore().signRealisation(realisation); +} + void LocalDerivationGoal::checkOutputs(const std::map & outputs) { diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 4bbf27a1b..088a57209 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -108,6 +108,9 @@ struct LocalDerivationGoal : public DerivationGoal /* Paths that were added via recursive Nix calls. */ StorePathSet addedPaths; + /* Realisations that were added via recursive Nix calls. */ + std::set addedDrvOutputs; + /* Recursive Nix calls are only allowed to build or realize paths in the original input closure or added via a recursive Nix call (so e.g. you can't do 'nix-store -r /nix/store/' where @@ -116,6 +119,12 @@ struct LocalDerivationGoal : public DerivationGoal { return inputPaths.count(path) || addedPaths.count(path); } + bool isAllowed(const DrvOutput & id) + { + return addedDrvOutputs.count(id); + } + + bool isAllowed(const DerivedPath & req); friend struct RestrictedStore; @@ -161,6 +170,8 @@ struct LocalDerivationGoal : public DerivationGoal as valid. */ void registerOutputs() override; + void signRealisation(Realisation &) override; + /* Check that an output meets the requirements specified by the 'outputChecks' attribute (or the legacy '{allowed,disallowed}{References,Requisites}' attributes). */ diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index c4b0de78d..e56cfadbe 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -5,40 +5,32 @@ namespace nix { -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) +PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) : Goal(worker) , storePath(storePath) , repair(repair) , ca(ca) { - state = &SubstitutionGoal::init; + state = &PathSubstitutionGoal::init; name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); trace("created"); maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); } -SubstitutionGoal::~SubstitutionGoal() +PathSubstitutionGoal::~PathSubstitutionGoal() { - try { - if (thr.joinable()) { - // FIXME: signal worker thread to quit. - thr.join(); - worker.childTerminated(this); - } - } catch (...) { - ignoreException(); - } + cleanup(); } -void SubstitutionGoal::work() +void PathSubstitutionGoal::work() { (this->*state)(); } -void SubstitutionGoal::init() +void PathSubstitutionGoal::init() { trace("init"); @@ -59,10 +51,12 @@ void SubstitutionGoal::init() } -void SubstitutionGoal::tryNext() +void PathSubstitutionGoal::tryNext() { trace("trying next substituter"); + cleanup(); + if (subs.size() == 0) { /* None left. Terminate this goal and let someone else deal with it. */ @@ -142,7 +136,7 @@ void SubstitutionGoal::tryNext() /* Bail out early if this substituter lacks a valid signature. LocalStore::addToStore() also checks for this, but only after we've downloaded the path. */ - if (!sub->isTrusted && worker.store.pathInfoIsTrusted(*info)) + if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) { warn("substituter '%s' does not have a valid signature for path '%s'", sub->getUri(), worker.store.printStorePath(storePath)); @@ -154,16 +148,16 @@ void SubstitutionGoal::tryNext() paths referenced by this one. */ for (auto & i : info->references) if (i != storePath) /* ignore self-references */ - addWaitee(worker.makeSubstitutionGoal(i)); + addWaitee(worker.makePathSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ referencesValid(); else - state = &SubstitutionGoal::referencesValid; + state = &PathSubstitutionGoal::referencesValid; } -void SubstitutionGoal::referencesValid() +void PathSubstitutionGoal::referencesValid() { trace("all references realised"); @@ -177,12 +171,12 @@ void SubstitutionGoal::referencesValid() if (i != storePath) /* ignore self-references */ assert(worker.store.isValidPath(i)); - state = &SubstitutionGoal::tryToRun; + state = &PathSubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); } -void SubstitutionGoal::tryToRun() +void PathSubstitutionGoal::tryToRun() { trace("trying to run"); @@ -205,7 +199,7 @@ void SubstitutionGoal::tryToRun() thr = std::thread([this]() { try { /* Wake up the worker loop when we're done. */ - Finally updateStats([this]() { outPipe.writeSide = -1; }); + Finally updateStats([this]() { outPipe.writeSide.close(); }); Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); PushActivity pact(act.id); @@ -221,11 +215,11 @@ void SubstitutionGoal::tryToRun() worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); - state = &SubstitutionGoal::finished; + state = &PathSubstitutionGoal::finished; } -void SubstitutionGoal::finished() +void PathSubstitutionGoal::finished() { trace("substitute finished"); @@ -249,7 +243,7 @@ void SubstitutionGoal::finished() } /* Try the next substitute. */ - state = &SubstitutionGoal::tryNext; + state = &PathSubstitutionGoal::tryNext; worker.wakeUp(shared_from_this()); return; } @@ -278,14 +272,31 @@ void SubstitutionGoal::finished() } -void SubstitutionGoal::handleChildOutput(int fd, const string & data) +void PathSubstitutionGoal::handleChildOutput(int fd, const string & data) { } -void SubstitutionGoal::handleEOF(int fd) +void PathSubstitutionGoal::handleEOF(int fd) { if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); } + +void PathSubstitutionGoal::cleanup() +{ + try { + if (thr.joinable()) { + // FIXME: signal worker thread to quit. + thr.join(); + worker.childTerminated(this); + } + + outPipe.close(); + } catch (...) { + ignoreException(); + } +} + + } diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index dee2cecbf..70c806d23 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -8,13 +8,13 @@ namespace nix { class Worker; -struct SubstitutionGoal : public Goal +struct PathSubstitutionGoal : public Goal { /* The store path that should be realised through a substitute. */ StorePath storePath; /* The path the substituter refers to the path as. This will be - * different when the stores have different names. */ + different when the stores have different names. */ std::optional subPath; /* The remaining substituters. */ @@ -47,14 +47,15 @@ struct SubstitutionGoal : public Goal std::unique_ptr> maintainExpectedSubstitutions, maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - typedef void (SubstitutionGoal::*GoalState)(); + typedef void (PathSubstitutionGoal::*GoalState)(); GoalState state; /* Content address for recomputing store path */ std::optional ca; - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - ~SubstitutionGoal(); +public: + PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + ~PathSubstitutionGoal(); void timedOut(Error && ex) override { abort(); }; @@ -78,6 +79,8 @@ struct SubstitutionGoal : public Goal /* Callback used by the worker to write to the log. */ void handleChildOutput(int fd, const string & data) override; void handleEOF(int fd) override; + + void cleanup() override; }; } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b2223c3b6..0f2ade348 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -1,6 +1,7 @@ #include "machines.hh" #include "worker.hh" #include "substitution-goal.hh" +#include "drv-output-substitution-goal.hh" #include "local-derivation-goal.hh" #include "hook-instance.hh" @@ -78,20 +79,32 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -std::shared_ptr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +std::shared_ptr Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) { - std::weak_ptr & goal_weak = substitutionGoals[path]; + std::weak_ptr & goal_weak = substitutionGoals[path]; auto goal = goal_weak.lock(); // FIXME if (!goal) { - goal = std::make_shared(path, *this, repair, ca); + goal = std::make_shared(path, *this, repair, ca); goal_weak = goal; wakeUp(goal); } return goal; } -template -static void removeGoal(std::shared_ptr goal, std::map> & goalMap) +std::shared_ptr Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional ca) +{ + std::weak_ptr & goal_weak = drvOutputSubstitutionGoals[id]; + auto goal = goal_weak.lock(); // FIXME + if (!goal) { + goal = std::make_shared(id, *this, repair, ca); + goal_weak = goal; + wakeUp(goal); + } + return goal; +} + +template +static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { /* !!! inefficient */ for (auto i = goalMap.begin(); @@ -109,10 +122,13 @@ void Worker::removeGoal(GoalPtr goal) { if (auto drvGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(drvGoal, derivationGoals); - else if (auto subGoal = std::dynamic_pointer_cast(goal)) + else if (auto subGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(subGoal, substitutionGoals); + else if (auto subGoal = std::dynamic_pointer_cast(goal)) + nix::removeGoal(subGoal, drvOutputSubstitutionGoals); else assert(false); + if (topGoals.find(goal) != topGoals.end()) { topGoals.erase(goal); /* If a top-level goal failed, then kill all other goals @@ -211,14 +227,14 @@ void Worker::waitForAWhile(GoalPtr goal) void Worker::run(const Goals & _topGoals) { - std::vector topPaths; + std::vector topPaths; for (auto & i : _topGoals) { topGoals.insert(i); if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back({goal->drvPath, goal->wantedOutputs}); - } else if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back({goal->storePath}); + topPaths.push_back(DerivedPath::Built{goal->drvPath, goal->wantedOutputs}); + } else if (auto goal = dynamic_cast(i.get())) { + topPaths.push_back(DerivedPath::Opaque{goal->storePath}); } } @@ -471,7 +487,10 @@ void Worker::markContentsGood(const StorePath & path) } -GoalPtr upcast_goal(std::shared_ptr subGoal) { +GoalPtr upcast_goal(std::shared_ptr subGoal) { + return subGoal; +} +GoalPtr upcast_goal(std::shared_ptr subGoal) { return subGoal; } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 82e711191..918de35f6 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -4,6 +4,7 @@ #include "lock.hh" #include "store-api.hh" #include "goal.hh" +#include "realisation.hh" #include #include @@ -12,18 +13,20 @@ namespace nix { /* Forward definition. */ struct DerivationGoal; -struct SubstitutionGoal; +struct PathSubstitutionGoal; +class DrvOutputSubstitutionGoal; /* Workaround for not being able to declare a something like - class SubstitutionGoal : public Goal; + class PathSubstitutionGoal : public Goal; even when Goal is a complete type. This is still a static cast. The purpose of exporting it is to define it in - a place where `SubstitutionGoal` is concrete, and use it in a place where it + a place where `PathSubstitutionGoal` is concrete, and use it in a place where it is opaque. */ -GoalPtr upcast_goal(std::shared_ptr subGoal); +GoalPtr upcast_goal(std::shared_ptr subGoal); +GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; @@ -72,7 +75,8 @@ private: /* Maps used to prevent multiple instantiations of a goal for the same derivation / path. */ std::map> derivationGoals; - std::map> substitutionGoals; + std::map> substitutionGoals; + std::map> drvOutputSubstitutionGoals; /* Goals waiting for busy paths to be unlocked. */ WeakGoals waitingForAnyGoal; @@ -146,7 +150,8 @@ public: const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); /* substitution goal */ - std::shared_ptr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + std::shared_ptr makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + std::shared_ptr makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index 93c442826..08af0cc1f 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -3,9 +3,19 @@ -- is enabled create table if not exists Realisations ( + id integer primary key autoincrement not null, drvPath text not null, outputName text not null, -- symbolic output id, usually "out" outputPath integer not null, - primary key (drvPath, outputName), + signatures text, -- space-separated list foreign key (outputPath) references ValidPaths(id) on delete cascade ); + +create index if not exists IndexRealisations on Realisations(drvPath, outputName); + +create table if not exists RealisationsRefs ( + referrer integer not null, + realisationReference integer, + foreign key (referrer) references Realisations(id) on delete cascade, + foreign key (realisationReference) references Realisations(id) on delete restrict +); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index ba7959263..e06fb9ce2 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -2,6 +2,7 @@ #include "monitor-fd.hh" #include "worker-protocol.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include "finally.hh" #include "affinity.hh" #include "archive.hh" @@ -259,6 +260,18 @@ static void writeValidPathInfo( } } +static std::vector readDerivedPaths(Store & store, unsigned int clientVersion, Source & from) +{ + std::vector reqs; + if (GET_PROTOCOL_MINOR(clientVersion) >= 30) { + reqs = worker_proto::read(store, from, Phantom> {}); + } else { + for (auto & s : readStrings(from)) + reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath()); + } + return reqs; +} + static void performOp(TunnelLogger * logger, ref store, TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, Source & from, BufferedSink & to, unsigned int op) @@ -493,9 +506,7 @@ static void performOp(TunnelLogger * logger, ref store, } case wopBuildPaths: { - std::vector drvs; - for (auto & s : readStrings(from)) - drvs.push_back(store->parsePathWithOutputs(s)); + auto drvs = readDerivedPaths(*store, clientVersion, from); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode) readInt(from); @@ -575,7 +586,10 @@ static void performOp(TunnelLogger * logger, ref store, auto res = store->buildDerivation(drvPath, drv, buildMode); logger->stopWork(); to << res.status << res.errorMsg; - if (GET_PROTOCOL_MINOR(clientVersion) >= 0xc) { + if (GET_PROTOCOL_MINOR(clientVersion) >= 29) { + to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime; + } + if (GET_PROTOCOL_MINOR(clientVersion) >= 28) { worker_proto::write(*store, to, res.builtOutputs); } break; @@ -856,9 +870,7 @@ static void performOp(TunnelLogger * logger, ref store, } case wopQueryMissing: { - std::vector targets; - for (auto & s : readStrings(from)) - targets.push_back(store->parsePathWithOutputs(s)); + auto targets = readDerivedPaths(*store, clientVersion, from); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; @@ -873,11 +885,15 @@ static void performOp(TunnelLogger * logger, ref store, case wopRegisterDrvOutput: { logger->startWork(); - auto outputId = DrvOutput::parse(readString(from)); - auto outputPath = StorePath(readString(from)); - auto resolvedDrv = StorePath(readString(from)); - store->registerDrvOutput(Realisation{ - .id = outputId, .outPath = outputPath}); + if (GET_PROTOCOL_MINOR(clientVersion) < 31) { + auto outputId = DrvOutput::parse(readString(from)); + auto outputPath = StorePath(readString(from)); + store->registerDrvOutput(Realisation{ + .id = outputId, .outPath = outputPath}); + } else { + auto realisation = worker_proto::read(*store, from, Phantom()); + store->registerDrvOutput(realisation); + } logger->stopWork(); break; } @@ -887,9 +903,15 @@ static void performOp(TunnelLogger * logger, ref store, auto outputId = DrvOutput::parse(readString(from)); auto info = store->queryRealisation(outputId); logger->stopWork(); - std::set outPaths; - if (info) outPaths.insert(info->outPath); - worker_proto::write(*store, to, outPaths); + if (GET_PROTOCOL_MINOR(clientVersion) < 31) { + std::set outPaths; + if (info) outPaths.insert(info->outPath); + worker_proto::write(*store, to, outPaths); + } else { + std::set realisations; + if (info) realisations.insert(*info); + worker_proto::write(*store, to, realisations); + } break; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index fe98182bb..f6defd98f 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -590,14 +590,6 @@ std::map staticOutputHashes(Store& store, const Derivation& d } -std::string StorePathWithOutputs::to_string(const Store & store) const -{ - return outputs.empty() - ? store.printStorePath(path) - : store.printStorePath(path) + "!" + concatStringsSep(",", outputs); -} - - bool wantOutput(const string & output, const std::set & wanted) { return wanted.empty() || wanted.find(output) != wanted.end(); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 061d70f69..2df440536 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -52,7 +52,7 @@ struct DerivationOutput DerivationOutputCAFloating, DerivationOutputDeferred > output; - std::optional hashAlgoOpt(const Store & store) const; + /* Note, when you use this function you should make sure that you're passing the right derivation name. When in doubt, you should use the safer interface provided by BasicDerivation::outputsAndOptPaths */ diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc new file mode 100644 index 000000000..8da81d0ac --- /dev/null +++ b/src/libstore/derived-path.cc @@ -0,0 +1,118 @@ +#include "derived-path.hh" +#include "store-api.hh" + +#include + +namespace nix { + +nlohmann::json DerivedPath::Opaque::toJSON(ref store) const { + nlohmann::json res; + res["path"] = store->printStorePath(path); + return res; +} + +nlohmann::json BuiltPath::Built::toJSON(ref store) const { + nlohmann::json res; + res["drvPath"] = store->printStorePath(drvPath); + for (const auto& [output, path] : outputs) { + res["outputs"][output] = store->printStorePath(path); + } + return res; +} + +StorePathSet BuiltPath::outPaths() const +{ + return std::visit( + overloaded{ + [](BuiltPath::Opaque p) { return StorePathSet{p.path}; }, + [](BuiltPath::Built b) { + StorePathSet res; + for (auto & [_, path] : b.outputs) + res.insert(path); + return res; + }, + }, raw() + ); +} + +nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref store) { + auto res = nlohmann::json::array(); + for (const BuiltPath & buildable : buildables) { + std::visit([&res, store](const auto & buildable) { + res.push_back(buildable.toJSON(store)); + }, buildable.raw()); + } + return res; +} + + +std::string DerivedPath::Opaque::to_string(const Store & store) const { + return store.printStorePath(path); +} + +std::string DerivedPath::Built::to_string(const Store & store) const { + return store.printStorePath(drvPath) + + "!" + + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs)); +} + +std::string DerivedPath::to_string(const Store & store) const +{ + return std::visit( + [&](const auto & req) { return req.to_string(store); }, + this->raw()); +} + + +DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_view s) +{ + return {store.parseStorePath(s)}; +} + +DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view s) +{ + size_t n = s.find("!"); + assert(n != s.npos); + auto drvPath = store.parseStorePath(s.substr(0, n)); + auto outputsS = s.substr(n + 1); + std::set outputs; + if (outputsS != "*") + outputs = tokenizeString>(outputsS, ","); + return {drvPath, outputs}; +} + +DerivedPath DerivedPath::parse(const Store & store, std::string_view s) +{ + size_t n = s.find("!"); + return n == s.npos + ? (DerivedPath) DerivedPath::Opaque::parse(store, s) + : (DerivedPath) DerivedPath::Built::parse(store, s); +} + +RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const +{ + RealisedPath::Set res; + std::visit( + overloaded{ + [&](BuiltPath::Opaque p) { res.insert(p.path); }, + [&](BuiltPath::Built p) { + auto drvHashes = + staticOutputHashes(store, store.readDerivation(p.drvPath)); + for (auto& [outputName, outputPath] : p.outputs) { + if (settings.isExperimentalFeatureEnabled( + "ca-derivations")) { + auto thisRealisation = store.queryRealisation( + DrvOutput{drvHashes.at(outputName), outputName}); + assert(thisRealisation); // We’ve built it, so we must h + // ve the realisation + res.insert(*thisRealisation); + } else { + res.insert(outputPath); + } + } + }, + }, + raw()); + return res; +} +} diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh new file mode 100644 index 000000000..9d6ace069 --- /dev/null +++ b/src/libstore/derived-path.hh @@ -0,0 +1,123 @@ +#pragma once + +#include "util.hh" +#include "path.hh" +#include "realisation.hh" + +#include + +#include + +namespace nix { + +class Store; + +/** + * An opaque derived path. + * + * Opaque derived paths are just store paths, and fully evaluated. They + * cannot be simplified further. Since they are opaque, they cannot be + * built, but they can fetched. + */ +struct DerivedPathOpaque { + StorePath path; + + nlohmann::json toJSON(ref store) const; + std::string to_string(const Store & store) const; + static DerivedPathOpaque parse(const Store & store, std::string_view); +}; + +/** + * A derived path that is built from a derivation + * + * Built derived paths are pair of a derivation and some output names. + * They are evaluated by building the derivation, and then replacing the + * output names with the resulting outputs. + * + * Note that does mean a derived store paths evaluates to multiple + * opaque paths, which is sort of icky as expressions are supposed to + * evaluate to single values. Perhaps this should have just a single + * output name. + */ +struct DerivedPathBuilt { + StorePath drvPath; + std::set outputs; + + std::string to_string(const Store & store) const; + static DerivedPathBuilt parse(const Store & store, std::string_view); +}; + +using _DerivedPathRaw = std::variant< + DerivedPathOpaque, + DerivedPathBuilt +>; + +/** + * A "derived path" is a very simple sort of expression that evaluates + * to (concrete) store path. It is either: + * + * - opaque, in which case it is just a concrete store path with + * possibly no known derivation + * + * - built, in which case it is a pair of a derivation path and an + * output name. + */ +struct DerivedPath : _DerivedPathRaw { + using Raw = _DerivedPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = DerivedPathBuilt; + + inline const Raw & raw() const { + return static_cast(*this); + } + + std::string to_string(const Store & store) const; + static DerivedPath parse(const Store & store, std::string_view); +}; + +/** + * A built derived path with hints in the form of optional concrete output paths. + * + * See 'BuiltPath' for more an explanation. + */ +struct BuiltPathBuilt { + StorePath drvPath; + std::map outputs; + + nlohmann::json toJSON(ref store) const; + static BuiltPathBuilt parse(const Store & store, std::string_view); +}; + +using _BuiltPathRaw = std::variant< + DerivedPath::Opaque, + BuiltPathBuilt +>; + +/** + * A built path. Similar to a `DerivedPath`, but enriched with the corresponding + * output path(s). + */ +struct BuiltPath : _BuiltPathRaw { + using Raw = _BuiltPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = BuiltPathBuilt; + + inline const Raw & raw() const { + return static_cast(*this); + } + + StorePathSet outPaths() const; + RealisedPath::Set toRealisedPaths(Store & store) const; + +}; + +typedef std::vector DerivedPaths; +typedef std::vector BuiltPaths; + +nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref store); + +} diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 8ea5cdc9d..2cf35ec83 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -7,7 +7,7 @@ #include "finally.hh" #include "callback.hh" -#ifdef ENABLE_S3 +#if ENABLE_S3 #include #endif @@ -148,7 +148,7 @@ struct curlFileTransfer : public FileTransfer } LambdaSink finalSink; - std::shared_ptr decompressionSink; + std::shared_ptr decompressionSink; std::optional errorSink; std::exception_ptr writeException; @@ -665,7 +665,7 @@ struct curlFileTransfer : public FileTransfer writeFull(wakeupPipe.writeSide.get(), " "); } -#ifdef ENABLE_S3 +#if ENABLE_S3 std::tuple parseS3Uri(std::string uri) { auto [path, params] = splitUriAndParams(uri); @@ -688,7 +688,7 @@ struct curlFileTransfer : public FileTransfer if (hasPrefix(request.uri, "s3://")) { // FIXME: do this on a worker thread try { -#ifdef ENABLE_S3 +#if ENABLE_S3 auto [bucketName, key, params] = parseS3Uri(request.uri); std::string profile = get(params, "profile").value_or(""); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index bc692ca42..5a62c6529 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -775,7 +775,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) try { - AutoCloseDir dir(opendir(realStoreDir.c_str())); + AutoCloseDir dir(opendir(realStoreDir.get().c_str())); if (!dir) throw SysError("opening directory '%1%'", realStoreDir); /* Read the store and immediately delete all paths that @@ -856,7 +856,7 @@ void LocalStore::autoGC(bool sync) return std::stoll(readFile(*fakeFreeSpaceFile)); struct statvfs st; - if (statvfs(realStoreDir.c_str(), &st)) + if (statvfs(realStoreDir.get().c_str(), &st)) throw SysError("getting filesystem info about '%s'", realStoreDir); return (uint64_t) st.f_bavail * st.f_frsize; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 8d44003f4..d3b27d7be 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -81,7 +81,7 @@ void loadConfFile() /* We only want to send overrides to the daemon, i.e. stuff from ~/.nix/nix.conf or the command line. */ - globalConfig.resetOverriden(); + globalConfig.resetOverridden(); auto files = settings.nixUserConfFiles; for (auto file = files.rbegin(); file != files.rend(); file++) { diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index a51d9c2f1..dd570cd63 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -206,7 +206,10 @@ public: Setting builders{ this, "@" + nixConfDir + "/machines", "builders", - "A semicolon-separated list of build machines, in the format of `nix.machines`."}; + R"( + A semicolon-separated list of build machines. + For the exact format and examples, see [the manual chapter on remote builds](../advanced-topics/distributed-builds.md) + )"}; Setting buildersUseSubstitutes{ this, false, "builders-use-substitutes", @@ -614,8 +617,10 @@ public: Strings{"https://cache.nixos.org/"}, "substituters", R"( - A list of URLs of substituters, separated by whitespace. The default - is `https://cache.nixos.org`. + A list of URLs of substituters, separated by whitespace. Substituters + are tried based on their Priority value, which each substituter can set + independently. Lower value means higher priority. + The default is `https://cache.nixos.org`, with a Priority of 40. )", {"binary-caches"}}; @@ -698,7 +703,7 @@ public: send a series of commands to modify various settings to stdout. The currently recognized commands are: - - `extra-sandbox-paths` + - `extra-sandbox-paths`\ Pass a list of files and directories to be included in the sandbox for this build. One entry per line, terminated by an empty line. Entries have the same format as `sandbox-paths`. diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index a9f53bad9..edaf75136 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -3,6 +3,7 @@ #include "remote-store.hh" #include "serve-protocol.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include "worker-protocol.hh" #include "ssh.hh" #include "derivations.hh" @@ -266,14 +267,23 @@ public: return status; } - void buildPaths(const std::vector & drvPaths, BuildMode buildMode) override + void buildPaths(const std::vector & drvPaths, BuildMode buildMode) override { auto conn(connections->get()); conn->to << cmdBuildPaths; Strings ss; - for (auto & p : drvPaths) - ss.push_back(p.to_string(*this)); + for (auto & p : drvPaths) { + auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); + std::visit(overloaded { + [&](StorePathWithOutputs s) { + ss.push_back(s.to_string(*this)); + }, + [&](StorePath drvPath) { + throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath)); + }, + }, sOrDrvPath); + } conn->to << ss; putBuildSettings(*conn); diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index a58b7733f..f93111fce 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -2,6 +2,8 @@ #include "globals.hh" #include "nar-info-disk-cache.hh" +#include + namespace nix { struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig @@ -50,7 +52,8 @@ protected: const std::string & mimeType) override { auto path2 = binaryCacheDir + "/" + path; - Path tmp = path2 + ".tmp." + std::to_string(getpid()); + static std::atomic counter{0}; + Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter); AutoDelete del(tmp, false); StreamToSourceAdapter source(istream); writeFile(tmp, source); @@ -90,7 +93,7 @@ protected: void LocalBinaryCacheStore::init() { createDirs(binaryCacheDir + "/nar"); - createDirs(binaryCacheDir + realisationsPrefix); + createDirs(binaryCacheDir + "/" + realisationsPrefix); if (writeDebugInfo) createDirs(binaryCacheDir + "/debuginfo"); BinaryCacheStore::init(); diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 55941b771..f8b19d00d 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -18,6 +18,9 @@ struct LocalFSStoreConfig : virtual StoreConfig const PathSetting logDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, "log", "directory where Nix will store state"}; + const PathSetting realStoreDir{(StoreConfig*) this, false, + rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", + "physical path to the Nix store"}; }; class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store @@ -34,7 +37,7 @@ public: /* Register a permanent GC root. */ Path addPermRoot(const StorePath & storePath, const Path & gcRoot); - virtual Path getRealStoreDir() { return storeDir; } + virtual Path getRealStoreDir() { return realStoreDir; } Path toRealPath(const Path & storePath) override { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 90fb4a4bd..d7c7f8e1d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -53,12 +53,15 @@ struct LocalStore::State::Stmts { SQLiteStmt InvalidatePath; SQLiteStmt AddDerivationOutput; SQLiteStmt RegisterRealisedOutput; + SQLiteStmt UpdateRealisedOutput; SQLiteStmt QueryValidDerivers; SQLiteStmt QueryDerivationOutputs; SQLiteStmt QueryRealisedOutput; SQLiteStmt QueryAllRealisedOutputs; SQLiteStmt QueryPathFromHashPart; SQLiteStmt QueryValidPaths; + SQLiteStmt QueryRealisationReferences; + SQLiteStmt AddRealisationReference; }; int getSchema(Path schemaPath) @@ -76,7 +79,7 @@ int getSchema(Path schemaPath) void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) { - const int nixCASchemaVersion = 1; + const int nixCASchemaVersion = 2; int curCASchema = getSchema(schemaPath); if (curCASchema != nixCASchemaVersion) { if (curCASchema > nixCASchemaVersion) { @@ -94,7 +97,39 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) #include "ca-specific-schema.sql.gen.hh" ; db.exec(schema); + curCASchema = nixCASchemaVersion; } + + if (curCASchema < 2) { + SQLiteTxn txn(db); + // Ugly little sql dance to add a new `id` column and make it the primary key + db.exec(R"( + create table Realisations2 ( + id integer primary key autoincrement not null, + drvPath text not null, + outputName text not null, -- symbolic output id, usually "out" + outputPath integer not null, + signatures text, -- space-separated list + foreign key (outputPath) references ValidPaths(id) on delete cascade + ); + insert into Realisations2 (drvPath, outputName, outputPath, signatures) + select drvPath, outputName, outputPath, signatures from Realisations; + drop table Realisations; + alter table Realisations2 rename to Realisations; + )"); + db.exec(R"( + create index if not exists IndexRealisations on Realisations(drvPath, outputName); + + create table if not exists RealisationsRefs ( + referrer integer not null, + realisationReference integer, + foreign key (referrer) references Realisations(id) on delete cascade, + foreign key (realisationReference) references Realisations(id) on delete restrict + ); + )"); + txn.commit(); + } + writeFile(schemaPath, fmt("%d", nixCASchemaVersion)); lockFile(lockFd.get(), ltRead, true); } @@ -106,9 +141,6 @@ LocalStore::LocalStore(const Params & params) , LocalStoreConfig(params) , Store(params) , LocalFSStore(params) - , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", - "physical path to the Nix store"} - , realStoreDir(realStoreDir_) , dbDir(stateDir + "/db") , linksDir(realStoreDir + "/.links") , reservedPath(dbDir + "/reserved") @@ -153,13 +185,13 @@ LocalStore::LocalStore(const Params & params) printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup); else { struct stat st; - if (stat(realStoreDir.c_str(), &st)) + if (stat(realStoreDir.get().c_str(), &st)) throw SysError("getting attributes of path '%1%'", realStoreDir); if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) { - if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1) + if (chown(realStoreDir.get().c_str(), 0, gr->gr_gid) == -1) throw SysError("changing ownership of path '%1%'", realStoreDir); - if (chmod(realStoreDir.c_str(), perm) == -1) + if (chmod(realStoreDir.get().c_str(), perm) == -1) throw SysError("changing permissions on path '%1%'", realStoreDir); } } @@ -310,13 +342,22 @@ LocalStore::LocalStore(const Params & params) if (settings.isExperimentalFeatureEnabled("ca-derivations")) { state->stmts->RegisterRealisedOutput.create(state->db, R"( - insert or replace into Realisations (drvPath, outputName, outputPath) - values (?, ?, (select id from ValidPaths where path = ?)) + insert or replace into Realisations (drvPath, outputName, outputPath, signatures) + values (?, ?, (select id from ValidPaths where path = ?), ?) + ; + )"); + state->stmts->UpdateRealisedOutput.create(state->db, + R"( + update Realisations + set signatures = ? + where + drvPath = ? and + outputName = ? ; )"); state->stmts->QueryRealisedOutput.create(state->db, R"( - select Output.path from Realisations + select Realisations.id, Output.path, Realisations.signatures from Realisations inner join ValidPaths as Output on Output.id = Realisations.outputPath where drvPath = ? and outputName = ? ; @@ -328,6 +369,19 @@ LocalStore::LocalStore(const Params & params) where drvPath = ? ; )"); + state->stmts->QueryRealisationReferences.create(state->db, + R"( + select drvPath, outputName from Realisations + join RealisationsRefs on realisationReference = Realisations.id + where referrer = ?; + )"); + state->stmts->AddRealisationReference.create(state->db, + R"( + insert or replace into RealisationsRefs (referrer, realisationReference) + values ( + ?, + (select id from Realisations where drvPath = ? and outputName = ?)); + )"); } } @@ -437,14 +491,14 @@ void LocalStore::makeStoreWritable() if (getuid() != 0) return; /* Check if /nix/store is on a read-only mount. */ struct statvfs stat; - if (statvfs(realStoreDir.c_str(), &stat) != 0) + if (statvfs(realStoreDir.get().c_str(), &stat) != 0) throw SysError("getting info about the Nix store mount point"); if (stat.f_flag & ST_RDONLY) { if (unshare(CLONE_NEWNS) == -1) throw SysError("setting up a private mount namespace"); - if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) + if (mount(0, realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) throw SysError("remounting %1% writable", realStoreDir); } #endif @@ -652,17 +706,66 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat } } +void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) +{ + settings.requireExperimentalFeature("ca-derivations"); + if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info)) + registerDrvOutput(info); + else + throw Error("cannot register realisation '%s' because it lacks a valid signature", info.outPath.to_string()); +} void LocalStore::registerDrvOutput(const Realisation & info) { settings.requireExperimentalFeature("ca-derivations"); - auto state(_state.lock()); retrySQLite([&]() { - state->stmts->RegisterRealisedOutput.use() - (info.id.strHash()) - (info.id.outputName) - (printStorePath(info.outPath)) - .exec(); + auto state(_state.lock()); + if (auto oldR = queryRealisation_(*state, info.id)) { + if (info.isCompatibleWith(*oldR)) { + auto combinedSignatures = oldR->signatures; + combinedSignatures.insert(info.signatures.begin(), + info.signatures.end()); + state->stmts->UpdateRealisedOutput.use() + (concatStringsSep(" ", combinedSignatures)) + (info.id.strHash()) + (info.id.outputName) + .exec(); + } else { + throw Error("Trying to register a realisation of '%s', but we already " + "have another one locally.\n" + "Local: %s\n" + "Remote: %s", + info.id.to_string(), + printStorePath(oldR->outPath), + printStorePath(info.outPath) + ); + } + } else { + state->stmts->RegisterRealisedOutput.use() + (info.id.strHash()) + (info.id.outputName) + (printStorePath(info.outPath)) + (concatStringsSep(" ", info.signatures)) + .exec(); + } + uint64_t myId = state->db.getLastInsertedRowId(); + for (auto & [outputId, depPath] : info.dependentRealisations) { + auto localRealisation = queryRealisationCore_(*state, outputId); + if (!localRealisation) + throw Error("unable to register the derivation '%s' as it " + "depends on the non existent '%s'", + info.id.to_string(), outputId.to_string()); + if (localRealisation->second.outPath != depPath) + throw Error("unable to register the derivation '%s' as it " + "depends on a realisation of '%s' that doesn’t" + "match what we have locally", + info.id.to_string(), outputId.to_string()); + state->stmts->AddRealisationReference.use() + (myId) + (outputId.strHash()) + (outputId.outputName) + .exec(); + } }); } @@ -1102,15 +1205,20 @@ const PublicKeys & LocalStore::getPublicKeys() return *state->publicKeys; } -bool LocalStore::pathInfoIsTrusted(const ValidPathInfo & info) +bool LocalStore::pathInfoIsUntrusted(const ValidPathInfo & info) { return requireSigs && !info.checkSignatures(*this, getPublicKeys()); } +bool LocalStore::realisationIsUntrusted(const Realisation & realisation) +{ + return requireSigs && !realisation.checkSignatures(getPublicKeys()); +} + void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { - if (checkSigs && pathInfoIsTrusted(info)) + if (checkSigs && pathInfoIsUntrusted(info)) throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path)); addTempRoot(info.path); @@ -1138,17 +1246,13 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, /* While restoring the path from the NAR, compute the hash of the NAR. */ - std::unique_ptr hashSink; - if (!info.ca.has_value() || !info.references.count(info.path)) - hashSink = std::make_unique(htSHA256); - else - hashSink = std::make_unique(htSHA256, std::string(info.path.hashPart())); + HashSink hashSink(htSHA256); - TeeSource wrapperSource { source, *hashSink }; + TeeSource wrapperSource { source, hashSink }; restorePath(realPath, wrapperSource); - auto hashResult = hashSink->finish(); + auto hashResult = hashSink.finish(); if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s", @@ -1158,6 +1262,31 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), info.narSize, hashResult.second); + if (info.ca) { + if (auto foHash = std::get_if(&*info.ca)) { + auto actualFoHash = hashCAPath( + foHash->method, + foHash->hash.type, + info.path + ); + if (foHash->hash != actualFoHash.hash) { + throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", + printStorePath(info.path), + foHash->hash.to_string(Base32, true), + actualFoHash.hash.to_string(Base32, true)); + } + } + if (auto textHash = std::get_if(&*info.ca)) { + auto actualTextHash = hashString(htSHA256, readFile(realPath)); + if (textHash->hash != actualTextHash) { + throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", + printStorePath(info.path), + textHash->hash.to_string(Base32, true), + actualTextHash.to_string(Base32, true)); + } + } + } + autoGC(); canonicalisePathMetaData(realPath, -1); @@ -1426,14 +1555,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Check the content hash (optionally - slow). */ printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i)); - std::unique_ptr hashSink; - if (!info->ca || !info->references.count(info->path)) - hashSink = std::make_unique(info->narHash.type); - else - hashSink = std::make_unique(info->narHash.type, std::string(info->path.hashPart())); + auto hashSink = HashSink(info->narHash.type); - dumpPath(Store::toRealPath(i), *hashSink); - auto current = hashSink->finish(); + dumpPath(Store::toRealPath(i), hashSink); + auto current = hashSink.finish(); if (info->narHash != nullHash && info->narHash != current.first) { printError("path '%s' was modified! expected hash '%s', got '%s'", @@ -1612,6 +1737,18 @@ void LocalStore::addSignatures(const StorePath & storePath, const StringSet & si } +void LocalStore::signRealisation(Realisation & realisation) +{ + // FIXME: keep secret keys in memory. + + auto secretKeyFiles = settings.secretKeyFiles; + + for (auto & secretKeyFile : secretKeyFiles.get()) { + SecretKey secretKey(readFile(secretKeyFile)); + realisation.sign(secretKey); + } +} + void LocalStore::signPathInfo(ValidPathInfo & info) { // FIXME: keep secret keys in memory. @@ -1639,18 +1776,97 @@ void LocalStore::createUser(const std::string & userName, uid_t userId) } } -std::optional LocalStore::queryRealisation( - const DrvOutput& id) { - typedef std::optional Ret; - return retrySQLite([&]() -> Ret { +std::optional> LocalStore::queryRealisationCore_( + LocalStore::State & state, + const DrvOutput & id) +{ + auto useQueryRealisedOutput( + state.stmts->QueryRealisedOutput.use() + (id.strHash()) + (id.outputName)); + if (!useQueryRealisedOutput.next()) + return std::nullopt; + auto realisationDbId = useQueryRealisedOutput.getInt(0); + auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1)); + auto signatures = + tokenizeString(useQueryRealisedOutput.getStr(2)); + + return {{ + realisationDbId, + Realisation{ + .id = id, + .outPath = outputPath, + .signatures = signatures, + } + }}; +} + +std::optional LocalStore::queryRealisation_( + LocalStore::State & state, + const DrvOutput & id) +{ + auto maybeCore = queryRealisationCore_(state, id); + if (!maybeCore) + return std::nullopt; + auto [realisationDbId, res] = *maybeCore; + + std::map dependentRealisations; + auto useRealisationRefs( + state.stmts->QueryRealisationReferences.use() + (realisationDbId)); + while (useRealisationRefs.next()) { + auto depId = DrvOutput { + Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)), + useRealisationRefs.getStr(1), + }; + auto dependentRealisation = queryRealisationCore_(state, depId); + assert(dependentRealisation); // Enforced by the db schema + auto outputPath = dependentRealisation->second.outPath; + dependentRealisations.insert({depId, outputPath}); + } + + res.dependentRealisations = dependentRealisations; + + return { res }; +} + +std::optional +LocalStore::queryRealisation(const DrvOutput & id) +{ + return retrySQLite>([&]() { auto state(_state.lock()); - auto use(state->stmts->QueryRealisedOutput.use()(id.strHash())( - id.outputName)); - if (!use.next()) - return std::nullopt; - auto outputPath = parseStorePath(use.getStr(0)); - return Ret{ - Realisation{.id = id, .outPath = outputPath}}; + return queryRealisation_(*state, id); }); } + +FixedOutputHash LocalStore::hashCAPath( + const FileIngestionMethod & method, const HashType & hashType, + const StorePath & path) +{ + return hashCAPath(method, hashType, Store::toRealPath(path), path.hashPart()); +} + +FixedOutputHash LocalStore::hashCAPath( + const FileIngestionMethod & method, + const HashType & hashType, + const Path & path, + const std::string_view pathHash +) +{ + HashModuloSink caSink ( hashType, std::string(pathHash) ); + switch (method) { + case FileIngestionMethod::Recursive: + dumpPath(path, caSink); + break; + case FileIngestionMethod::Flat: + readFile(path, caSink); + break; + } + auto hash = caSink.finish().first; + return FixedOutputHash{ + .method = method, + .hash = hash, + }; +} + } // namespace nix diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 03bb0218d..a01d48c4b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -83,9 +83,6 @@ private: public: - PathSetting realStoreDir_; - - const Path realStoreDir; const Path dbDir; const Path linksDir; const Path reservedPath; @@ -136,7 +133,8 @@ public: void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; - bool pathInfoIsTrusted(const ValidPathInfo &) override; + bool pathInfoIsUntrusted(const ValidPathInfo &) override; + bool realisationIsUntrusted(const Realisation & ) override; void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override; @@ -202,8 +200,11 @@ public: /* Register the store path 'output' as the output named 'outputName' of derivation 'deriver'. */ void registerDrvOutput(const Realisation & info) override; + void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override; void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output); + std::optional queryRealisation_(State & state, const DrvOutput & id); + std::optional> queryRealisationCore_(State & state, const DrvOutput & id); std::optional queryRealisation(const DrvOutput&) override; private: @@ -272,16 +273,30 @@ private: bool isValidPath_(State & state, const StorePath & path); void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers); - /* Add signatures to a ValidPathInfo using the secret keys + /* Add signatures to a ValidPathInfo or Realisation using the secret keys specified by the ‘secret-key-files’ option. */ void signPathInfo(ValidPathInfo & info); - - Path getRealStoreDir() override { return realStoreDir; } + void signRealisation(Realisation &); void createUser(const std::string & userName, uid_t userId) override; + // XXX: Make a generic `Store` method + FixedOutputHash hashCAPath( + const FileIngestionMethod & method, + const HashType & hashType, + const StorePath & path); + + FixedOutputHash hashCAPath( + const FileIngestionMethod & method, + const HashType & hashType, + const Path & path, + const std::string_view pathHash + ); + friend struct LocalDerivationGoal; + friend struct PathSubstitutionGoal; friend struct SubstitutionGoal; + friend struct DerivationGoal; }; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index cf0933705..b6652984c 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -9,7 +9,7 @@ libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc) libstore_LIBS = libutil libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread -ifneq ($(OS), FreeBSD) +ifeq ($(OS), Linux) libstore_LDFLAGS += -ldl endif diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index f58816ad8..b4929b445 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -6,98 +6,73 @@ #include "thread-pool.hh" #include "topo-sort.hh" #include "callback.hh" +#include "closure.hh" namespace nix { - void Store::computeFSClosure(const StorePathSet & startPaths, StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) { - struct State - { - size_t pending; - StorePathSet & paths; - std::exception_ptr exc; - }; + std::function(const StorePath & path, std::future> &)> queryDeps; + if (flipDirection) + queryDeps = [&](const StorePath& path, + std::future> & fut) { + StorePathSet res; + StorePathSet referrers; + queryReferrers(path, referrers); + for (auto& ref : referrers) + if (ref != path) + res.insert(ref); - Sync state_(State{0, paths_, 0}); + if (includeOutputs) + for (auto& i : queryValidDerivers(path)) + res.insert(i); - std::function enqueue; + if (includeDerivers && path.isDerivation()) + for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + if (maybeOutPath && isValidPath(*maybeOutPath)) + res.insert(*maybeOutPath); + return res; + }; + else + queryDeps = [&](const StorePath& path, + std::future> & fut) { + StorePathSet res; + auto info = fut.get(); + for (auto& ref : info->references) + if (ref != path) + res.insert(ref); - std::condition_variable done; + if (includeOutputs && path.isDerivation()) + for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + if (maybeOutPath && isValidPath(*maybeOutPath)) + res.insert(*maybeOutPath); - enqueue = [&](const StorePath & path) -> void { - { - auto state(state_.lock()); - if (state->exc) return; - if (!state->paths.insert(path).second) return; - state->pending++; - } + if (includeDerivers && info->deriver && isValidPath(*info->deriver)) + res.insert(*info->deriver); + return res; + }; - queryPathInfo(path, {[&](std::future> fut) { - // FIXME: calls to isValidPath() should be async - - try { - auto info = fut.get(); - - if (flipDirection) { - - StorePathSet referrers; - queryReferrers(path, referrers); - for (auto & ref : referrers) - if (ref != path) - enqueue(ref); - - if (includeOutputs) - for (auto & i : queryValidDerivers(path)) - enqueue(i); - - if (includeDerivers && path.isDerivation()) - for (auto & i : queryDerivationOutputs(path)) - if (isValidPath(i) && queryPathInfo(i)->deriver == path) - enqueue(i); - - } else { - - for (auto & ref : info->references) - if (ref != path) - enqueue(ref); - - if (includeOutputs && path.isDerivation()) - for (auto & i : queryDerivationOutputs(path)) - if (isValidPath(i)) enqueue(i); - - if (includeDerivers && info->deriver && isValidPath(*info->deriver)) - enqueue(*info->deriver); - - } - - { - auto state(state_.lock()); - assert(state->pending); - if (!--state->pending) done.notify_one(); - } - - } catch (...) { - auto state(state_.lock()); - if (!state->exc) state->exc = std::current_exception(); - assert(state->pending); - if (!--state->pending) done.notify_one(); - }; - }}); - }; - - for (auto & startPath : startPaths) - enqueue(startPath); - - { - auto state(state_.lock()); - while (state->pending) state.wait(done); - if (state->exc) std::rethrow_exception(state->exc); - } + computeClosure( + startPaths, paths_, + [&](const StorePath& path, + std::function>&)> + processEdges) { + std::promise> promise; + std::function>)> + getDependencies = + [&](std::future> fut) { + try { + promise.set_value(queryDeps(path, fut)); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }; + queryPathInfo(path, getDependencies); + processEdges(promise); + }); } - void Store::computeFSClosure(const StorePath & startPath, StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) { @@ -117,7 +92,7 @@ std::optional getDerivationCA(const BasicDerivation & drv) return std::nullopt; } -void Store::queryMissing(const std::vector & targets, +void Store::queryMissing(const std::vector & targets, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, uint64_t & downloadSize_, uint64_t & narSize_) { @@ -145,7 +120,7 @@ void Store::queryMissing(const std::vector & targets, Sync state_(State{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_}); - std::function doPath; + std::function doPath; auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { { @@ -154,7 +129,7 @@ void Store::queryMissing(const std::vector & targets, } for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, StorePathWithOutputs { i.first, i.second })); + pool.enqueue(std::bind(doPath, DerivedPath::Built { i.first, i.second })); }; auto checkOutput = [&]( @@ -177,24 +152,25 @@ void Store::queryMissing(const std::vector & targets, drvState->outPaths.insert(outPath); if (!drvState->left) { for (auto & path : drvState->outPaths) - pool.enqueue(std::bind(doPath, StorePathWithOutputs { path } )); + pool.enqueue(std::bind(doPath, DerivedPath::Opaque { path } )); } } } }; - doPath = [&](const StorePathWithOutputs & path) { + doPath = [&](const DerivedPath & req) { { auto state(state_.lock()); - if (!state->done.insert(path.to_string(*this)).second) return; + if (!state->done.insert(req.to_string(*this)).second) return; } - if (path.path.isDerivation()) { - if (!isValidPath(path.path)) { + std::visit(overloaded { + [&](DerivedPath::Built bfd) { + if (!isValidPath(bfd.drvPath)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); - state->unknown.insert(path.path); + state->unknown.insert(bfd.drvPath); return; } @@ -202,52 +178,54 @@ void Store::queryMissing(const std::vector & targets, /* true for regular derivations, and CA derivations for which we have a trust mapping for all wanted outputs. */ auto knownOutputPaths = true; - for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(path.path)) { + for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(bfd.drvPath)) { if (!pathOpt) { knownOutputPaths = false; break; } - if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt)) + if (wantOutput(outputName, bfd.outputs) && !isValidPath(*pathOpt)) invalid.insert(*pathOpt); } if (knownOutputPaths && invalid.empty()) return; - auto drv = make_ref(derivationFromPath(path.path)); - ParsedDerivation parsedDrv(StorePath(path.path), *drv); + auto drv = make_ref(derivationFromPath(bfd.drvPath)); + ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv); if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref>(DrvState(invalid.size())); for (auto & output : invalid) - pool.enqueue(std::bind(checkOutput, path.path, drv, output, drvState)); + pool.enqueue(std::bind(checkOutput, bfd.drvPath, drv, output, drvState)); } else - mustBuildDrv(path.path, *drv); + mustBuildDrv(bfd.drvPath, *drv); - } else { + }, + [&](DerivedPath::Opaque bo) { - if (isValidPath(path.path)) return; + if (isValidPath(bo.path)) return; SubstitutablePathInfos infos; - querySubstitutablePathInfos({{path.path, std::nullopt}}, infos); + querySubstitutablePathInfos({{bo.path, std::nullopt}}, infos); if (infos.empty()) { auto state(state_.lock()); - state->unknown.insert(path.path); + state->unknown.insert(bo.path); return; } - auto info = infos.find(path.path); + auto info = infos.find(bo.path); assert(info != infos.end()); { auto state(state_.lock()); - state->willSubstitute.insert(path.path); + state->willSubstitute.insert(bo.path); state->downloadSize += info->second.downloadSize; state->narSize += info->second.narSize; } for (auto & ref : info->second.references) - pool.enqueue(std::bind(doPath, StorePathWithOutputs { ref })); - } + pool.enqueue(std::bind(doPath, DerivedPath::Opaque { ref })); + }, + }, req.raw()); }; for (auto & path : targets) @@ -276,5 +254,44 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) }}); } +std::map drvOutputReferences( + const std::set & inputRealisations, + const StorePathSet & pathReferences) +{ + std::map res; + for (const auto & input : inputRealisations) { + if (pathReferences.count(input.outPath)) { + res.insert({input.id, input.outPath}); + } + } + + return res; +} + +std::map drvOutputReferences( + Store & store, + const Derivation & drv, + const StorePath & outputPath) +{ + std::set inputRealisations; + + for (const auto& [inputDrv, outputNames] : drv.inputDrvs) { + auto outputHashes = + staticOutputHashes(store, store.readDerivation(inputDrv)); + for (const auto& outputName : outputNames) { + auto thisRealisation = store.queryRealisation( + DrvOutput{outputHashes.at(outputName), outputName}); + if (!thisRealisation) + throw Error( + "output '%s' of derivation '%s' isn’t built", outputName, + store.printStorePath(inputDrv)); + inputRealisations.insert(*thisRealisation); + } + } + + auto info = store.queryPathInfo(outputPath); + + return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); +} } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 1d8d2d57e..9dd81ddfb 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -4,6 +4,7 @@ #include "globals.hh" #include +#include namespace nix { @@ -38,6 +39,15 @@ create table if not exists NARs ( foreign key (cache) references BinaryCaches(id) on delete cascade ); +create table if not exists Realisations ( + cache integer not null, + outputId text not null, + content blob, -- Json serialisation of the realisation, or null if the realisation is absent + timestamp integer not null, + primary key (cache, outputId), + foreign key (cache) references BinaryCaches(id) on delete cascade +); + create table if not exists LastPurge ( dummy text primary key, value integer @@ -63,7 +73,9 @@ public: struct State { SQLite db; - SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR, purgeCache; + SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, + queryNAR, insertRealisation, insertMissingRealisation, + queryRealisation, purgeCache; std::map caches; }; @@ -98,6 +110,26 @@ public: state->queryNAR.create(state->db, "select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))"); + state->insertRealisation.create(state->db, + R"( + insert or replace into Realisations(cache, outputId, content, timestamp) + values (?, ?, ?, ?) + )"); + + state->insertMissingRealisation.create(state->db, + R"( + insert or replace into Realisations(cache, outputId, timestamp) + values (?, ?, ?) + )"); + + state->queryRealisation.create(state->db, + R"( + select content from Realisations + where cache = ? and outputId = ? and + ((content is null and timestamp > ?) or + (content is not null and timestamp > ?)) + )"); + /* Periodically purge expired entries from the database. */ retrySQLite([&]() { auto now = time(0); @@ -212,6 +244,38 @@ public: }); } + std::pair> lookupRealisation( + const std::string & uri, const DrvOutput & id) override + { + return retrySQLite>>( + [&]() -> std::pair> { + auto state(_state.lock()); + + auto & cache(getCache(*state, uri)); + + auto now = time(0); + + auto queryRealisation(state->queryRealisation.use() + (cache.id) + (id.to_string()) + (now - settings.ttlNegativeNarInfoCache) + (now - settings.ttlPositiveNarInfoCache)); + + if (!queryRealisation.next()) + return {oUnknown, 0}; + + if (queryRealisation.isNull(0)) + return {oInvalid, 0}; + + auto realisation = + std::make_shared(Realisation::fromJSON( + nlohmann::json::parse(queryRealisation.getStr(0)), + "Local disk cache")); + + return {oValid, realisation}; + }); + } + void upsertNarInfo( const std::string & uri, const std::string & hashPart, std::shared_ptr info) override @@ -251,6 +315,39 @@ public: } }); } + + void upsertRealisation( + const std::string & uri, + const Realisation & realisation) override + { + retrySQLite([&]() { + auto state(_state.lock()); + + auto & cache(getCache(*state, uri)); + + state->insertRealisation.use() + (cache.id) + (realisation.id.to_string()) + (realisation.toJSON().dump()) + (time(0)).exec(); + }); + + } + + virtual void upsertAbsentRealisation( + const std::string & uri, + const DrvOutput & id) override + { + retrySQLite([&]() { + auto state(_state.lock()); + + auto & cache(getCache(*state, uri)); + state->insertMissingRealisation.use() + (cache.id) + (id.to_string()) + (time(0)).exec(); + }); + } }; ref getNarInfoDiskCache() diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 04de2c5eb..2dcaa76a4 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -2,6 +2,7 @@ #include "ref.hh" #include "nar-info.hh" +#include "realisation.hh" namespace nix { @@ -29,6 +30,15 @@ public: virtual void upsertNarInfo( const std::string & uri, const std::string & hashPart, std::shared_ptr info) = 0; + + virtual void upsertRealisation( + const std::string & uri, + const Realisation & realisation) = 0; + virtual void upsertAbsentRealisation( + const std::string & uri, + const DrvOutput & id) = 0; + virtual std::pair> lookupRealisation( + const std::string & uri, const DrvOutput & id) = 0; }; /* Return a singleton cache object that can be used concurrently by diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 78d587139..d95e54af1 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -198,7 +198,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Make the containing directory writable, but only if it's not the store itself (we don't want or need to mess with its permissions). */ - bool mustToggle = dirOf(path) != realStoreDir; + bool mustToggle = dirOf(path) != realStoreDir.get(); if (mustToggle) makeWritable(dirOf(path)); /* When we're done, make the directory read-only again and reset diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index c5c3ae3dc..5e383a9a4 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -91,6 +91,8 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const StringSet res; for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) res.insert(i); + if (!derivationHasKnownOutputPaths(drv.type())) + res.insert("ca-derivations"); return res; } diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc new file mode 100644 index 000000000..865d64cf2 --- /dev/null +++ b/src/libstore/path-with-outputs.cc @@ -0,0 +1,71 @@ +#include "path-with-outputs.hh" +#include "store-api.hh" + +namespace nix { + +std::string StorePathWithOutputs::to_string(const Store & store) const +{ + return outputs.empty() + ? store.printStorePath(path) + : store.printStorePath(path) + "!" + concatStringsSep(",", outputs); +} + + +DerivedPath StorePathWithOutputs::toDerivedPath() const +{ + if (!outputs.empty() || path.isDerivation()) + return DerivedPath::Built { path, outputs }; + else + return DerivedPath::Opaque { path }; +} + + +std::vector toDerivedPaths(const std::vector ss) +{ + std::vector reqs; + for (auto & s : ss) reqs.push_back(s.toDerivedPath()); + return reqs; +} + + +std::variant StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) +{ + return std::visit(overloaded { + [&](DerivedPath::Opaque bo) -> std::variant { + if (bo.path.isDerivation()) { + // drv path gets interpreted as "build", not "get drv file itself" + return bo.path; + } + return StorePathWithOutputs { bo.path }; + }, + [&](DerivedPath::Built bfd) -> std::variant { + return StorePathWithOutputs { bfd.drvPath, bfd.outputs }; + }, + }, p.raw()); +} + + +std::pair parsePathWithOutputs(std::string_view s) +{ + size_t n = s.find("!"); + return n == s.npos + ? std::make_pair(s, std::set()) + : std::make_pair(((std::string_view) s).substr(0, n), + tokenizeString>(((std::string_view) s).substr(n + 1), ",")); +} + + +StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs) +{ + auto [path, outputs] = parsePathWithOutputs(pathWithOutputs); + return StorePathWithOutputs { store.parseStorePath(path), std::move(outputs) }; +} + + +StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs) +{ + auto [path, outputs] = parsePathWithOutputs(pathWithOutputs); + return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) }; +} + +} diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh new file mode 100644 index 000000000..4c4023dcb --- /dev/null +++ b/src/libstore/path-with-outputs.hh @@ -0,0 +1,35 @@ +#pragma once + +#include + +#include "path.hh" +#include "derived-path.hh" + +namespace nix { + +struct StorePathWithOutputs +{ + StorePath path; + std::set outputs; + + std::string to_string(const Store & store) const; + + DerivedPath toDerivedPath() const; + + static std::variant tryFromDerivedPath(const DerivedPath &); +}; + +std::vector toDerivedPaths(const std::vector); + +std::pair parsePathWithOutputs(std::string_view s); + +class Store; + +/* Split a string specifying a derivation and a set of outputs + (/nix/store/hash-foo!out1,out2,...) into the derivation path + and the outputs. */ +StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs); + +StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); + +} diff --git a/src/libstore/path.cc b/src/libstore/path.cc index dc9dc3897..e642abcd5 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -82,19 +82,4 @@ PathSet Store::printStorePathSet(const StorePathSet & paths) const return res; } -std::pair parsePathWithOutputs(std::string_view s) -{ - size_t n = s.find("!"); - return n == s.npos - ? std::make_pair(s, std::set()) - : std::make_pair(((std::string_view) s).substr(0, n), - tokenizeString>(((std::string_view) s).substr(n + 1), ",")); -} - -StorePathWithOutputs Store::parsePathWithOutputs(const std::string & s) -{ - auto [path, outputs] = nix::parsePathWithOutputs(s); - return {parseStorePath(path), std::move(outputs)}; -} - } diff --git a/src/libstore/path.hh b/src/libstore/path.hh index b03a0f69d..06ba0663b 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -69,16 +69,6 @@ typedef std::map> StorePathCAMap; /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; -struct StorePathWithOutputs -{ - StorePath path; - std::set outputs; - - std::string to_string(const Store & store) const; -}; - -std::pair parsePathWithOutputs(std::string_view s); - } namespace std { diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index cd74af4ee..eadec594c 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -1,5 +1,6 @@ #include "realisation.hh" #include "store-api.hh" +#include "closure.hh" #include namespace nix { @@ -21,35 +22,134 @@ std::string DrvOutput::to_string() const { return strHash() + "!" + outputName; } +std::set Realisation::closure(Store & store, const std::set & startOutputs) +{ + std::set res; + Realisation::closure(store, startOutputs, res); + return res; +} + +void Realisation::closure(Store & store, const std::set & startOutputs, std::set & res) +{ + auto getDeps = [&](const Realisation& current) -> std::set { + std::set res; + for (auto& [currentDep, _] : current.dependentRealisations) { + if (auto currentRealisation = store.queryRealisation(currentDep)) + res.insert(*currentRealisation); + else + throw Error( + "Unrealised derivation '%s'", currentDep.to_string()); + } + return res; + }; + + computeClosure( + startOutputs, res, + [&](const Realisation& current, + std::function>&)> + processEdges) { + std::promise> promise; + try { + auto res = getDeps(current); + promise.set_value(res); + } catch (...) { + promise.set_exception(std::current_exception()); + } + return processEdges(promise); + }); +} + nlohmann::json Realisation::toJSON() const { + auto jsonDependentRealisations = nlohmann::json::object(); + for (auto & [depId, depOutPath] : dependentRealisations) + jsonDependentRealisations.emplace(depId.to_string(), depOutPath.to_string()); return nlohmann::json{ {"id", id.to_string()}, {"outPath", outPath.to_string()}, + {"signatures", signatures}, + {"dependentRealisations", jsonDependentRealisations}, }; } Realisation Realisation::fromJSON( const nlohmann::json& json, const std::string& whence) { - auto getField = [&](std::string fieldName) -> std::string { + auto getOptionalField = [&](std::string fieldName) -> std::optional { auto fieldIterator = json.find(fieldName); if (fieldIterator == json.end()) + return std::nullopt; + return *fieldIterator; + }; + auto getField = [&](std::string fieldName) -> std::string { + if (auto field = getOptionalField(fieldName)) + return *field; + else throw Error( "Drv output info file '%1%' is corrupt, missing field %2%", whence, fieldName); - return *fieldIterator; }; + StringSet signatures; + if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end()) + signatures.insert(signaturesIterator->begin(), signaturesIterator->end()); + + std::map dependentRealisations; + if (auto jsonDependencies = json.find("dependentRealisations"); jsonDependencies != json.end()) + for (auto & [jsonDepId, jsonDepOutPath] : jsonDependencies->get>()) + dependentRealisations.insert({DrvOutput::parse(jsonDepId), StorePath(jsonDepOutPath)}); + return Realisation{ .id = DrvOutput::parse(getField("id")), .outPath = StorePath(getField("outPath")), + .signatures = signatures, + .dependentRealisations = dependentRealisations, }; } +std::string Realisation::fingerprint() const +{ + auto serialized = toJSON(); + serialized.erase("signatures"); + return serialized.dump(); +} + +void Realisation::sign(const SecretKey & secretKey) +{ + signatures.insert(secretKey.signDetached(fingerprint())); +} + +bool Realisation::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const +{ + return verifyDetached(fingerprint(), sig, publicKeys); +} + +size_t Realisation::checkSignatures(const PublicKeys & publicKeys) const +{ + // FIXME: Maybe we should return `maxSigs` if the realisation corresponds to + // an input-addressed one − because in that case the drv is enough to check + // it − but we can't know that here. + + size_t good = 0; + for (auto & sig : signatures) + if (checkSignature(publicKeys, sig)) + good++; + return good; +} + StorePath RealisedPath::path() const { return std::visit([](auto && arg) { return arg.getPath(); }, raw); } +bool Realisation::isCompatibleWith(const Realisation & other) const +{ + assert (id == other.id); + if (outPath == other.outPath) { + assert(dependentRealisations == other.dependentRealisations); + return true; + } + return false; +} + void RealisedPath::closure( Store& store, const RealisedPath::Set& startPaths, diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index fc92d3c17..05d2bc44f 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -3,6 +3,7 @@ #include "path.hh" #include #include "comparator.hh" +#include "crypto.hh" namespace nix { @@ -25,9 +26,29 @@ struct Realisation { DrvOutput id; StorePath outPath; + StringSet signatures; + + /** + * The realisations that are required for the current one to be valid. + * + * When importing this realisation, the store will first check that all its + * dependencies exist, and map to the correct output path + */ + std::map dependentRealisations; + nlohmann::json toJSON() const; static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); + std::string fingerprint() const; + void sign(const SecretKey &); + bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; + size_t checkSignatures(const PublicKeys & publicKeys) const; + + static std::set closure(Store &, const std::set &); + static void closure(Store &, const std::set &, std::set& res); + + bool isCompatibleWith(const Realisation & other) const; + StorePath getPath() const { return outPath; } GENERATE_CMP(Realisation, me->id, me->outPath); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 0d884389a..aec243637 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1,5 +1,6 @@ #include "serialise.hh" #include "util.hh" +#include "path-with-outputs.hh" #include "remote-fs-accessor.hh" #include "remote-store.hh" #include "worker-protocol.hh" @@ -50,6 +51,19 @@ void write(const Store & store, Sink & out, const ContentAddress & ca) out << renderContentAddress(ca); } + +DerivedPath read(const Store & store, Source & from, Phantom _) +{ + auto s = readString(from); + return DerivedPath::parse(store, s); +} + +void write(const Store & store, Sink & out, const DerivedPath & req) +{ + out << req.to_string(store); +} + + Realisation read(const Store & store, Source & from, Phantom _) { std::string rawInput = readString(from); @@ -58,13 +72,23 @@ Realisation read(const Store & store, Source & from, Phantom _) "remote-protocol" ); } + void write(const Store & store, Sink & out, const Realisation & realisation) -{ out << realisation.toJSON().dump(); } +{ + out << realisation.toJSON().dump(); +} + DrvOutput read(const Store & store, Source & from, Phantom _) -{ return DrvOutput::parse(readString(from)); } +{ + return DrvOutput::parse(readString(from)); +} + void write(const Store & store, Sink & out, const DrvOutput & drvOutput) -{ out << drvOutput.to_string(); } +{ + out << drvOutput.to_string(); +} + std::optional read(const Store & store, Source & from, Phantom> _) { @@ -629,8 +653,12 @@ void RemoteStore::registerDrvOutput(const Realisation & info) { auto conn(getConnection()); conn->to << wopRegisterDrvOutput; - conn->to << info.id.to_string(); - conn->to << std::string(info.outPath.to_string()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + conn->to << info.id.to_string(); + conn->to << std::string(info.outPath.to_string()); + } else { + worker_proto::write(*this, conn->to, info); + } conn.processStderr(); } @@ -640,22 +668,49 @@ std::optional RemoteStore::queryRealisation(const DrvOutput & conn->to << wopQueryRealisation; conn->to << id.to_string(); conn.processStderr(); - auto outPaths = worker_proto::read(*this, conn->from, Phantom>{}); - if (outPaths.empty()) - return std::nullopt; - return {Realisation{.id = id, .outPath = *outPaths.begin()}}; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + auto outPaths = worker_proto::read(*this, conn->from, Phantom>{}); + if (outPaths.empty()) + return std::nullopt; + return {Realisation{.id = id, .outPath = *outPaths.begin()}}; + } else { + auto realisations = worker_proto::read(*this, conn->from, Phantom>{}); + if (realisations.empty()) + return std::nullopt; + return *realisations.begin(); + } } +static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector & reqs) +{ + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 30) { + worker_proto::write(store, conn->to, reqs); + } else { + Strings ss; + for (auto & p : reqs) { + auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); + std::visit(overloaded { + [&](StorePathWithOutputs s) { + ss.push_back(s.to_string(store)); + }, + [&](StorePath drvPath) { + throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", + store.printStorePath(drvPath), + GET_PROTOCOL_MAJOR(conn->daemonVersion), + GET_PROTOCOL_MINOR(conn->daemonVersion)); + }, + }, sOrDrvPath); + } + conn->to << ss; + } +} -void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) +void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) { auto conn(getConnection()); conn->to << wopBuildPaths; assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); - Strings ss; - for (auto & p : drvPaths) - ss.push_back(p.to_string(*this)); - conn->to << ss; + writeDerivedPaths(*this, conn, drvPaths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) conn->to << buildMode; else @@ -677,10 +732,12 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD conn->to << buildMode; conn.processStderr(); BuildResult res; - unsigned int status; - conn->from >> status >> res.errorMsg; - res.status = (BuildResult::Status) status; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0xc) { + res.status = (BuildResult::Status) readInt(conn->from); + conn->from >> res.errorMsg; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { + conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime; + } + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) { auto builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); res.builtOutputs = builtOutputs; } @@ -792,7 +849,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s } -void RemoteStore::queryMissing(const std::vector & targets, +void RemoteStore::queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) { @@ -803,10 +860,7 @@ void RemoteStore::queryMissing(const std::vector & targets // to prevent a deadlock. goto fallback; conn->to << wopQueryMissing; - Strings ss; - for (auto & p : targets) - ss.push_back(p.to_string(*this)); - conn->to << ss; + writeDerivedPaths(*this, conn, targets); conn.processStderr(); willBuild = worker_proto::read(*this, conn->from, Phantom {}); willSubstitute = worker_proto::read(*this, conn->from, Phantom {}); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index b3a9910a3..6cf76a46d 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -85,7 +85,7 @@ public: std::optional queryRealisation(const DrvOutput &) override; - void buildPaths(const std::vector & paths, BuildMode buildMode) override; + void buildPaths(const std::vector & paths, BuildMode buildMode) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; @@ -108,7 +108,7 @@ public: void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - void queryMissing(const std::vector & targets, + void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) override; diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb index 351037822..2bb1ea130 100644 --- a/src/libstore/sandbox-defaults.sb +++ b/src/libstore/sandbox-defaults.sb @@ -32,7 +32,9 @@ (literal "/tmp") (subpath TMPDIR)) ; Some packages like to read the system version. -(allow file-read* (literal "/System/Library/CoreServices/SystemVersion.plist")) +(allow file-read* + (literal "/System/Library/CoreServices/SystemVersion.plist") + (literal "/System/Library/CoreServices/SystemVersionCompat.plist")) ; Without this line clang cannot write to /dev/null, breaking some configure tests. (allow file-read-metadata (literal "/dev")) diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 0a17387cb..02d0810cc 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -5,7 +5,7 @@ namespace nix { #define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_2 0x5452eecb -#define SERVE_PROTOCOL_VERSION 0x206 +#define SERVE_PROTOCOL_VERSION (2 << 8 | 6) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 235eed37a..93f72675d 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -50,7 +50,7 @@ std::unique_ptr SSHMaster::startCommand(const std::string options.dieWithParent = false; conn->sshPid = startProcess([&]() { - restoreSignals(); + restoreProcessContext(); close(in.writeSide.get()); close(out.readSide.get()); @@ -110,7 +110,7 @@ Path SSHMaster::startMaster() options.dieWithParent = false; state->sshMaster = startProcess([&]() { - restoreSignals(); + restoreProcessContext(); close(out.readSide.get()); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 77c310988..6736adb24 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -53,13 +53,6 @@ StorePath Store::followLinksToStorePath(std::string_view path) const } -StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const -{ - auto [path2, outputs] = nix::parsePathWithOutputs(path); - return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) }; -} - - /* Store paths have the following form: = /- @@ -344,6 +337,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, return info; } +StringSet StoreConfig::getDefaultSystemFeatures() +{ + auto res = settings.systemFeatures.get(); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) + res.insert("ca-derivations"); + return res; +} Store::Store(const Params & params) : StoreConfig(params) @@ -536,10 +536,10 @@ void Store::queryPathInfo(const StorePath & storePath, void Store::substitutePaths(const StorePathSet & paths) { - std::vector paths2; + std::vector paths2; for (auto & path : paths) if (!path.isDerivation()) - paths2.push_back({path}); + paths2.push_back(DerivedPath::Opaque{path}); uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; queryMissing(paths2, @@ -547,8 +547,8 @@ void Store::substitutePaths(const StorePathSet & paths) if (!willSubstitute.empty()) try { - std::vector subs; - for (auto & p : willSubstitute) subs.push_back({p}); + std::vector subs; + for (auto & p : willSubstitute) subs.push_back(DerivedPath::Opaque{p}); buildPaths(subs); } catch (Error & e) { logWarning(e.info()); @@ -787,20 +787,39 @@ std::map copyPaths(ref srcStore, ref dstStor RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { StorePathSet storePaths; - std::set realisations; + std::set toplevelRealisations; for (auto & path : paths) { storePaths.insert(path.path()); if (auto realisation = std::get_if(&path.raw)) { settings.requireExperimentalFeature("ca-derivations"); - realisations.insert(*realisation); + toplevelRealisations.insert(*realisation); } } auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); + + ThreadPool pool; + try { - for (auto & realisation : realisations) { - dstStore->registerDrvOutput(realisation); - } - } catch (MissingExperimentalFeature & e) { + // Copy the realisation closure + processGraph( + pool, Realisation::closure(*srcStore, toplevelRealisations), + [&](const Realisation& current) -> std::set { + std::set children; + for (const auto& [drvOutput, _] : current.dependentRealisations) { + auto currentChild = srcStore->queryRealisation(drvOutput); + if (!currentChild) + throw Error( + "Incomplete realisation closure: '%s' is a " + "dependency of '%s' but isn’t registered", + drvOutput.to_string(), current.id.to_string()); + children.insert(*currentChild); + } + return children; + }, + [&](const Realisation& current) -> void { + dstStore->registerDrvOutput(current, checkSigs); + }); + } catch (MissingExperimentalFeature& e) { // Don't fail if the remote doesn't support CA derivations is it might // not be within our control to change that, and we might still want // to at least copy the output paths. diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 71a28eeb8..9657d2adf 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -2,6 +2,7 @@ #include "realisation.hh" #include "path.hh" +#include "derived-path.hh" #include "hash.hh" #include "content-address.hh" #include "serialise.hh" @@ -179,6 +180,8 @@ struct StoreConfig : public Config StoreConfig() = delete; + StringSet getDefaultSystemFeatures(); + virtual ~StoreConfig() { } virtual const std::string name() = 0; @@ -195,7 +198,7 @@ struct StoreConfig : public Config Setting wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; - Setting systemFeatures{this, settings.systemFeatures, + Setting systemFeatures{this, getDefaultSystemFeatures(), "system-features", "Optional features that the system this store builds on implements (like \"kvm\")."}; @@ -261,11 +264,6 @@ public: PathSet printStorePathSet(const StorePathSet & path) const; - /* Split a string specifying a derivation and a set of outputs - (/nix/store/hash-foo!out1,out2,...) into the derivation path - and the outputs. */ - StorePathWithOutputs parsePathWithOutputs(const string & s); - /* Display a set of paths in human-readable form (i.e., between quotes and separated by commas). */ std::string showPaths(const StorePathSet & paths); @@ -289,8 +287,6 @@ public: result. */ StorePath followLinksToStorePath(std::string_view path) const; - StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const; - /* Constructs a unique store path name. */ StorePath makeStorePath(std::string_view type, std::string_view hash, std::string_view name) const; @@ -384,7 +380,12 @@ public: we don't really want to add the dependencies listed in a nar info we don't trust anyyways. */ - virtual bool pathInfoIsTrusted(const ValidPathInfo &) + virtual bool pathInfoIsUntrusted(const ValidPathInfo &) + { + return true; + } + + virtual bool realisationIsUntrusted(const Realisation & ) { return true; } @@ -480,6 +481,8 @@ public: */ virtual void registerDrvOutput(const Realisation & output) { unsupported("registerDrvOutput"); } + virtual void registerDrvOutput(const Realisation & output, CheckSigsFlag checkSigs) + { return registerDrvOutput(output); } /* Write a NAR dump of a store path. */ virtual void narFromPath(const StorePath & path, Sink & sink) = 0; @@ -493,7 +496,7 @@ public: recursively building any sub-derivations. For inputs that are not derivations, substitute them. */ virtual void buildPaths( - const std::vector & paths, + const std::vector & paths, BuildMode buildMode = bmNormal); /* Build a single non-materialized derivation (i.e. not from an @@ -655,7 +658,7 @@ public: /* Given a set of paths that are to be built, return the set of derivations that will be built, and the set of output paths that will be substituted. */ - virtual void queryMissing(const std::vector & targets, + virtual void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize); @@ -863,4 +866,9 @@ std::pair splitUriAndParams(const std::string & uri) std::optional getDerivationCA(const BasicDerivation & drv); +std::map drvOutputReferences( + Store & store, + const Derivation & drv, + const StorePath & outputPath); + } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 95f08bc9a..e89183d40 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -9,7 +9,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x11c +#define PROTOCOL_VERSION (1 << 8 | 31) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -86,9 +86,11 @@ namespace worker_proto { MAKE_WORKER_PROTO(, std::string); MAKE_WORKER_PROTO(, StorePath); MAKE_WORKER_PROTO(, ContentAddress); +MAKE_WORKER_PROTO(, DerivedPath); MAKE_WORKER_PROTO(, Realisation); MAKE_WORKER_PROTO(, DrvOutput); +MAKE_WORKER_PROTO(template, std::vector); MAKE_WORKER_PROTO(template, std::set); #define X_ template @@ -113,6 +115,26 @@ MAKE_WORKER_PROTO(X_, Y_); MAKE_WORKER_PROTO(, std::optional); MAKE_WORKER_PROTO(, std::optional); +template +std::vector read(const Store & store, Source & from, Phantom> _) +{ + std::vector resSet; + auto size = readNum(from); + while (size--) { + resSet.push_back(read(store, from, Phantom {})); + } + return resSet; +} + +template +void write(const Store & store, Sink & out, const std::vector & resSet) +{ + out << resSet.size(); + for (auto & key : resSet) { + write(store, out, key); + } +} + template std::set read(const Store & store, Source & from, Phantom> _) { diff --git a/src/libutil/closure.hh b/src/libutil/closure.hh new file mode 100644 index 000000000..779b9b2d5 --- /dev/null +++ b/src/libutil/closure.hh @@ -0,0 +1,69 @@ +#include +#include +#include "sync.hh" + +using std::set; + +namespace nix { + +template +using GetEdgesAsync = std::function> &)>)>; + +template +void computeClosure( + const set startElts, + set & res, + GetEdgesAsync getEdgesAsync +) +{ + struct State + { + size_t pending; + set & res; + std::exception_ptr exc; + }; + + Sync state_(State{0, res, 0}); + + std::function enqueue; + + std::condition_variable done; + + enqueue = [&](const T & current) -> void { + { + auto state(state_.lock()); + if (state->exc) return; + if (!state->res.insert(current).second) return; + state->pending++; + } + + getEdgesAsync(current, [&](std::promise> & prom) { + try { + auto children = prom.get_future().get(); + for (auto & child : children) + enqueue(child); + { + auto state(state_.lock()); + assert(state->pending); + if (!--state->pending) done.notify_one(); + } + } catch (...) { + auto state(state_.lock()); + if (!state->exc) state->exc = std::current_exception(); + assert(state->pending); + if (!--state->pending) done.notify_one(); + }; + }); + }; + + for (auto & startElt : startElts) + enqueue(startElt); + + { + auto state(state_.lock()); + while (state->pending) state.wait(done); + if (state->exc) std::rethrow_exception(state->exc); + } +} + +} diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index 0315dc506..eecd5b819 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -25,6 +25,8 @@ } #define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) #define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args) +#define GENERATE_NEQ(args...) GENERATE_ONE_CMP(!=, args) #define GENERATE_CMP(args...) \ GENERATE_EQUAL(args) \ - GENERATE_LEQ(args) + GENERATE_LEQ(args) \ + GENERATE_NEQ(args) diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 986ba2976..7e725cae1 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,10 +1,11 @@ #include "compression.hh" +#include "tarfile.hh" #include "util.hh" #include "finally.hh" #include "logging.hh" -#include -#include +#include +#include #include #include @@ -27,7 +28,7 @@ struct ChunkedCompressionSink : CompressionSink const size_t CHUNK_SIZE = sizeof(outbuf) << 2; while (!data.empty()) { size_t n = std::min(CHUNK_SIZE, data.size()); - writeInternal(data); + writeInternal(data.substr(0, n)); data.remove_prefix(n); } } @@ -35,6 +36,95 @@ struct ChunkedCompressionSink : CompressionSink virtual void writeInternal(std::string_view data) = 0; }; +struct ArchiveDecompressionSource : Source +{ + std::unique_ptr archive = 0; + Source & src; + ArchiveDecompressionSource(Source & src) : src(src) {} + ~ArchiveDecompressionSource() override {} + size_t read(char * data, size_t len) override { + struct archive_entry * ae; + if (!archive) { + archive = std::make_unique(src, true); + this->archive->check(archive_read_next_header(this->archive->archive, &ae), + "failed to read header (%s)"); + if (archive_filter_count(this->archive->archive) < 2) { + throw CompressionError("input compression not recognized"); + } + } + ssize_t result = archive_read_data(this->archive->archive, data, len); + if (result > 0) return result; + if (result == 0) { + throw EndOfFile("reached end of compressed file"); + } + this->archive->check(result, "failed to read compressed data (%s)"); + return result; + } +}; + +struct ArchiveCompressionSink : CompressionSink +{ + Sink & nextSink; + struct archive * archive; + + ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel) : nextSink(nextSink) { + archive = archive_write_new(); + if (!archive) throw Error("failed to initialize libarchive"); + check(archive_write_add_filter_by_name(archive, format.c_str()), "couldn't initialize compression (%s)"); + check(archive_write_set_format_raw(archive)); + if (format == "xz" && parallel) { + check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0")); + } + // disable internal buffering + check(archive_write_set_bytes_per_block(archive, 0)); + // disable output padding + check(archive_write_set_bytes_in_last_block(archive, 1)); + open(); + } + + ~ArchiveCompressionSink() override + { + if (archive) archive_write_free(archive); + } + + void finish() override + { + flush(); + check(archive_write_close(archive)); + } + + void check(int err, const std::string & reason = "failed to compress (%s)") + { + if (err == ARCHIVE_EOF) + throw EndOfFile("reached end of archive"); + else if (err != ARCHIVE_OK) + throw Error(reason, archive_error_string(this->archive)); + } + + void write(std::string_view data) override + { + ssize_t result = archive_write_data(archive, data.data(), data.length()); + if (result <= 0) check(result); + } + +private: + void open() + { + check(archive_write_open(archive, this, nullptr, ArchiveCompressionSink::callback_write, nullptr)); + auto ae = archive_entry_new(); + archive_entry_set_filetype(ae, AE_IFREG); + check(archive_write_header(archive, ae)); + archive_entry_free(ae); + } + + static ssize_t callback_write(struct archive * archive, void * _self, const void * buffer, size_t length) + { + auto self = (ArchiveCompressionSink *) _self; + self->nextSink({(const char *) buffer, length}); + return length; + } +}; + struct NoneSink : CompressionSink { Sink & nextSink; @@ -43,171 +133,6 @@ struct NoneSink : CompressionSink void write(std::string_view data) override { nextSink(data); } }; -struct GzipDecompressionSink : CompressionSink -{ - Sink & nextSink; - z_stream strm; - bool finished = false; - uint8_t outbuf[BUFSIZ]; - - GzipDecompressionSink(Sink & nextSink) : nextSink(nextSink) - { - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - strm.avail_in = 0; - strm.next_in = Z_NULL; - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); - - // Enable gzip and zlib decoding (+32) with 15 windowBits - int ret = inflateInit2(&strm,15+32); - if (ret != Z_OK) - throw CompressionError("unable to initialise gzip encoder"); - } - - ~GzipDecompressionSink() - { - inflateEnd(&strm); - } - - void finish() override - { - CompressionSink::flush(); - write({}); - } - - void write(std::string_view data) override - { - assert(data.size() <= std::numeric_limits::max()); - - strm.next_in = (Bytef *) data.data(); - strm.avail_in = data.size(); - - while (!finished && (!data.data() || strm.avail_in)) { - checkInterrupt(); - - int ret = inflate(&strm,Z_SYNC_FLUSH); - if (ret != Z_OK && ret != Z_STREAM_END) - throw CompressionError("error while decompressing gzip file: %d (%d, %d)", - zError(ret), data.size(), strm.avail_in); - - finished = ret == Z_STREAM_END; - - if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out}); - strm.next_out = (Bytef *) outbuf; - strm.avail_out = sizeof(outbuf); - } - } - } -}; - -struct XzDecompressionSink : CompressionSink -{ - Sink & nextSink; - uint8_t outbuf[BUFSIZ]; - lzma_stream strm = LZMA_STREAM_INIT; - bool finished = false; - - XzDecompressionSink(Sink & nextSink) : nextSink(nextSink) - { - lzma_ret ret = lzma_stream_decoder( - &strm, UINT64_MAX, LZMA_CONCATENATED); - if (ret != LZMA_OK) - throw CompressionError("unable to initialise lzma decoder"); - - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); - } - - ~XzDecompressionSink() - { - lzma_end(&strm); - } - - void finish() override - { - CompressionSink::flush(); - write({}); - } - - void write(std::string_view data) override - { - strm.next_in = (const unsigned char *) data.data(); - strm.avail_in = data.size(); - - while (!finished && (!data.data() || strm.avail_in)) { - checkInterrupt(); - - lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH); - if (ret != LZMA_OK && ret != LZMA_STREAM_END) - throw CompressionError("error %d while decompressing xz file", ret); - - finished = ret == LZMA_STREAM_END; - - if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out}); - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); - } - } - } -}; - -struct BzipDecompressionSink : ChunkedCompressionSink -{ - Sink & nextSink; - bz_stream strm; - bool finished = false; - - BzipDecompressionSink(Sink & nextSink) : nextSink(nextSink) - { - memset(&strm, 0, sizeof(strm)); - int ret = BZ2_bzDecompressInit(&strm, 0, 0); - if (ret != BZ_OK) - throw CompressionError("unable to initialise bzip2 decoder"); - - strm.next_out = (char *) outbuf; - strm.avail_out = sizeof(outbuf); - } - - ~BzipDecompressionSink() - { - BZ2_bzDecompressEnd(&strm); - } - - void finish() override - { - flush(); - write({}); - } - - void writeInternal(std::string_view data) override - { - assert(data.size() <= std::numeric_limits::max()); - - strm.next_in = (char *) data.data(); - strm.avail_in = data.size(); - - while (strm.avail_in) { - checkInterrupt(); - - int ret = BZ2_bzDecompress(&strm); - if (ret != BZ_OK && ret != BZ_STREAM_END) - throw CompressionError("error while decompressing bzip2 file"); - - finished = ret == BZ_STREAM_END; - - if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out}); - strm.next_out = (char *) outbuf; - strm.avail_out = sizeof(outbuf); - } - } - } -}; - struct BrotliDecompressionSink : ChunkedCompressionSink { Sink & nextSink; @@ -268,159 +193,24 @@ ref decompress(const std::string & method, const std::string & in) return ssink.s; } -ref makeDecompressionSink(const std::string & method, Sink & nextSink) +std::unique_ptr makeDecompressionSink(const std::string & method, Sink & nextSink) { if (method == "none" || method == "") - return make_ref(nextSink); - else if (method == "xz") - return make_ref(nextSink); - else if (method == "bzip2") - return make_ref(nextSink); - else if (method == "gzip") - return make_ref(nextSink); + return std::make_unique(nextSink); else if (method == "br") - return make_ref(nextSink); + return std::make_unique(nextSink); else - throw UnknownCompressionMethod("unknown compression method '%s'", method); + return sourceToSink([&](Source & source) { + auto decompressionSource = std::make_unique(source); + decompressionSource->drainInto(nextSink); + }); } -struct XzCompressionSink : CompressionSink -{ - Sink & nextSink; - uint8_t outbuf[BUFSIZ]; - lzma_stream strm = LZMA_STREAM_INIT; - bool finished = false; - - XzCompressionSink(Sink & nextSink, bool parallel) : nextSink(nextSink) - { - lzma_ret ret; - bool done = false; - - if (parallel) { -#ifdef HAVE_LZMA_MT - lzma_mt mt_options = {}; - mt_options.flags = 0; - mt_options.timeout = 300; // Using the same setting as the xz cmd line - mt_options.preset = LZMA_PRESET_DEFAULT; - mt_options.filters = NULL; - mt_options.check = LZMA_CHECK_CRC64; - mt_options.threads = lzma_cputhreads(); - mt_options.block_size = 0; - if (mt_options.threads == 0) - mt_options.threads = 1; - // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the - // number of threads. - ret = lzma_stream_encoder_mt(&strm, &mt_options); - done = true; -#else - printMsg(lvlError, "warning: parallel XZ compression requested but not supported, falling back to single-threaded compression"); -#endif - } - - if (!done) - ret = lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64); - - if (ret != LZMA_OK) - throw CompressionError("unable to initialise lzma encoder"); - - // FIXME: apply the x86 BCJ filter? - - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); - } - - ~XzCompressionSink() - { - lzma_end(&strm); - } - - void finish() override - { - CompressionSink::flush(); - write({}); - } - - void write(std::string_view data) override - { - strm.next_in = (const unsigned char *) data.data(); - strm.avail_in = data.size(); - - while (!finished && (!data.data() || strm.avail_in)) { - checkInterrupt(); - - lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH); - if (ret != LZMA_OK && ret != LZMA_STREAM_END) - throw CompressionError("error %d while compressing xz file", ret); - - finished = ret == LZMA_STREAM_END; - - if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out}); - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); - } - } - } -}; - -struct BzipCompressionSink : ChunkedCompressionSink -{ - Sink & nextSink; - bz_stream strm; - bool finished = false; - - BzipCompressionSink(Sink & nextSink) : nextSink(nextSink) - { - memset(&strm, 0, sizeof(strm)); - int ret = BZ2_bzCompressInit(&strm, 9, 0, 30); - if (ret != BZ_OK) - throw CompressionError("unable to initialise bzip2 encoder"); - - strm.next_out = (char *) outbuf; - strm.avail_out = sizeof(outbuf); - } - - ~BzipCompressionSink() - { - BZ2_bzCompressEnd(&strm); - } - - void finish() override - { - flush(); - writeInternal({}); - } - - void writeInternal(std::string_view data) override - { - assert(data.size() <= std::numeric_limits::max()); - - strm.next_in = (char *) data.data(); - strm.avail_in = data.size(); - - while (!finished && (!data.data() || strm.avail_in)) { - checkInterrupt(); - - int ret = BZ2_bzCompress(&strm, data.data() ? BZ_RUN : BZ_FINISH); - if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END) - throw CompressionError("error %d while compressing bzip2 file", ret); - - finished = ret == BZ_STREAM_END; - - if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { - nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out}); - strm.next_out = (char *) outbuf; - strm.avail_out = sizeof(outbuf); - } - } - } -}; - struct BrotliCompressionSink : ChunkedCompressionSink { Sink & nextSink; uint8_t outbuf[BUFSIZ]; - BrotliEncoderState *state; + BrotliEncoderState * state; bool finished = false; BrotliCompressionSink(Sink & nextSink) : nextSink(nextSink) @@ -471,12 +261,14 @@ struct BrotliCompressionSink : ChunkedCompressionSink ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel) { + std::vector la_supports = { + "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd" + }; + if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) { + return make_ref(nextSink, method, parallel); + } if (method == "none") return make_ref(nextSink); - else if (method == "xz") - return make_ref(nextSink, parallel); - else if (method == "bzip2") - return make_ref(nextSink); else if (method == "br") return make_ref(nextSink); else diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index dd666a4e1..338a0d9f2 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -8,14 +8,16 @@ namespace nix { -struct CompressionSink : BufferedSink +struct CompressionSink : BufferedSink, FinishSink { - virtual void finish() = 0; + using BufferedSink::operator (); + using BufferedSink::write; + using FinishSink::finish; }; ref decompress(const std::string & method, const std::string & in); -ref makeDecompressionSink(const std::string & method, Sink & nextSink); +std::unique_ptr makeDecompressionSink(const std::string & method, Sink & nextSink); ref compress(const std::string & method, const std::string & in, const bool parallel = false); diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 7467e5ac0..bda07cd55 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -20,7 +20,7 @@ bool Config::set(const std::string & name, const std::string & value) return false; } i->second.setting->set(value, append); - i->second.setting->overriden = true; + i->second.setting->overridden = true; return true; } @@ -35,7 +35,7 @@ void Config::addSetting(AbstractSetting * setting) auto i = unknownSettings.find(setting->name); if (i != unknownSettings.end()) { setting->set(i->second); - setting->overriden = true; + setting->overridden = true; unknownSettings.erase(i); set = true; } @@ -48,7 +48,7 @@ void Config::addSetting(AbstractSetting * setting) alias, setting->name); else { setting->set(i->second); - setting->overriden = true; + setting->overridden = true; unknownSettings.erase(i); set = true; } @@ -69,10 +69,10 @@ void AbstractConfig::reapplyUnknownSettings() set(s.first, s.second); } -void Config::getSettings(std::map & res, bool overridenOnly) +void Config::getSettings(std::map & res, bool overriddenOnly) { for (auto & opt : _settings) - if (!opt.second.isAlias && (!overridenOnly || opt.second.setting->overriden)) + if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden)) res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } @@ -136,10 +136,10 @@ void AbstractConfig::applyConfigFile(const Path & path) } catch (SysError &) { } } -void Config::resetOverriden() +void Config::resetOverridden() { for (auto & s : _settings) - s.second.setting->overriden = false; + s.second.setting->overridden = false; } nlohmann::json Config::toJSON() @@ -169,7 +169,7 @@ AbstractSetting::AbstractSetting( void AbstractSetting::setDefault(const std::string & str) { - if (!overriden) set(str); + if (!overridden) set(str); } nlohmann::json AbstractSetting::toJSON() @@ -203,7 +203,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) .description = fmt("Set the `%s` setting.", name), .category = category, .labels = {"value"}, - .handler = {[=](std::string s) { overriden = true; set(s); }}, + .handler = {[=](std::string s) { overridden = true; set(s); }}, }); if (isAppendable()) @@ -212,7 +212,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) .description = fmt("Append to the `%s` setting.", name), .category = category, .labels = {"value"}, - .handler = {[=](std::string s) { overriden = true; set(s, true); }}, + .handler = {[=](std::string s) { overridden = true; set(s, true); }}, }); } @@ -365,16 +365,16 @@ bool GlobalConfig::set(const std::string & name, const std::string & value) return false; } -void GlobalConfig::getSettings(std::map & res, bool overridenOnly) +void GlobalConfig::getSettings(std::map & res, bool overriddenOnly) { for (auto & config : *configRegistrations) - config->getSettings(res, overridenOnly); + config->getSettings(res, overriddenOnly); } -void GlobalConfig::resetOverriden() +void GlobalConfig::resetOverridden() { for (auto & config : *configRegistrations) - config->resetOverriden(); + config->resetOverridden(); } nlohmann::json GlobalConfig::toJSON() diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 71e31656d..bf81b4892 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -71,9 +71,9 @@ public: /** * Adds the currently known settings to the given result map `res`. * - res: map to store settings in - * - overridenOnly: when set to true only overridden settings will be added to `res` + * - overriddenOnly: when set to true only overridden settings will be added to `res` */ - virtual void getSettings(std::map & res, bool overridenOnly = false) = 0; + virtual void getSettings(std::map & res, bool overriddenOnly = false) = 0; /** * Parses the configuration in `contents` and applies it @@ -91,7 +91,7 @@ public: /** * Resets the `overridden` flag of all Settings */ - virtual void resetOverriden() = 0; + virtual void resetOverridden() = 0; /** * Outputs all settings to JSON @@ -127,7 +127,7 @@ public: MyClass() : Config(readConfigFile("/etc/my-app.conf")) { - std::cout << foo << "\n"; // will print 123 unless overriden + std::cout << foo << "\n"; // will print 123 unless overridden } }; */ @@ -163,9 +163,9 @@ public: void addSetting(AbstractSetting * setting); - void getSettings(std::map & res, bool overridenOnly = false) override; + void getSettings(std::map & res, bool overriddenOnly = false) override; - void resetOverriden() override; + void resetOverridden() override; nlohmann::json toJSON() override; @@ -184,7 +184,7 @@ public: int created = 123; - bool overriden = false; + bool overridden = false; void setDefault(const std::string & str); @@ -215,7 +215,7 @@ protected: virtual void convertToArg(Args & args, const std::string & category); - bool isOverriden() const { return overriden; } + bool isOverridden() const { return overridden; } }; /* A setting of type T. */ @@ -252,7 +252,7 @@ public: virtual void override(const T & v) { - overriden = true; + overridden = true; value = v; } @@ -324,9 +324,9 @@ struct GlobalConfig : public AbstractConfig bool set(const std::string & name, const std::string & value) override; - void getSettings(std::map & res, bool overridenOnly = false) override; + void getSettings(std::map & res, bool overriddenOnly = false) override; - void resetOverriden() override; + void resetOverridden() override; nlohmann::json toJSON() override; diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 5341c58e6..3a6415ee3 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,7 +6,7 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) -libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context +libutil_LDFLAGS = -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context ifeq ($(HAVE_LIBCPUID), 1) libutil_LDFLAGS += -lcpuid diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index d1a16b6ba..374b48d79 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -201,6 +201,61 @@ static DefaultStackAllocator defaultAllocatorSingleton; StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton; +std::unique_ptr sourceToSink(std::function fun) +{ + struct SourceToSink : FinishSink + { + typedef boost::coroutines2::coroutine coro_t; + + std::function fun; + std::optional coro; + + SourceToSink(std::function fun) : fun(fun) + { + } + + std::string_view cur; + + void operator () (std::string_view in) override + { + if (in.empty()) return; + cur = in; + + if (!coro) + coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) { + LambdaSource source([&](char *out, size_t out_len) { + if (cur.empty()) { + yield(); + if (yield.get()) { + return (size_t)0; + } + } + + size_t n = std::min(cur.size(), out_len); + memcpy(out, cur.data(), n); + cur.remove_prefix(n); + return n; + }); + fun(source); + }); + + if (!*coro) { abort(); } + + if (!cur.empty()) (*coro)(false); + } + + void finish() { + if (!coro) return; + if (!*coro) abort(); + (*coro)(true); + if (*coro) abort(); + } + }; + + return std::make_unique(fun); +} + + std::unique_ptr sinkToSource( std::function fun, std::function eof) @@ -212,7 +267,6 @@ std::unique_ptr sinkToSource( std::function fun; std::function eof; std::optional coro; - bool started = false; SinkToSource(std::function fun, std::function eof) : fun(fun), eof(eof) diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 5bbbc7ce3..0fe6e8332 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -25,6 +25,13 @@ struct NullSink : Sink { } }; + +struct FinishSink : virtual Sink +{ + virtual void finish() = 0; +}; + + /* A buffered abstract sink. Warning: a BufferedSink should not be used from multiple threads concurrently. */ struct BufferedSink : virtual Sink @@ -281,6 +288,7 @@ struct ChainSource : Source size_t read(char * data, size_t len) override; }; +std::unique_ptr sourceToSink(std::function fun); /* Convert a function that feeds data into a Sink into a Source. The Source executes the function as a coroutine. */ diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 2da169ba7..24905130d 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -2,83 +2,78 @@ #include #include "serialise.hh" +#include "tarfile.hh" namespace nix { -struct TarArchive { - struct archive * archive; - Source * source; - std::vector buffer; +static int callback_open(struct archive *, void * self) +{ + return ARCHIVE_OK; +} - void check(int err, const char * reason = "failed to extract archive: %s") - { - if (err == ARCHIVE_EOF) - throw EndOfFile("reached end of archive"); - else if (err != ARCHIVE_OK) - throw Error(reason, archive_error_string(this->archive)); +static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer) +{ + auto self = (TarArchive *) _self; + *buffer = self->buffer.data(); + + try { + return self->source->read((char *) self->buffer.data(), 4096); + } catch (EndOfFile &) { + return 0; + } catch (std::exception & err) { + archive_set_error(archive, EIO, "Source threw exception: %s", err.what()); + return -1; } +} - TarArchive(Source & source) : buffer(4096) - { - this->archive = archive_read_new(); - this->source = &source; +static int callback_close(struct archive *, void * self) +{ + return ARCHIVE_OK; +} +void TarArchive::check(int err, const std::string & reason) +{ + if (err == ARCHIVE_EOF) + throw EndOfFile("reached end of archive"); + else if (err != ARCHIVE_OK) + throw Error(reason, archive_error_string(this->archive)); +} + +TarArchive::TarArchive(Source & source, bool raw) : buffer(4096) +{ + this->archive = archive_read_new(); + this->source = &source; + + if (!raw) { archive_read_support_filter_all(archive); archive_read_support_format_all(archive); - check(archive_read_open(archive, - (void *)this, - TarArchive::callback_open, - TarArchive::callback_read, - TarArchive::callback_close), - "failed to open archive: %s"); - } - - TarArchive(const Path & path) - { - this->archive = archive_read_new(); - + } else { archive_read_support_filter_all(archive); - archive_read_support_format_all(archive); - check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s"); + archive_read_support_format_raw(archive); + archive_read_support_format_empty(archive); } + check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)"); +} - TarArchive(const TarArchive &) = delete; - void close() - { - check(archive_read_close(archive), "failed to close archive: %s"); - } +TarArchive::TarArchive(const Path & path) +{ + this->archive = archive_read_new(); - ~TarArchive() - { - if (this->archive) archive_read_free(this->archive); - } + archive_read_support_filter_all(archive); + archive_read_support_format_all(archive); + check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s"); +} -private: +void TarArchive::close() +{ + check(archive_read_close(this->archive), "Failed to close archive (%s)"); +} - static int callback_open(struct archive *, void * self) { - return ARCHIVE_OK; - } - - static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer) - { - auto self = (TarArchive *)_self; - *buffer = self->buffer.data(); - - try { - return self->source->read((char *) self->buffer.data(), 4096); - } catch (EndOfFile &) { - return 0; - } catch (std::exception & err) { - archive_set_error(archive, EIO, "source threw exception: %s", err.what()); - return -1; - } - } - - static int callback_close(struct archive *, void * self) { - return ARCHIVE_OK; - } -}; +TarArchive::~TarArchive() +{ + if (this->archive) archive_read_free(this->archive); +} static void extract_archive(TarArchive & archive, const Path & destDir) { diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index 89a024f1d..4d9141fd4 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -1,7 +1,26 @@ #include "serialise.hh" +#include namespace nix { +struct TarArchive { + struct archive * archive; + Source * source; + std::vector buffer; + + void check(int err, const std::string & reason = "failed to extract archive (%s)"); + + TarArchive(Source & source, bool raw = false); + + TarArchive(const Path & path); + + // disable copy constructor + TarArchive(const TarArchive &) = delete; + + void close(); + + ~TarArchive(); +}; void unpackTarfile(Source & source, const Path & destDir); void unpackTarfile(const Path & tarFile, const Path & destDir); diff --git a/src/libutil/tests/closure.cc b/src/libutil/tests/closure.cc new file mode 100644 index 000000000..7597e7807 --- /dev/null +++ b/src/libutil/tests/closure.cc @@ -0,0 +1,70 @@ +#include "closure.hh" +#include + +namespace nix { + +using namespace std; + +map> testGraph = { + { "A", { "B", "C", "G" } }, + { "B", { "A" } }, // Loops back to A + { "C", { "F" } }, // Indirect reference + { "D", { "A" } }, // Not reachable, but has backreferences + { "E", {} }, // Just not reachable + { "F", {} }, + { "G", { "G" } }, // Self reference +}; + +TEST(closure, correctClosure) { + set aClosure; + set expectedClosure = {"A", "B", "C", "F", "G"}; + computeClosure( + {"A"}, + aClosure, + [&](const string currentNode, function> &)> processEdges) { + promise> promisedNodes; + promisedNodes.set_value(testGraph[currentNode]); + processEdges(promisedNodes); + } + ); + + ASSERT_EQ(aClosure, expectedClosure); +} + +TEST(closure, properlyHandlesDirectExceptions) { + struct TestExn {}; + set aClosure; + EXPECT_THROW( + computeClosure( + {"A"}, + aClosure, + [&](const string currentNode, function> &)> processEdges) { + throw TestExn(); + } + ), + TestExn + ); +} + +TEST(closure, properlyHandlesExceptionsInPromise) { + struct TestExn {}; + set aClosure; + EXPECT_THROW( + computeClosure( + {"A"}, + aClosure, + [&](const string currentNode, function> &)> processEdges) { + promise> promise; + try { + throw TestExn(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + processEdges(promise); + } + ), + TestExn + ); +} + +} diff --git a/src/libutil/tests/compression.cc b/src/libutil/tests/compression.cc index 5b7a2c5b9..2efa3266b 100644 --- a/src/libutil/tests/compression.cc +++ b/src/libutil/tests/compression.cc @@ -17,6 +17,24 @@ namespace nix { ASSERT_EQ(*o, "this-is-a-test"); } + TEST(decompress, decompressNoneCompressed) { + auto method = "none"; + auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; + ref o = decompress(method, str); + + ASSERT_EQ(*o, str); + } + + TEST(decompress, decompressEmptyCompressed) { + // Empty-method decompression used e.g. by S3 store + // (Content-Encoding == ""). + auto method = ""; + auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; + ref o = decompress(method, str); + + ASSERT_EQ(*o, str); + } + TEST(decompress, decompressXzCompressed) { auto method = "xz"; auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc index c305af9f5..0ebdaf3db 100644 --- a/src/libutil/tests/config.cc +++ b/src/libutil/tests/config.cc @@ -29,20 +29,20 @@ namespace nix { std::map settings; Setting foo{&config, value, "name-of-the-setting", "description"}; - config.getSettings(settings, /* overridenOnly = */ false); + config.getSettings(settings, /* overriddenOnly = */ false); const auto iter = settings.find("name-of-the-setting"); ASSERT_NE(iter, settings.end()); ASSERT_EQ(iter->second.value, ""); ASSERT_EQ(iter->second.description, "description\n"); } - TEST(Config, getDefinedOverridenSettingNotSet) { + TEST(Config, getDefinedOverriddenSettingNotSet) { Config config; std::string value; std::map settings; Setting foo{&config, value, "name-of-the-setting", "description"}; - config.getSettings(settings, /* overridenOnly = */ true); + config.getSettings(settings, /* overriddenOnly = */ true); const auto e = settings.find("name-of-the-setting"); ASSERT_EQ(e, settings.end()); } @@ -55,7 +55,7 @@ namespace nix { setting.assign("value"); - config.getSettings(settings, /* overridenOnly = */ false); + config.getSettings(settings, /* overriddenOnly = */ false); const auto iter = settings.find("name-of-the-setting"); ASSERT_NE(iter, settings.end()); ASSERT_EQ(iter->second.value, "value"); @@ -69,7 +69,7 @@ namespace nix { ASSERT_TRUE(config.set("name-of-the-setting", "value")); - config.getSettings(settings, /* overridenOnly = */ false); + config.getSettings(settings, /* overriddenOnly = */ false); const auto e = settings.find("name-of-the-setting"); ASSERT_NE(e, settings.end()); ASSERT_EQ(e->second.value, "value"); @@ -100,7 +100,7 @@ namespace nix { { std::map settings; - config.getSettings(settings, /* overridenOnly = */ false); + config.getSettings(settings, /* overriddenOnly = */ false); ASSERT_EQ(settings.find("key"), settings.end()); } @@ -108,17 +108,17 @@ namespace nix { { std::map settings; - config.getSettings(settings, /* overridenOnly = */ false); + config.getSettings(settings, /* overriddenOnly = */ false); ASSERT_EQ(settings["key"].value, "value"); } } - TEST(Config, resetOverriden) { + TEST(Config, resetOverridden) { Config config; - config.resetOverriden(); + config.resetOverridden(); } - TEST(Config, resetOverridenWithSetting) { + TEST(Config, resetOverriddenWithSetting) { Config config; Setting setting{&config, "", "name-of-the-setting", "description"}; @@ -127,7 +127,7 @@ namespace nix { setting.set("foo"); ASSERT_EQ(setting.get(), "foo"); - config.getSettings(settings, /* overridenOnly = */ true); + config.getSettings(settings, /* overriddenOnly = */ true); ASSERT_TRUE(settings.empty()); } @@ -135,18 +135,18 @@ namespace nix { std::map settings; setting.override("bar"); - ASSERT_TRUE(setting.overriden); + ASSERT_TRUE(setting.overridden); ASSERT_EQ(setting.get(), "bar"); - config.getSettings(settings, /* overridenOnly = */ true); + config.getSettings(settings, /* overriddenOnly = */ true); ASSERT_FALSE(settings.empty()); } { std::map settings; - config.resetOverriden(); - ASSERT_FALSE(setting.overriden); - config.getSettings(settings, /* overridenOnly = */ true); + config.resetOverridden(); + ASSERT_FALSE(setting.overridden); + config.getSettings(settings, /* overriddenOnly = */ true); ASSERT_TRUE(settings.empty()); } } diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc index 80646ad3e..aff58e9ee 100644 --- a/src/libutil/tests/url.cc +++ b/src/libutil/tests/url.cc @@ -117,6 +117,24 @@ namespace nix { ASSERT_EQ(parsed, expected); } + TEST(parseURL, parseScopedRFC4007IPv6Address) { + auto s = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080", + .base = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080", + .scheme = "http", + .authority = "[fe80::818c:da4d:8975:415c\%enp0s25]:8080", + .path = "", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + + } + TEST(parseURL, parseIPv6Address) { auto s = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080"; auto parsed = parseURL(s); diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 862d9fa6e..da10a6bbc 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -8,7 +8,7 @@ namespace nix { // URI stuff. const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)"; -const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+"; +const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+(?:%\\w+)?"; const static std::string ipv6AddressRegex = "(?:\\[" + ipv6AddressSegmentRegex + "\\]|" + ipv6AddressSegmentRegex + ")"; const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index ef37275ac..7e57fd7ca 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -32,6 +32,7 @@ #ifdef __linux__ #include +#include #endif @@ -143,16 +144,21 @@ Path canonPath(const Path & path, bool resolveSymlinks) s += '/'; while (i != end && *i != '/') s += *i++; - /* If s points to a symlink, resolve it and restart (since - the symlink target might contain new symlinks). */ + /* If s points to a symlink, resolve it and continue from there */ if (resolveSymlinks && isLink(s)) { if (++followCount >= maxFollow) throw Error("infinite symlink recursion in path '%1%'", path); - temp = absPath(readLink(s), dirOf(s)) - + string(i, end); - i = temp.begin(); /* restart */ + temp = readLink(s) + string(i, end); + i = temp.begin(); end = temp.end(); - s = ""; + if (!temp.empty() && temp[0] == '/') { + s.clear(); /* restart for symlinks pointing to absolute path */ + } else { + s = dirOf(s); + if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = / + s.clear(); + } + } } } } @@ -752,13 +758,13 @@ AutoCloseFD::AutoCloseFD() : fd{-1} {} AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} -AutoCloseFD::AutoCloseFD(AutoCloseFD&& that) : fd{that.fd} +AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} { that.fd = -1; } -AutoCloseFD& AutoCloseFD::operator =(AutoCloseFD&& that) +AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) { close(); fd = that.fd; @@ -789,6 +795,7 @@ void AutoCloseFD::close() if (::close(fd) == -1) /* This should never happen. */ throw SysError("closing file descriptor %1%", fd); + fd = -1; } } @@ -822,6 +829,12 @@ void Pipe::create() } +void Pipe::close() +{ + readSide.close(); + writeSide.close(); +} + ////////////////////////////////////////////////////////////////////// @@ -1109,7 +1122,7 @@ void runProgram2(const RunOptions & options) Strings args_(options.args); args_.push_front(options.program); - restoreSignals(); + restoreProcessContext(); if (options.searchPath) execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); @@ -1121,7 +1134,7 @@ void runProgram2(const RunOptions & options) throw SysError("executing '%1%'", options.program); }, processOptions); - out.writeSide = -1; + out.writeSide.close(); std::thread writerThread; @@ -1134,7 +1147,7 @@ void runProgram2(const RunOptions & options) if (source) { - in.readSide = -1; + in.readSide.close(); writerThread = std::thread([&]() { try { std::vector buf(8 * 1024); @@ -1151,7 +1164,7 @@ void runProgram2(const RunOptions & options) } catch (...) { promise.set_exception(std::current_exception()); } - in.writeSide = -1; + in.writeSide.close(); }); } @@ -1590,7 +1603,7 @@ void startSignalHandlerThread() updateWindowSize(); if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) - throw SysError("quering signal mask"); + throw SysError("querying signal mask"); sigset_t set; sigemptyset(&set); @@ -1605,12 +1618,45 @@ void startSignalHandlerThread() std::thread(signalHandlerThread, set).detach(); } -void restoreSignals() +static void restoreSignals() { if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) throw SysError("restoring signals"); } +#if __linux__ +rlim_t savedStackSize = 0; +#endif + +void setStackSize(size_t stackSize) +{ + #if __linux__ + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { + savedStackSize = limit.rlim_cur; + limit.rlim_cur = stackSize; + setrlimit(RLIMIT_STACK, &limit); + } + #endif +} + +void restoreProcessContext() +{ + restoreSignals(); + + restoreAffinity(); + + #if __linux__ + if (savedStackSize) { + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0) { + limit.rlim_cur = savedStackSize; + setrlimit(RLIMIT_STACK, &limit); + } + } + #endif +} + /* RAII helper to automatically deregister a callback. */ struct InterruptCallbackImpl : InterruptCallback { @@ -1673,10 +1719,11 @@ string showBytes(uint64_t bytes) } +// FIXME: move to libstore/build void commonChildInit(Pipe & logPipe) { const static string pathNullDevice = "/dev/null"; - restoreSignals(); + restoreProcessContext(); /* Put the child in a separate session (and thus a separate process group) so that it has no controlling terminal (meaning diff --git a/src/libutil/util.hh b/src/libutil/util.hh index ad49c65b3..f84d0fb31 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -188,7 +188,6 @@ public: class AutoCloseFD { int fd; - void close(); public: AutoCloseFD(); AutoCloseFD(int fd); @@ -200,6 +199,7 @@ public: int get() const; explicit operator bool() const; int release(); + void close(); }; @@ -216,6 +216,7 @@ class Pipe public: AutoCloseFD readSide, writeSide; void create(); + void close(); }; @@ -299,6 +300,15 @@ std::pair runProgram(const RunOptions & options); void runProgram2(const RunOptions & options); +/* Change the stack size. */ +void setStackSize(size_t stackSize); + + +/* Restore the original inherited Unix process context (such as signal + masks, stack size, CPU affinity). */ +void restoreProcessContext(); + + class ExecError : public Error { public: @@ -512,9 +522,6 @@ class Callback; on the current thread (and thus any threads created by it). */ void startSignalHandlerThread(); -/* Restore default signal handling. */ -void restoreSignals(); - struct InterruptCallback { virtual ~InterruptCallback() { }; diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 65b85b304..3fec2c06c 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -12,6 +12,7 @@ #include "affinity.hh" #include "util.hh" #include "shared.hh" +#include "path-with-outputs.hh" #include "eval.hh" #include "eval-inline.hh" #include "get-drvs.hh" @@ -321,7 +322,8 @@ static void main_nix_build(int argc, char * * argv) state->printStats(); - auto buildPaths = [&](const std::vector & paths) { + auto buildPaths = [&](const std::vector & paths0) { + auto paths = toDerivedPaths(paths0); /* Note: we do this even when !printMissing to efficiently fetch binary cache data. */ uint64_t downloadSize, narSize; @@ -385,6 +387,12 @@ static void main_nix_build(int argc, char * * argv) if (dryRun) return; + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + auto resolvedDrv = drv.tryResolve(*store); + assert(resolvedDrv && "Successfully resolved the derivation"); + drv = *resolvedDrv; + } + // Set the environment. auto env = getEnv(); @@ -420,8 +428,6 @@ static void main_nix_build(int argc, char * * argv) } else env[var.first] = var.second; - restoreAffinity(); - /* Run a shell using the derivation's environment. For convenience, source $stdenv/setup to setup additional environment variables and shell functions. Also don't @@ -471,7 +477,7 @@ static void main_nix_build(int argc, char * * argv) auto argPtrs = stringsToCharPtrs(args); - restoreSignals(); + restoreProcessContext(); logger->stop(); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 0f10a4cbb..e04954d45 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -6,6 +6,7 @@ #include "globals.hh" #include "names.hh" #include "profiles.hh" +#include "path-with-outputs.hh" #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" @@ -418,13 +419,13 @@ static void queryInstSources(EvalState & state, static void printMissing(EvalState & state, DrvInfos & elems) { - std::vector targets; + std::vector targets; for (auto & i : elems) { Path drvPath = i.queryDrvPath(); if (drvPath != "") - targets.push_back({state.store->parseStorePath(drvPath)}); + targets.push_back(DerivedPath::Built{state.store->parseStorePath(drvPath)}); else - targets.push_back({state.store->parseStorePath(i.queryOutPath())}); + targets.push_back(DerivedPath::Opaque{state.store->parseStorePath(i.queryOutPath())}); } printMissing(state.store, targets); @@ -693,17 +694,18 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) if (globals.forceName != "") drv.setName(globals.forceName); - if (drv.queryDrvPath() != "") { - std::vector paths{{globals.state->store->parseStorePath(drv.queryDrvPath())}}; - printMissing(globals.state->store, paths); - if (globals.dryRun) return; - globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); - } else { - printMissing(globals.state->store, - {{globals.state->store->parseStorePath(drv.queryOutPath())}}); - if (globals.dryRun) return; - globals.state->store->ensurePath(globals.state->store->parseStorePath(drv.queryOutPath())); - } + std::vector paths { + (drv.queryDrvPath() != "") + ? (DerivedPath) (DerivedPath::Built { + globals.state->store->parseStorePath(drv.queryDrvPath()) + }) + : (DerivedPath) (DerivedPath::Opaque { + globals.state->store->parseStorePath(drv.queryOutPath()) + }), + }; + printMissing(globals.state->store, paths); + if (globals.dryRun) return; + globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); debug(format("switching to new user environment")); Path generation = createGeneration( diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 168ac492b..5ceb2ae67 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "derivations.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include "local-fs-store.hh" #include "globals.hh" #include "shared.hh" @@ -41,7 +42,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, drvsToBuild.push_back({state.store->parseStorePath(i.queryDrvPath())}); debug(format("building user environment dependencies")); - state.store->buildPaths(drvsToBuild, state.repair ? bmRepair : bmNormal); + state.store->buildPaths( + toDerivedPaths(drvsToBuild), + state.repair ? bmRepair : bmNormal); /* Construct the whole top level derivation. */ StorePathSet references; @@ -136,7 +139,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, debug("building user environment"); std::vector topLevelDrvs; topLevelDrvs.push_back({topLevelDrv}); - state.store->buildPaths(topLevelDrvs, state.repair ? bmRepair : bmNormal); + state.store->buildPaths( + toDerivedPaths(topLevelDrvs), + state.repair ? bmRepair : bmNormal); /* Switch the current user environment to the output path. */ auto store2 = state.store.dynamic_pointer_cast(); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 94d4881dd..b327793e7 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -10,6 +10,7 @@ #include "worker-protocol.hh" #include "graphml.hh" #include "legacy.hh" +#include "path-with-outputs.hh" #include #include @@ -62,7 +63,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true) auto store2 = std::dynamic_pointer_cast(store); if (path.path.isDerivation()) { - if (build) store->buildPaths({path}); + if (build) store->buildPaths({path.toDerivedPath()}); auto outputPaths = store->queryDerivationOutputMap(path.path); Derivation drv = store->derivationFromPath(path.path); rootNr++; @@ -128,11 +129,13 @@ static void opRealise(Strings opFlags, Strings opArgs) std::vector paths; for (auto & i : opArgs) - paths.push_back(store->followLinksToStorePathWithOutputs(i)); + paths.push_back(followLinksToStorePathWithOutputs(*store, i)); uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; - store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); + store->queryMissing( + toDerivedPaths(paths), + willBuild, willSubstitute, unknown, downloadSize, narSize); if (ignoreUnknown) { std::vector paths2; @@ -148,7 +151,7 @@ static void opRealise(Strings opFlags, Strings opArgs) if (dryRun) return; /* Build all paths at the same time to exploit parallelism. */ - store->buildPaths(paths, buildMode); + store->buildPaths(toDerivedPaths(paths), buildMode); if (!ignoreUnknown) for (auto & i : paths) { @@ -873,13 +876,13 @@ static void opServe(Strings opFlags, Strings opArgs) std::vector paths; for (auto & s : readStrings(in)) - paths.push_back(store->parsePathWithOutputs(s)); + paths.push_back(parsePathWithOutputs(*store, s)); getBuildSettings(); try { MonitorFdHup monitor(in.fd); - store->buildPaths(paths); + store->buildPaths(toDerivedPaths(paths)); out << 0; } catch (Error & e) { assert(e.status); @@ -905,7 +908,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (GET_PROTOCOL_MINOR(clientVersion) >= 3) out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; - if (GET_PROTOCOL_MINOR(clientVersion >= 5)) { + if (GET_PROTOCOL_MINOR(clientVersion >= 6)) { worker_proto::write(*store, out, status.builtOutputs); } diff --git a/src/nix/app.cc b/src/nix/app.cc index cf147c631..01a0064db 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -3,34 +3,79 @@ #include "eval-inline.hh" #include "eval-cache.hh" #include "names.hh" +#include "command.hh" namespace nix { -App Installable::toApp(EvalState & state) +struct InstallableDerivedPath : Installable +{ + ref store; + const DerivedPath derivedPath; + + InstallableDerivedPath(ref store, const DerivedPath & derivedPath) + : store(store) + , derivedPath(derivedPath) + { + } + + + std::string what() override { return derivedPath.to_string(*store); } + + DerivedPaths toDerivedPaths() override + { + return {derivedPath}; + } + + std::optional getStorePath() override + { + return std::nullopt; + } +}; + +/** + * Return the rewrites that are needed to resolve a string whose context is + * included in `dependencies` + */ +StringPairs resolveRewrites(Store & store, const BuiltPaths dependencies) +{ + StringPairs res; + for (auto & dep : dependencies) + if (auto drvDep = std::get_if(&dep)) + for (auto & [ outputName, outputPath ] : drvDep->outputs) + res.emplace( + downstreamPlaceholder(store, drvDep->drvPath, outputName), + store.printStorePath(outputPath) + ); + return res; +} + +/** + * Resolve the given string assuming the given context + */ +std::string resolveString(Store & store, const std::string & toResolve, const BuiltPaths dependencies) +{ + auto rewrites = resolveRewrites(store, dependencies); + return rewriteStrings(toResolve, rewrites); +} + +UnresolvedApp Installable::toApp(EvalState & state) { auto [cursor, attrPath] = getCursor(state); auto type = cursor->getAttr("type")->getString(); - auto checkProgram = [&](const Path & program) - { - if (!state.store->isInStore(program)) - throw Error("app program '%s' is not in the Nix store", program); - }; - if (type == "app") { auto [program, context] = cursor->getAttr("program")->getStringWithContext(); - checkProgram(program); std::vector context2; for (auto & [path, name] : context) context2.push_back({state.store->parseStorePath(path), {name}}); - return App { + return UnresolvedApp{App { .context = std::move(context2), .program = program, - }; + }}; } else if (type == "derivation") { @@ -45,15 +90,32 @@ App Installable::toApp(EvalState & state) ? aMainProgram->getString() : DrvName(name).name; auto program = outPath + "/bin/" + mainProgram; - checkProgram(program); - return App { + return UnresolvedApp { App { .context = { { drvPath, {outputName} } }, .program = program, - }; + }}; } else throw Error("attribute '%s' has unsupported type '%s'", attrPath, type); } +App UnresolvedApp::resolve(ref store) +{ + auto res = unresolved; + + std::vector> installableContext; + + for (auto & ctxElt : unresolved.context) + installableContext.push_back( + std::make_shared(store, ctxElt.toDerivedPath())); + + auto builtContext = build(store, Realise::Outputs, installableContext); + res.program = resolveString(*store, unresolved.program, builtContext); + if (!store->isInStore(res.program)) + throw Error("app program '%s' is not in the Nix store", res.program); + + return res; +} + } diff --git a/src/nix/build.cc b/src/nix/build.cc index 724ce9d79..15923ebc3 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -54,6 +54,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile { auto buildables = build(store, dryRun ? Realise::Nothing : Realise::Outputs, installables, buildMode); + if (json) logger->cout("%s", derivedPathsWithHintsToJSON(buildables, store).dump()); + if (dryRun) return; if (outLink != "") @@ -61,26 +63,23 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile for (const auto & [_i, buildable] : enumerate(buildables)) { auto i = _i; std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](BuiltPath::Opaque bo) { std::string symlink = outLink; if (i) symlink += fmt("-%d", i); store2->addPermRoot(bo.path, absPath(symlink)); }, - [&](BuildableFromDrv bfd) { - auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath); - for (auto & output : builtOutputs) { + [&](BuiltPath::Built bfd) { + for (auto & output : bfd.outputs) { std::string symlink = outLink; if (i) symlink += fmt("-%d", i); if (output.first != "out") symlink += fmt("-%s", output.first); store2->addPermRoot(output.second, absPath(symlink)); } }, - }, buildable); + }, buildable.raw()); } updateProfile(buildables); - - if (json) logger->cout("%s", buildablesToJSON(buildables, store).dump()); } }; diff --git a/src/nix/build.md b/src/nix/build.md index c2f3e387a..20138b7e0 100644 --- a/src/nix/build.md +++ b/src/nix/build.md @@ -81,7 +81,7 @@ path installables are substituted. Unless `--no-link` is specified, after a successful build, it creates symlinks to the store paths of the installables. These symlinks have -the prefix `./result` by default; this can be overriden using the +the prefix `./result` by default; this can be overridden using the `--out-link` option. Each symlink has a suffix `--`, where *N* is the index of the installable (with the left-most installable having index 0), and *outname* is the symbolic derivation output name diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 48f4eb6e3..88bc3d1d1 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -69,8 +69,7 @@ struct CmdBundle : InstallableCommand { auto evalState = getEvalState(); - auto app = installable->toApp(*evalState); - store->buildPaths(app.context); + auto app = installable->toApp(*evalState).resolve(store); auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; @@ -110,7 +109,7 @@ struct CmdBundle : InstallableCommand StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2)); - store->buildPaths({{drvPath}}); + store->buildPaths({ DerivedPath::Built { drvPath } }); auto outPathS = store->printStorePath(outPath); diff --git a/src/nix/copy.cc b/src/nix/copy.cc index f59f7c76b..674cce4b4 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -8,7 +8,7 @@ using namespace nix; -struct CmdCopy : RealisedPathsCommand +struct CmdCopy : BuiltPathsCommand { std::string srcUri, dstUri; @@ -16,10 +16,10 @@ struct CmdCopy : RealisedPathsCommand SubstituteFlag substitute = NoSubstitute; - using RealisedPathsCommand::run; + using BuiltPathsCommand::run; CmdCopy() - : RealisedPathsCommand(true) + : BuiltPathsCommand(true) { addFlag({ .longName = "from", @@ -75,16 +75,22 @@ struct CmdCopy : RealisedPathsCommand if (srcUri.empty() && dstUri.empty()) throw UsageError("you must pass '--from' and/or '--to'"); - RealisedPathsCommand::run(store); + BuiltPathsCommand::run(store); } - void run(ref srcStore, std::vector paths) override + void run(ref srcStore, BuiltPaths paths) override { ref dstStore = dstUri.empty() ? openStore() : openStore(dstUri); + RealisedPath::Set stuffToCopy; + + for (auto & builtPath : paths) { + auto theseRealisations = builtPath.toRealisedPaths(*srcStore); + stuffToCopy.insert(theseRealisations.begin(), theseRealisations.end()); + } + copyPaths( - srcStore, dstStore, RealisedPath::Set(paths.begin(), paths.end()), - NoRepair, checkSigs, substitute); + srcStore, dstStore, stuffToCopy, NoRepair, checkSigs, substitute); } }; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index d0b140570..d77ff52d7 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -3,6 +3,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include "derivations.hh" #include "affinity.hh" #include "progress-bar.hh" @@ -143,26 +144,34 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) /* Rehash and write the derivation. FIXME: would be nice to use 'buildDerivation', but that's privileged. */ drv.name += "-env"; - for (auto & output : drv.outputs) { - output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } }; - drv.env[output.first] = ""; - } drv.inputSrcs.insert(std::move(getEnvShPath)); - Hash h = std::get<0>(hashDerivationModulo(*store, drv, true)); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + for (auto & output : drv.outputs) { + output.second = { + .output = DerivationOutputDeferred{}, + }; + drv.env[output.first] = hashPlaceholder(output.first); + } + } else { + for (auto & output : drv.outputs) { + output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } }; + drv.env[output.first] = ""; + } + Hash h = std::get<0>(hashDerivationModulo(*store, drv, true)); - for (auto & output : drv.outputs) { - auto outPath = store->makeOutputPath(output.first, h, drv.name); - output.second = { .output = DerivationOutputInputAddressed { .path = outPath } }; - drv.env[output.first] = store->printStorePath(outPath); + for (auto & output : drv.outputs) { + auto outPath = store->makeOutputPath(output.first, h, drv.name); + output.second = { .output = DerivationOutputInputAddressed { .path = outPath } }; + drv.env[output.first] = store->printStorePath(outPath); + } } auto shellDrvPath = writeDerivation(*store, drv); /* Build the derivation. */ - store->buildPaths({{shellDrvPath}}); + store->buildPaths({DerivedPath::Built{shellDrvPath}}); - for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) { - auto & [_1, optPath] = outputAndOptPath; + for (auto & [_0, optPath] : store->queryPartialDerivationOutputMap(shellDrvPath)) { assert(optPath); auto & outPath = *optPath; assert(store->isValidPath(outPath)); @@ -184,6 +193,7 @@ struct Common : InstallableCommand, MixProfile "NIX_BUILD_TOP", "NIX_ENFORCE_PURITY", "NIX_LOG_FD", + "NIX_REMOTE", "PPID", "PWD", "SHELLOPTS", @@ -264,9 +274,9 @@ struct Common : InstallableCommand, MixProfile for (auto & [installable_, dir_] : redirects) { auto dir = absPath(dir_); auto installable = parseInstallable(store, installable_); - auto buildable = installable->toBuildable(); - auto doRedirect = [&](const StorePath & path) - { + auto builtPaths = toStorePaths( + store, Realise::Nothing, OperateOn::Output, {installable}); + for (auto & path: builtPaths) { auto from = store->printStorePath(path); if (script.find(from) == std::string::npos) warn("'%s' (path '%s') is not used by this build environment", installable->what(), from); @@ -274,16 +284,7 @@ struct Common : InstallableCommand, MixProfile printInfo("redirecting '%s' to '%s'", from, dir); rewrites.insert({from, dir}); } - }; - std::visit(overloaded { - [&](const BuildableOpaque & bo) { - doRedirect(bo.path); - }, - [&](const BuildableFromDrv & bfd) { - for (auto & [outputName, path] : bfd.outputs) - if (path) doRedirect(*path); - }, - }, buildable); + } } return rewriteStrings(script, rewrites); @@ -403,7 +404,7 @@ struct CmdDevelop : Common, MixEnvironment if (verbosity >= lvlDebug) script += "set -x\n"; - script += fmt("rm -f '%s'\n", rcFilePath); + script += fmt("command rm -f '%s'\n", rcFilePath); if (phase) { if (!command.empty()) @@ -422,7 +423,7 @@ struct CmdDevelop : Common, MixEnvironment } else { - script += "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\n"; + script = "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\n" + script; if (developSettings.bashPrompt != "") script += fmt("[ -n \"$PS1\" ] && PS1=%s;\n", shellEscape(developSettings.bashPrompt)); if (developSettings.bashPromptSuffix != "") @@ -461,8 +462,7 @@ struct CmdDevelop : Common, MixEnvironment auto args = phase || !command.empty() ? Strings{std::string(baseNameOf(shell)), rcFilePath} : Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; - restoreAffinity(); - restoreSignals(); + restoreProcessContext(); execvp(shell.c_str(), stringsToCharPtrs(args).data()); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 6472dd27a..b26417b18 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -42,7 +42,8 @@ struct CmdEdit : InstallableCommand auto args = editorFor(pos); - restoreSignals(); + restoreProcessContext(); + execvp(args.front().c_str(), stringsToCharPtrs(args).data()); std::string command; diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md index dc079ba0c..8ef932954 100644 --- a/src/nix/flake-check.md +++ b/src/nix/flake-check.md @@ -22,9 +22,13 @@ This command verifies that the flake specified by flake reference that the derivations specified by the flake's `checks` output can be built successfully. +If the `keep-going` option is set to `true`, Nix will keep evaluating as much +as it can and report the errors as it encounters them. Otherwise it will stop +at the first error. + # Evaluation checks -This following flake output attributes must be derivations: +The following flake output attributes must be derivations: * `checks.`*system*`.`*name* * `defaultPackage.`*system*` diff --git a/src/nix/flake-init.md b/src/nix/flake-init.md index c66154ad5..890038016 100644 --- a/src/nix/flake-init.md +++ b/src/nix/flake-init.md @@ -24,7 +24,7 @@ R""( This command creates a flake in the current directory by copying the files of a template. It will not overwrite existing files. The default -template is `templates#defaultTemplate`, but this can be overriden +template is `templates#defaultTemplate`, but this can be overridden using `-t`. # Template definitions diff --git a/src/nix/flake-list-inputs.md b/src/nix/flake-list-inputs.md deleted file mode 100644 index 250e13be0..000000000 --- a/src/nix/flake-list-inputs.md +++ /dev/null @@ -1,23 +0,0 @@ -R""( - -# Examples - -* Show the inputs of the `hydra` flake: - - ```console - # nix flake list-inputs github:NixOS/hydra - github:NixOS/hydra/bde8d81876dfc02143e5070e42c78d8f0d83d6f7 - ├───nix: github:NixOS/nix/79aa7d95183cbe6c0d786965f0dbff414fd1aa67 - │ ├───lowdown-src: github:kristapsdz/lowdown/1705b4a26fbf065d9574dce47a94e8c7c79e052f - │ └───nixpkgs: github:NixOS/nixpkgs/ad0d20345219790533ebe06571f82ed6b034db31 - └───nixpkgs follows input 'nix/nixpkgs' - ``` - -# Description - -This command shows the inputs of the flake specified by the flake -referenced *flake-url*. Since it prints the locked inputs that result -from generating or updating the lock file, this command essentially -displays the contents of the flake's lock file in human-readable form. - -)"" diff --git a/src/nix/flake-info.md b/src/nix/flake-metadata.md similarity index 75% rename from src/nix/flake-info.md rename to src/nix/flake-metadata.md index fda3171db..5a009409b 100644 --- a/src/nix/flake-info.md +++ b/src/nix/flake-metadata.md @@ -5,19 +5,24 @@ R""( * Show what `nixpkgs` resolves to: ```console - # nix flake info nixpkgs - Resolved URL: github:NixOS/nixpkgs - Locked URL: github:NixOS/nixpkgs/b67ba0bfcc714453cdeb8d713e35751eb8b4c8f4 - Description: A collection of packages for the Nix package manager - Path: /nix/store/23qapccs6cfmwwrlq8kr41vz5vdmns3r-source - Revision: b67ba0bfcc714453cdeb8d713e35751eb8b4c8f4 - Last modified: 2020-12-23 12:36:12 + # nix flake metadata nixpkgs + Resolved URL: github:edolstra/dwarffs + Locked URL: github:edolstra/dwarffs/f691e2c991e75edb22836f1dbe632c40324215c5 + Description: A filesystem that fetches DWARF debug info from the Internet on demand + Path: /nix/store/769s05vjydmc2lcf6b02az28wsa9ixh1-source + Revision: f691e2c991e75edb22836f1dbe632c40324215c5 + Last modified: 2021-01-21 15:41:26 + Inputs: + ├───nix: github:NixOS/nix/6254b1f5d298ff73127d7b0f0da48f142bdc753c + │ ├───lowdown-src: github:kristapsdz/lowdown/1705b4a26fbf065d9574dce47a94e8c7c79e052f + │ └───nixpkgs: github:NixOS/nixpkgs/ad0d20345219790533ebe06571f82ed6b034db31 + └───nixpkgs follows input 'nix/nixpkgs' ``` * Show information about `dwarffs` in JSON format: ```console - # nix flake info dwarffs --json | jq . + # nix flake metadata dwarffs --json | jq . { "description": "A filesystem that fetches DWARF debug info from the Internet on demand", "lastModified": 1597153508, @@ -29,6 +34,7 @@ R""( "rev": "d181d714fd36eb06f4992a1997cd5601e26db8f5", "type": "github" }, + "locks": { ... }, "original": { "id": "dwarffs", "type": "indirect" @@ -75,6 +81,9 @@ data. This includes: time of the commit of the locked flake; for tarball flakes, it's the most recent timestamp of any file inside the tarball. +* `Inputs`: The flake inputs with their corresponding lock file + entries. + With `--json`, the output is a JSON object with the following fields: * `original` and `originalUrl`: The flake reference specified by the @@ -96,4 +105,6 @@ With `--json`, the output is a JSON object with the following fields: * `lastModified`: See `Last modified` above. +* `locks`: The contents of `flake.lock`. + )"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 2f0c468a8..64fcfc000 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -7,6 +7,7 @@ #include "get-drvs.hh" #include "store-api.hh" #include "derivations.hh" +#include "path-with-outputs.hh" #include "attr-path.hh" #include "fetchers.hh" #include "registry.hh" @@ -43,12 +44,6 @@ public: return parseFlakeRef(flakeUrl, absPath(".")); //FIXME } - Flake getFlake() - { - auto evalState = getEvalState(); - return flake::getFlake(*evalState, getFlakeRef(), lockFlags.useRegistries); - } - LockedFlake lockFlake() { return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags); @@ -60,43 +55,6 @@ public: } }; -static void printFlakeInfo(const Store & store, const Flake & flake) -{ - logger->cout("Resolved URL: %s", flake.resolvedRef.to_string()); - logger->cout("Locked URL: %s", flake.lockedRef.to_string()); - if (flake.description) - logger->cout("Description: %s", *flake.description); - logger->cout("Path: %s", store.printStorePath(flake.sourceInfo->storePath)); - if (auto rev = flake.lockedRef.input.getRev()) - logger->cout("Revision: %s", rev->to_string(Base16, false)); - if (auto revCount = flake.lockedRef.input.getRevCount()) - logger->cout("Revisions: %s", *revCount); - if (auto lastModified = flake.lockedRef.input.getLastModified()) - logger->cout("Last modified: %s", - std::put_time(std::localtime(&*lastModified), "%F %T")); -} - -static nlohmann::json flakeToJSON(const Store & store, const Flake & flake) -{ - nlohmann::json j; - if (flake.description) - j["description"] = *flake.description; - j["originalUrl"] = flake.originalRef.to_string(); - j["original"] = fetchers::attrsToJSON(flake.originalRef.toAttrs()); - j["resolvedUrl"] = flake.resolvedRef.to_string(); - j["resolved"] = fetchers::attrsToJSON(flake.resolvedRef.toAttrs()); - j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl - j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs()); - if (auto rev = flake.lockedRef.input.getRev()) - j["revision"] = rev->to_string(Base16, false); - if (auto revCount = flake.lockedRef.input.getRevCount()) - j["revCount"] = *revCount; - if (auto lastModified = flake.lockedRef.input.getLastModified()) - j["lastModified"] = *lastModified; - j["path"] = store.printStorePath(flake.sourceInfo->storePath); - return j; -} - struct CmdFlakeUpdate : FlakeCommand { std::string description() override @@ -110,6 +68,7 @@ struct CmdFlakeUpdate : FlakeCommand removeFlag("recreate-lock-file"); removeFlag("update-input"); removeFlag("no-update-lock-file"); + removeFlag("no-write-lock-file"); } std::string doc() override @@ -124,6 +83,7 @@ struct CmdFlakeUpdate : FlakeCommand settings.tarballTtl = 0; lockFlags.recreateLockFile = true; + lockFlags.writeLockFile = true; lockFlake(); } @@ -136,6 +96,12 @@ struct CmdFlakeLock : FlakeCommand return "create missing lock file entries"; } + CmdFlakeLock() + { + /* Remove flags that don't make sense. */ + removeFlag("no-write-lock-file"); + } + std::string doc() override { return @@ -147,6 +113,8 @@ struct CmdFlakeLock : FlakeCommand { settings.tarballTtl = 0; + lockFlags.writeLockFile = true; + lockFlake(); } }; @@ -165,54 +133,72 @@ static void enumerateOutputs(EvalState & state, Value & vFlake, callback(attr.name, *attr.value, *attr.pos); } -struct CmdFlakeInfo : FlakeCommand, MixJSON +struct CmdFlakeMetadata : FlakeCommand, MixJSON { std::string description() override { - return "list info about a given flake"; + return "show flake metadata"; } std::string doc() override { return - #include "flake-info.md" + #include "flake-metadata.md" ; } void run(nix::ref store) override { - auto flake = getFlake(); + auto lockedFlake = lockFlake(); + auto & flake = lockedFlake.flake; if (json) { - auto json = flakeToJSON(*store, flake); - logger->cout("%s", json.dump()); - } else - printFlakeInfo(*store, flake); - } -}; + nlohmann::json j; + if (flake.description) + j["description"] = *flake.description; + j["originalUrl"] = flake.originalRef.to_string(); + j["original"] = fetchers::attrsToJSON(flake.originalRef.toAttrs()); + j["resolvedUrl"] = flake.resolvedRef.to_string(); + j["resolved"] = fetchers::attrsToJSON(flake.resolvedRef.toAttrs()); + j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl + j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs()); + if (auto rev = flake.lockedRef.input.getRev()) + j["revision"] = rev->to_string(Base16, false); + if (auto revCount = flake.lockedRef.input.getRevCount()) + j["revCount"] = *revCount; + if (auto lastModified = flake.lockedRef.input.getLastModified()) + j["lastModified"] = *lastModified; + j["path"] = store->printStorePath(flake.sourceInfo->storePath); + j["locks"] = lockedFlake.lockFile.toJSON(); + logger->cout("%s", j.dump()); + } else { + logger->cout( + ANSI_BOLD "Resolved URL:" ANSI_NORMAL " %s", + flake.resolvedRef.to_string()); + logger->cout( + ANSI_BOLD "Locked URL:" ANSI_NORMAL " %s", + flake.lockedRef.to_string()); + if (flake.description) + logger->cout( + ANSI_BOLD "Description:" ANSI_NORMAL " %s", + *flake.description); + logger->cout( + ANSI_BOLD "Path:" ANSI_NORMAL " %s", + store->printStorePath(flake.sourceInfo->storePath)); + if (auto rev = flake.lockedRef.input.getRev()) + logger->cout( + ANSI_BOLD "Revision:" ANSI_NORMAL " %s", + rev->to_string(Base16, false)); + if (auto revCount = flake.lockedRef.input.getRevCount()) + logger->cout( + ANSI_BOLD "Revisions:" ANSI_NORMAL " %s", + *revCount); + if (auto lastModified = flake.lockedRef.input.getLastModified()) + logger->cout( + ANSI_BOLD "Last modified:" ANSI_NORMAL " %s", + std::put_time(std::localtime(&*lastModified), "%F %T")); -struct CmdFlakeListInputs : FlakeCommand, MixJSON -{ - std::string description() override - { - return "list flake inputs"; - } - - std::string doc() override - { - return - #include "flake-list-inputs.md" - ; - } - - void run(nix::ref store) override - { - auto flake = lockFlake(); - - if (json) - logger->cout("%s", flake.lockFile.toJSON()); - else { - logger->cout("%s", flake.flake.lockedRef); + logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); std::unordered_set> visited; @@ -226,7 +212,7 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON if (auto lockedNode = std::get_if<0>(&input.second)) { logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", prefix + (last ? treeLast : treeConn), input.first, - *lockedNode ? (*lockedNode)->lockedRef : flake.flake.lockedRef); + *lockedNode ? (*lockedNode)->lockedRef : flake.lockedRef); bool firstVisit = visited.insert(*lockedNode).second; @@ -239,12 +225,21 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON } }; - visited.insert(flake.lockFile.root); - recurse(*flake.lockFile.root, ""); + visited.insert(lockedFlake.lockFile.root); + recurse(*lockedFlake.lockFile.root, ""); } } }; +struct CmdFlakeInfo : CmdFlakeMetadata +{ + void run(nix::ref store) override + { + warn("'nix flake info' is a deprecated alias for 'nix flake metadata'"); + CmdFlakeMetadata::run(store); + } +}; + struct CmdFlakeCheck : FlakeCommand { bool build = true; @@ -277,28 +272,43 @@ struct CmdFlakeCheck : FlakeCommand auto state = getEvalState(); auto flake = lockFlake(); + bool hasErrors = false; + auto reportError = [&](const Error & e) { + try { + throw e; + } catch (Error & e) { + if (settings.keepGoing) { + ignoreException(); + hasErrors = true; + } + else + throw; + } + }; + // FIXME: rewrite to use EvalCache. auto checkSystemName = [&](const std::string & system, const Pos & pos) { // FIXME: what's the format of "system"? if (system.find('-') == std::string::npos) - throw Error("'%s' is not a valid system type, at %s", system, pos); + reportError(Error("'%s' is not a valid system type, at %s", system, pos)); }; - auto checkDerivation = [&](const std::string & attrPath, Value & v, const Pos & pos) { + auto checkDerivation = [&](const std::string & attrPath, Value & v, const Pos & pos) -> std::optional { try { auto drvInfo = getDerivation(*state, v, false); if (!drvInfo) throw Error("flake attribute '%s' is not a derivation", attrPath); // FIXME: check meta attributes - return store->parseStorePath(drvInfo->queryDrvPath()); + return std::make_optional(store->parseStorePath(drvInfo->queryDrvPath())); } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the derivation '%s'", attrPath)); - throw; + reportError(e); } + return std::nullopt; }; - std::vector drvPaths; + std::vector drvPaths; auto checkApp = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { @@ -312,7 +322,7 @@ struct CmdFlakeCheck : FlakeCommand #endif } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the app definition '%s'", attrPath)); - throw; + reportError(e); } }; @@ -328,7 +338,7 @@ struct CmdFlakeCheck : FlakeCommand // evaluate the overlay. } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the overlay '%s'", attrPath)); - throw; + reportError(e); } }; @@ -352,7 +362,7 @@ struct CmdFlakeCheck : FlakeCommand // check the module. } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the NixOS module '%s'", attrPath)); - throw; + reportError(e); } }; @@ -374,7 +384,7 @@ struct CmdFlakeCheck : FlakeCommand } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the Hydra jobset '%s'", attrPath)); - throw; + reportError(e); } }; @@ -389,7 +399,7 @@ struct CmdFlakeCheck : FlakeCommand throw Error("attribute 'config.system.build.toplevel' is not a derivation"); } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the NixOS configuration '%s'", attrPath)); - throw; + reportError(e); } }; @@ -423,7 +433,7 @@ struct CmdFlakeCheck : FlakeCommand } } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); - throw; + reportError(e); } }; @@ -438,7 +448,7 @@ struct CmdFlakeCheck : FlakeCommand throw Error("bundler must take formal arguments 'program' and 'system'"); } catch (Error & e) { e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); - throw; + reportError(e); } }; @@ -466,8 +476,8 @@ struct CmdFlakeCheck : FlakeCommand auto drvPath = checkDerivation( fmt("%s.%s.%s", name, attr.name, attr2.name), *attr2.value, *attr2.pos); - if ((std::string) attr.name == settings.thisSystem.get()) - drvPaths.push_back({drvPath}); + if (drvPath && (std::string) attr.name == settings.thisSystem.get()) + drvPaths.push_back(DerivedPath::Built{*drvPath}); } } } @@ -579,7 +589,7 @@ struct CmdFlakeCheck : FlakeCommand } catch (Error & e) { e.addTrace(pos, hintfmt("while checking flake output '%s'", name)); - throw; + reportError(e); } }); } @@ -588,6 +598,8 @@ struct CmdFlakeCheck : FlakeCommand Activity act(*logger, lvlInfo, actUnknown, "running flake checks"); store->buildPaths(drvPaths); } + if (hasErrors) + throw Error("Some errors were encountered during the evaluation"); } }; @@ -1038,8 +1050,8 @@ struct CmdFlake : NixMultiCommand : MultiCommand({ {"update", []() { return make_ref(); }}, {"lock", []() { return make_ref(); }}, + {"metadata", []() { return make_ref(); }}, {"info", []() { return make_ref(); }}, - {"list-inputs", []() { return make_ref(); }}, {"check", []() { return make_ref(); }}, {"init", []() { return make_ref(); }}, {"new", []() { return make_ref(); }}, diff --git a/src/nix/flake.md b/src/nix/flake.md index 440c45dd1..3d273100b 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -70,7 +70,7 @@ Here are some examples of flake references in their URL-like representation: * `/home/alice/src/patchelf`: A flake in some other directory. * `nixpkgs`: The `nixpkgs` entry in the flake registry. * `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: The `nixpkgs` - entry in the flake registry, with its Git revision overriden to a + entry in the flake registry, with its Git revision overridden to a specific value. * `github:NixOS/nixpkgs`: The `master` branch of the `NixOS/nixpkgs` repository on GitHub. @@ -186,8 +186,8 @@ Currently the `type` attribute can be one of the following: attribute `url`. In URL form, the schema must be `http://`, `https://` or `file://` - URLs and the extension must be `.zip`, `.tar`, `.tar.gz`, `.tar.xz` - or `.tar.bz2`. + URLs and the extension must be `.zip`, `.tar`, `.tar.gz`, `.tar.xz`, + `.tar.bz2` or `.tar.zst`. * `github`: A more efficient way to fetch repositories from GitHub. The following attributes are required: @@ -377,7 +377,7 @@ outputs = { self, nixpkgs, grcov }: { }; ``` -Transitive inputs can be overriden from a `flake.nix` file. For +Transitive inputs can be overridden from a `flake.nix` file. For example, the following overrides the `nixpkgs` input of the `nixops` input: @@ -395,7 +395,7 @@ the `nixpkgs` input of the top-level flake to be equal to the `nixpkgs` input of the `dwarffs` input of the top-level flake: ```nix -inputs.nixops.follows = "dwarffs/nixpkgs"; +inputs.nixpkgs.follows = "dwarffs/nixpkgs"; ``` The value of the `follows` attribute is a `/`-separated sequence of diff --git a/src/nix/log.cc b/src/nix/log.cc index 67d3742d6..962c47525 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -30,18 +30,18 @@ struct CmdLog : InstallableCommand subs.push_front(store); - auto b = installable->toBuildable(); + auto b = installable->toDerivedPath(); RunPager pager; for (auto & sub : subs) { auto log = std::visit(overloaded { - [&](BuildableOpaque bo) { + [&](DerivedPath::Opaque bo) { return sub->getBuildLog(bo.path); }, - [&](BuildableFromDrv bfd) { + [&](DerivedPath::Built bfd) { return sub->getBuildLog(bfd.drvPath); }, - }, b); + }, b.raw()); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri()); diff --git a/src/nix/main.cc b/src/nix/main.cc index 06e221682..008482be3 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -17,10 +17,6 @@ #include #include -#if __linux__ -#include -#endif - #include extern std::string chrootHelperName; @@ -309,13 +305,13 @@ void mainWrapped(int argc, char * * argv) if (!args.useNet) { // FIXME: should check for command line overrides only. - if (!settings.useSubstitutes.overriden) + if (!settings.useSubstitutes.overridden) settings.useSubstitutes = false; - if (!settings.tarballTtl.overriden) + if (!settings.tarballTtl.overridden) settings.tarballTtl = std::numeric_limits::max(); - if (!fileTransferSettings.tries.overriden) + if (!fileTransferSettings.tries.overridden) fileTransferSettings.tries = 0; - if (!fileTransferSettings.connectTimeout.overriden) + if (!fileTransferSettings.connectTimeout.overridden) fileTransferSettings.connectTimeout = 1; } @@ -335,14 +331,7 @@ int main(int argc, char * * argv) { // Increase the default stack size for the evaluator and for // libstdc++'s std::regex. - #if __linux__ - rlim_t stackSize = 64 * 1024 * 1024; - struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { - limit.rlim_cur = stackSize; - setrlimit(RLIMIT_STACK, &limit); - } - #endif + nix::setStackSize(64 * 1024 * 1024); return nix::handleExceptions(argv[0], [&]() { nix::mainWrapped(argc, argv); diff --git a/src/nix/profile-upgrade.md b/src/nix/profile-upgrade.md index 2bd5d256d..e06e74abe 100644 --- a/src/nix/profile-upgrade.md +++ b/src/nix/profile-upgrade.md @@ -18,7 +18,7 @@ R""( * Upgrade a specific profile element by number: ```console - # nix profile info + # nix profile list 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify … # nix profile upgrade 0 diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 4d275f577..511771f89 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -233,7 +233,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile { ProfileManifest manifest(*getEvalState(), *profile); - std::vector pathsToBuild; + std::vector pathsToBuild; for (auto & installable : installables) { if (auto installable2 = std::dynamic_pointer_cast(installable)) { @@ -249,7 +249,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile attrPath, }; - pathsToBuild.push_back({drv.drvPath, StringSet{drv.outputName}}); + pathsToBuild.push_back(DerivedPath::Built{drv.drvPath, StringSet{drv.outputName}}); manifest.elements.emplace_back(std::move(element)); } else { @@ -259,17 +259,20 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile ProfileElement element; std::visit(overloaded { - [&](BuildableOpaque bo) { - pathsToBuild.push_back({bo.path, {}}); + [&](BuiltPath::Opaque bo) { + pathsToBuild.push_back(bo); element.storePaths.insert(bo.path); }, - [&](BuildableFromDrv bfd) { + [&](BuiltPath::Built bfd) { + // TODO: Why are we querying if we know the output + // names already? Is it just to figure out what the + // default one is? for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) { - pathsToBuild.push_back({bfd.drvPath, {output.first}}); + pathsToBuild.push_back(DerivedPath::Built{bfd.drvPath, {output.first}}); element.storePaths.insert(output.second); } }, - }, buildable); + }, buildable.raw()); manifest.elements.emplace_back(std::move(element)); } @@ -388,7 +391,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf auto matchers = getMatchers(store); // FIXME: code duplication - std::vector pathsToBuild; + std::vector pathsToBuild; for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); @@ -423,7 +426,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf attrPath, }; - pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME + pathsToBuild.push_back(DerivedPath::Built{drv.drvPath, {drv.outputName}}); } } diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc new file mode 100644 index 000000000..d59e594df --- /dev/null +++ b/src/nix/realisation.cc @@ -0,0 +1,85 @@ +#include "command.hh" +#include "common-args.hh" + +#include + +using namespace nix; + +struct CmdRealisation : virtual NixMultiCommand +{ + CmdRealisation() : MultiCommand(RegisterCommand::getCommandsFor({"realisation"})) + { } + + std::string description() override + { + return "manipulate a Nix realisation"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix realisation' requires a sub-command."); + command->second->prepare(); + command->second->run(); + } +}; + +static auto rCmdRealisation = registerCommand("realisation"); + +struct CmdRealisationInfo : BuiltPathsCommand, MixJSON +{ + std::string description() override + { + return "query information about one or several realisations"; + } + + std::string doc() override + { + return + #include "realisation/info.md" + ; + } + + Category category() override { return catSecondary; } + + void run(ref store, BuiltPaths paths) override + { + settings.requireExperimentalFeature("ca-derivations"); + RealisedPath::Set realisations; + + for (auto & builtPath : paths) { + auto theseRealisations = builtPath.toRealisedPaths(*store); + realisations.insert(theseRealisations.begin(), theseRealisations.end()); + } + + if (json) { + nlohmann::json res = nlohmann::json::array(); + for (auto & path : realisations) { + nlohmann::json currentPath; + if (auto realisation = std::get_if(&path.raw)) + currentPath = realisation->toJSON(); + else + currentPath["opaquePath"] = store->printStorePath(path.path()); + + res.push_back(currentPath); + } + std::cout << res.dump(); + } + else { + for (auto & path : realisations) { + if (auto realisation = std::get_if(&path.raw)) { + std::cout << + realisation->id.to_string() << " " << + store->printStorePath(realisation->outPath); + } else + std::cout << store->printStorePath(path.path()); + + std::cout << std::endl; + } + } + } +}; + +static auto rCmdRealisationInfo = registerCommand2({"realisation", "info"}); diff --git a/src/nix/realisation/info.md b/src/nix/realisation/info.md new file mode 100644 index 000000000..852240f44 --- /dev/null +++ b/src/nix/realisation/info.md @@ -0,0 +1,15 @@ +R"MdBoundary( +# Description + +Display some informations about the given realisation + +# Examples + +Show some information about the realisation of the `hello` package: + +```console +$ nix realisation info nixpkgs#hello --json +[{"id":"sha256:3d382378a00588e064ee30be96dd0fa7e7df7cf3fbcace85a0e7b7dada1eef25!out","outPath":"fd3m7xawvrqcg98kgz5hc2vk3x9q0lh7-hello"}] +``` + +)MdBoundary" diff --git a/src/nix/repl.cc b/src/nix/repl.cc index bce8d31dc..eed79c332 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -343,24 +343,6 @@ StringSet NixRepl::completePrefix(string prefix) } -static int runProgram(const string & program, const Strings & args) -{ - Strings args2(args); - args2.push_front(program); - - Pid pid; - pid = fork(); - if (pid == -1) throw SysError("forking"); - if (pid == 0) { - restoreAffinity(); - execvp(program.c_str(), stringsToCharPtrs(args2).data()); - _exit(1); - } - - return pid.wait(); -} - - bool isVarName(const string & s) { if (s.size() == 0) return false; @@ -462,7 +444,7 @@ bool NixRepl::processLine(string line) auto args = editorFor(pos); auto editor = args.front(); args.pop_front(); - runProgram(editor, args); + runProgram(editor, true, args); // Reload right after exiting the editor state->resetFileCache(); @@ -481,7 +463,7 @@ bool NixRepl::processLine(string line) state->callFunction(f, v, result, Pos()); StorePath drvPath = getDerivationPath(result); - runProgram(settings.nixBinDir + "/nix-shell", Strings{state->store->printStorePath(drvPath)}); + runProgram(settings.nixBinDir + "/nix-shell", true, {state->store->printStorePath(drvPath)}); } else if (command == ":b" || command == ":i" || command == ":s") { @@ -494,16 +476,18 @@ bool NixRepl::processLine(string line) /* We could do the build in this process using buildPaths(), but doing it in a child makes it easier to recover from problems / SIGINT. */ - if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPathRaw}) == 0) { + try { + runProgram(settings.nixBinDir + "/nix", true, {"build", "--no-link", drvPathRaw}); auto drv = state->store->readDerivation(drvPath); std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; for (auto & i : drv.outputsAndOptPaths(*state->store)) std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(*i.second.second)); + } catch (ExecError &) { } } else if (command == ":i") { - runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPathRaw}); + runProgram(settings.nixBinDir + "/nix-env", true, {"-i", drvPathRaw}); } else { - runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPathRaw}); + runProgram(settings.nixBinDir + "/nix-shell", true, {drvPathRaw}); } } diff --git a/src/nix/run.cc b/src/nix/run.cc index ec9388234..c0ba05a3e 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -31,9 +31,7 @@ struct RunCommon : virtual Command { stopProgressBar(); - restoreSignals(); - - restoreAffinity(); + restoreProcessContext(); /* If this is a diverted store (i.e. its "logical" location (typically /nix/store) differs from its "physical" location @@ -45,8 +43,8 @@ struct RunCommon : virtual Command helper program (chrootHelper() below) to do the work. */ auto store2 = store.dynamic_pointer_cast(); - if (store2 && store->storeDir != store2->realStoreDir) { - Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, program }; + if (store2 && store->storeDir != store2->getRealStoreDir()) { + Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program }; for (auto & arg : args) helperArgs.push_back(arg); execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); @@ -180,9 +178,7 @@ struct CmdRun : InstallableCommand, RunCommon { auto state = getEvalState(); - auto app = installable->toApp(*state); - - state->store->buildPaths(app.context); + auto app = installable->toApp(*state).resolve(store); Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); diff --git a/src/nix/store-prefetch-file.md b/src/nix/store-prefetch-file.md index 1663b847b..f9fdcbc57 100644 --- a/src/nix/store-prefetch-file.md +++ b/src/nix/store-prefetch-file.md @@ -27,6 +27,6 @@ the resulting store path and the cryptographic hash of the contents of the file. The name component of the store path defaults to the last component of -*url*, but this can be overriden using `--name`. +*url*, but this can be overridden using `--name`. )"" diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 1721c7f16..f5a576064 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -97,15 +97,11 @@ struct CmdVerify : StorePathsCommand if (!noContents) { - std::unique_ptr hashSink; - if (!info->ca) - hashSink = std::make_unique(info->narHash.type); - else - hashSink = std::make_unique(info->narHash.type, std::string(info->path.hashPart())); + auto hashSink = HashSink(info->narHash.type); - store->narFromPath(info->path, *hashSink); + store->narFromPath(info->path, hashSink); - auto hash = hashSink->finish(); + auto hash = hashSink.finish(); if (hash.first != info->narHash) { corrupted++; diff --git a/src/nlohmann/json.hpp b/src/nlohmann/json.hpp new file mode 100644 index 000000000..c9af0bed3 --- /dev/null +++ b/src/nlohmann/json.hpp @@ -0,0 +1,20406 @@ +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ +| | |__ | | | | | | version 3.5.0 +|_____|_____|_____|_|___| https://github.com/nlohmann/json + +Licensed under the MIT License . +SPDX-License-Identifier: MIT +Copyright (c) 2013-2018 Niels Lohmann . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +#ifndef NLOHMANN_JSON_HPP +#define NLOHMANN_JSON_HPP + +#define NLOHMANN_JSON_VERSION_MAJOR 3 +#define NLOHMANN_JSON_VERSION_MINOR 5 +#define NLOHMANN_JSON_VERSION_PATCH 0 + +#include // all_of, find, for_each +#include // assert +#include // and, not, or +#include // nullptr_t, ptrdiff_t, size_t +#include // hash, less +#include // initializer_list +#include // istream, ostream +#include // random_access_iterator_tag +#include // accumulate +#include // string, stoi, to_string +#include // declval, forward, move, pair, swap + +// #include +#ifndef NLOHMANN_JSON_FWD_HPP +#define NLOHMANN_JSON_FWD_HPP + +#include // int64_t, uint64_t +#include // map +#include // allocator +#include // string +#include // vector + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ +/*! +@brief default JSONSerializer template argument + +This serializer ignores the template arguments and uses ADL +([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) +for serialization. +*/ +template +struct adl_serializer; + +template class ObjectType = + std::map, + template class ArrayType = std::vector, + class StringType = std::string, class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template class AllocatorType = std::allocator, + template class JSONSerializer = + adl_serializer> +class basic_json; + +/*! +@brief JSON Pointer + +A JSON pointer defines a string syntax for identifying a specific value +within a JSON document. It can be used with functions `at` and +`operator[]`. Furthermore, JSON pointers are the base for JSON patches. + +@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) + +@since version 2.0.0 +*/ +template +class json_pointer; + +/*! +@brief default JSON class + +This type is the default specialization of the @ref basic_json class which +uses the standard template types. + +@since version 1.0.0 +*/ +using json = basic_json<>; +} // namespace nlohmann + +#endif + +// #include + + +// This file contains all internal macro definitions +// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them + +// exclude unsupported compilers +#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) + #if defined(__clang__) + #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #endif +#endif + +// disable float-equal warnings on GCC/clang +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + +// disable documentation warnings on clang +#if defined(__clang__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdocumentation" +#endif + +// allow for portable deprecation warnings +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #define JSON_DEPRECATED __attribute__((deprecated)) +#elif defined(_MSC_VER) + #define JSON_DEPRECATED __declspec(deprecated) +#else + #define JSON_DEPRECATED +#endif + +// allow to disable exceptions +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) + #define JSON_THROW(exception) throw exception + #define JSON_TRY try + #define JSON_CATCH(exception) catch(exception) + #define JSON_INTERNAL_CATCH(exception) catch(exception) +#else + #define JSON_THROW(exception) std::abort() + #define JSON_TRY if(true) + #define JSON_CATCH(exception) if(false) + #define JSON_INTERNAL_CATCH(exception) if(false) +#endif + +// override exception macros +#if defined(JSON_THROW_USER) + #undef JSON_THROW + #define JSON_THROW JSON_THROW_USER +#endif +#if defined(JSON_TRY_USER) + #undef JSON_TRY + #define JSON_TRY JSON_TRY_USER +#endif +#if defined(JSON_CATCH_USER) + #undef JSON_CATCH + #define JSON_CATCH JSON_CATCH_USER + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_CATCH_USER +#endif +#if defined(JSON_INTERNAL_CATCH_USER) + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER +#endif + +// manual branch prediction +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #define JSON_LIKELY(x) __builtin_expect(!!(x), 1) + #define JSON_UNLIKELY(x) __builtin_expect(!!(x), 0) +#else + #define JSON_LIKELY(x) x + #define JSON_UNLIKELY(x) x +#endif + +// C++ language standard detection +#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 +#elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) + #define JSON_HAS_CPP_14 +#endif + +/*! +@brief macro to briefly define a mapping between an enum and JSON +@def NLOHMANN_JSON_SERIALIZE_ENUM +@since version 3.4.0 +*/ +#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ + template \ + inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [e](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.first == e; \ + }); \ + j = ((it != std::end(m)) ? it : std::begin(m))->second; \ + } \ + template \ + inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [j](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.second == j; \ + }); \ + e = ((it != std::end(m)) ? it : std::begin(m))->first; \ + } + +// Ugly macros to avoid uglier copy-paste when specializing basic_json. They +// may be removed in the future once the class is split. + +#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ + template class ObjectType, \ + template class ArrayType, \ + class StringType, class BooleanType, class NumberIntegerType, \ + class NumberUnsignedType, class NumberFloatType, \ + template class AllocatorType, \ + template class JSONSerializer> + +#define NLOHMANN_BASIC_JSON_TPL \ + basic_json + +// #include + + +#include // not +#include // size_t +#include // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type + +namespace nlohmann +{ +namespace detail +{ +// alias templates to reduce boilerplate +template +using enable_if_t = typename std::enable_if::type; + +template +using uncvref_t = typename std::remove_cv::type>::type; + +// implementation of C++14 index_sequence and affiliates +// source: https://stackoverflow.com/a/32223343 +template +struct index_sequence +{ + using type = index_sequence; + using value_type = std::size_t; + static constexpr std::size_t size() noexcept + { + return sizeof...(Ints); + } +}; + +template +struct merge_and_renumber; + +template +struct merge_and_renumber, index_sequence> + : index_sequence < I1..., (sizeof...(I1) + I2)... > {}; + +template +struct make_index_sequence + : merge_and_renumber < typename make_index_sequence < N / 2 >::type, + typename make_index_sequence < N - N / 2 >::type > {}; + +template<> struct make_index_sequence<0> : index_sequence<> {}; +template<> struct make_index_sequence<1> : index_sequence<0> {}; + +template +using index_sequence_for = make_index_sequence; + +// dispatch utility (taken from ranges-v3) +template struct priority_tag : priority_tag < N - 1 > {}; +template<> struct priority_tag<0> {}; + +// taken from ranges-v3 +template +struct static_const +{ + static constexpr T value{}; +}; + +template +constexpr T static_const::value; +} // namespace detail +} // namespace nlohmann + +// #include + + +#include // not +#include // numeric_limits +#include // false_type, is_constructible, is_integral, is_same, true_type +#include // declval + +// #include + +// #include + + +#include // random_access_iterator_tag + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template struct make_void +{ + using type = void; +}; +template using void_t = typename make_void::type; +} // namespace detail +} // namespace nlohmann + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template +struct iterator_types {}; + +template +struct iterator_types < + It, + void_t> +{ + using difference_type = typename It::difference_type; + using value_type = typename It::value_type; + using pointer = typename It::pointer; + using reference = typename It::reference; + using iterator_category = typename It::iterator_category; +}; + +// This is required as some compilers implement std::iterator_traits in a way that +// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. +template +struct iterator_traits +{ +}; + +template +struct iterator_traits < T, enable_if_t < !std::is_pointer::value >> + : iterator_types +{ +}; + +template +struct iterator_traits::value>> +{ + using iterator_category = std::random_access_iterator_tag; + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = T*; + using reference = T&; +}; +} +} + +// #include + +// #include + + +#include + +// #include + + +// http://en.cppreference.com/w/cpp/experimental/is_detected +namespace nlohmann +{ +namespace detail +{ +struct nonesuch +{ + nonesuch() = delete; + ~nonesuch() = delete; + nonesuch(nonesuch const&) = delete; + void operator=(nonesuch const&) = delete; +}; + +template class Op, + class... Args> +struct detector +{ + using value_t = std::false_type; + using type = Default; +}; + +template class Op, class... Args> +struct detector>, Op, Args...> +{ + using value_t = std::true_type; + using type = Op; +}; + +template