Merge pull request #9535 from tfc/package-nix

Simplify flake.nix by creating generic callPackage-able package.nix
This commit is contained in:
John Ericson 2023-12-13 16:22:03 -05:00 committed by GitHub
commit e6515bd47b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 683 additions and 545 deletions

View file

@ -61,7 +61,7 @@ include mk/lib.mk
# by the library. Rules are not "lazy" like variables, unfortunately.
ifeq ($(ENABLE_BUILD), yes)
$(eval $(call include-sub-makefile, doc/manual/local.mk))
$(eval $(call include-sub-makefile, doc/internal-api/local.mk))
endif
$(eval $(call include-sub-makefile, doc/internal-api/local.mk))
GLOBAL_CXXFLAGS += -g -Wall -include $(buildprefix)config.h -std=c++2a -I src

View file

@ -122,7 +122,6 @@ AC_PATH_PROG(flex, flex, false)
AC_PATH_PROG(bison, bison, false)
AC_PATH_PROG(dot, dot)
AC_PATH_PROG(lsof, lsof, lsof)
NEED_PROG(jq, jq)
AC_SUBST(coreutils, [$(dirname $(type -p cat))])
@ -133,6 +132,30 @@ AC_ARG_WITH(store-dir, AS_HELP_STRING([--with-store-dir=PATH],[path of the Nix s
AC_SUBST(storedir)
# Running the functional tests without building Nix is useful for testing
# different pre-built versions of Nix against each other.
AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]),
ENABLE_BUILD=$enableval, ENABLE_BUILD=yes)
AC_SUBST(ENABLE_BUILD)
# Building without tests is useful for bootstrapping with a smaller footprint
# or running the tests in a separate derivation. Otherwise, we do compile and
# run them.
AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]),
ENABLE_TESTS=$enableval, ENABLE_TESTS=yes)
AC_SUBST(ENABLE_TESTS)
# Building without API docs is the default as Nix' C++ interfaces are internal and unstable.
AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]),
internal_api_docs=$enableval, internal_api_docs=no)
AC_SUBST(internal_api_docs)
AS_IF(
[test "$ENABLE_BUILD" == "yes" || test "$ENABLE_TEST" == "yes"],
[NEED_PROG(jq, jq)])
AS_IF([test "$ENABLE_BUILD" == "yes"],[
# Look for boost, a required dependency.
# Note that AX_BOOST_BASE only exports *CPP* BOOST_CPPFLAGS, no CXX flags,
# and CPPFLAGS are not passed to the C++ compiler automatically.
@ -155,18 +178,6 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then
LDFLAGS="-latomic $LDFLAGS"
fi
# Running the functional tests without building Nix is useful for testing
# different pre-built versions of Nix against each other.
AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]),
ENABLE_BUILD=$enableval, ENABLE_BUILD=yes)
AC_SUBST(ENABLE_BUILD)
# Building without tests is useful for bootstrapping with a smaller footprint
# or running the tests in a separate derivation. Otherwise, we do compile and
# run them.
AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]),
ENABLE_TESTS=$enableval, ENABLE_TESTS=yes)
AC_SUBST(ENABLE_TESTS)
AC_ARG_ENABLE(install-unit-tests, AS_HELP_STRING([--enable-install-unit-tests],[Install the unit tests for running later (default no)]),
INSTALL_UNIT_TESTS=$enableval, INSTALL_UNIT_TESTS=no)
AC_SUBST(INSTALL_UNIT_TESTS)
@ -179,11 +190,6 @@ AC_ARG_WITH(check-lib-dir, AS_HELP_STRING([--with-check-lib-dir=PATH],[path to i
checklibdir=$withval, checklibdir=$libdir)
AC_SUBST(checklibdir)
# Building without API docs is the default as Nix' C++ interfaces are internal and unstable.
AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]),
internal_api_docs=$enableval, internal_api_docs=no)
AC_SUBST(internal_api_docs)
# LTO is currently broken with clang for unknown reasons; ld segfaults in the llvm plugin
AC_ARG_ENABLE(lto, AS_HELP_STRING([--enable-lto],[Enable LTO (only supported with GCC) [default=no]]),
lto=$enableval, lto=no)
@ -310,8 +316,7 @@ if test "$gc" = yes; then
AC_DEFINE(HAVE_BOEHMGC, 1, [Whether to use the Boehm garbage collector.])
fi
if test "$ENABLE_TESTS" = yes; then
AS_IF([test "$ENABLE_TESTS" == "yes"],[
# Look for gtest.
PKG_CHECK_MODULES([GTEST], [gtest_main])
@ -338,12 +343,11 @@ AC_LINK_IFELSE([
[AC_MSG_ERROR([librapidcheck is not found.])])
AC_LANG_POP(C++)
fi
])
# Look for nlohmann/json.
PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9])
# documentation generation switch
AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation generation]),
doc_generate=$enableval, doc_generate=yes)
@ -388,6 +392,8 @@ if test "$embedded_sandbox_shell" = yes; then
AC_DEFINE(HAVE_EMBEDDED_SANDBOX_SHELL, 1, [Include the sandbox shell in the Nix binary.])
fi
])
# Expand all variables in config.status.
test "$prefix" = NONE && prefix=$ac_default_prefix

650
flake.nix
View file

@ -17,11 +17,19 @@
inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; };
outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, flake-compat, libgit2 }:
outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, libgit2, ... }:
let
inherit (nixpkgs) lib;
# Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981
# Not an "idiomatic" flake input because:
# - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730
# - Subflake would download redundant and huge parent flake
# - No git tree hash support: https://github.com/NixOS/nix/issues/6044
inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; }))
fileset;
officialRelease = false;
# Set to true to build the release notes for the next release.
@ -53,7 +61,14 @@
"x86_64-w64-mingw32"
];
stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ];
stdenvs = [
"ccacheStdenv"
"clang11Stdenv"
"clangStdenv"
"gccStdenv"
"libcxxStdenv"
"stdenv"
];
forAllSystems = lib.genAttrs systems;
@ -68,57 +83,6 @@
})
stdenvs);
# Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981
# Not an "idiomatic" flake input because:
# - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730
# - Subflake would download redundant and huge parent flake
# - No git tree hash support: https://github.com/NixOS/nix/issues/6044
inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; }))
fileset;
baseFiles =
# .gitignore has already been processed, so any changes in it are irrelevant
# at this point. It is not represented verbatim for test purposes because
# that would interfere with repo semantics.
fileset.fileFilter (f: f.name != ".gitignore") ./.;
configureFiles = fileset.unions [
./.version
./configure.ac
./m4
# TODO: do we really need README.md? It doesn't seem used in the build.
./README.md
];
topLevelBuildFiles = fileset.unions [
./local.mk
./Makefile
./Makefile.config.in
./mk
];
functionalTestFiles = fileset.unions [
./tests/functional
(fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
];
nixSrc = fileset.toSource {
root = ./.;
fileset = fileset.intersect baseFiles (fileset.unions [
configureFiles
topLevelBuildFiles
./boehmgc-coroutine-sp-fallback.diff
./doc
./misc
./precompiled-headers.h
./src
./tests/unit
./COPYING
./scripts/local.mk
functionalTestFiles
]);
};
# Memoize nixpkgs for different platforms for efficiency.
nixpkgsFor = forAllSystems
(system: let
@ -143,408 +107,121 @@
cross = lib.genAttrs shellCrossSystems (crossSystem: make-pkgs crossSystem "stdenv");
});
commonDeps =
{ pkgs
, isStatic ? pkgs.stdenv.hostPlatform.isStatic
}:
with pkgs; rec {
# Use "busybox-sandbox-shell" if present,
# if not (legacy) fallback and hope it's sufficient.
sh = pkgs.busybox-sandbox-shell or (busybox.override {
useMusl = true;
enableStatic = true;
enableMinimal = true;
extraConfig = ''
CONFIG_FEATURE_FANCY_ECHO y
CONFIG_FEATURE_SH_MATH y
CONFIG_FEATURE_SH_MATH_64 y
CONFIG_ASH y
CONFIG_ASH_OPTIMIZE_FOR_SIZE y
CONFIG_ASH_ALIAS y
CONFIG_ASH_BASH_COMPAT y
CONFIG_ASH_CMDCMD y
CONFIG_ASH_ECHO y
CONFIG_ASH_GETOPTS y
CONFIG_ASH_INTERNAL_GLOB y
CONFIG_ASH_JOB_CONTROL y
CONFIG_ASH_PRINTF y
CONFIG_ASH_TEST y
'';
});
configureFlags =
lib.optionals stdenv.isLinux [
"--with-boost=${boost}/lib"
"--with-sandbox-shell=${sh}/bin/busybox"
]
++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [
"LDFLAGS=-fuse-ld=gold"
];
testConfigureFlags = [
"RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include"
] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
"--enable-install-unit-tests"
"--with-check-bin-dir=${builtins.placeholder "check"}/bin"
"--with-check-lib-dir=${builtins.placeholder "check"}/lib"
];
internalApiDocsConfigureFlags = [
"--enable-internal-api-docs"
];
changelog-d = pkgs.buildPackages.callPackage ./misc/changelog-d.nix { };
nativeBuildDeps =
[
buildPackages.bison
buildPackages.flex
(lib.getBin buildPackages.lowdown-nix)
buildPackages.mdbook
buildPackages.mdbook-linkcheck
buildPackages.autoconf-archive
buildPackages.autoreconfHook
buildPackages.pkg-config
# Tests
buildPackages.git
buildPackages.mercurial # FIXME: remove? only needed for tests
buildPackages.jq # Also for custom mdBook preprocessor.
buildPackages.openssh # only needed for tests (ssh-keygen)
]
++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]
# Official releases don't have rl-next, so we don't need to compile a changelog
++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d
;
buildDeps =
[ curl
bzip2 xz brotli
openssl sqlite
libarchive
(pkgs.libgit2.overrideAttrs (attrs: {
src = libgit2;
version = libgit2.lastModifiedDate;
cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"];
}))
boost
libsodium
]
++ lib.optionals (!stdenv.hostPlatform.isWindows) [
editline
lowdown-nix
]
++ lib.optional stdenv.isLinux libseccomp
++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid;
checkDeps = [
gtest
rapidcheck
];
internalApiDocsDeps = [
buildPackages.doxygen
];
awsDeps = lib.optional (stdenv.isLinux || stdenv.isDarwin)
(aws-sdk-cpp.override {
apis = ["s3" "transfer"];
customMemoryManagement = false;
});
propagatedDeps =
[ ((boehmgc.override {
enableLargeConfig = true;
}).overrideAttrs(o: {
patches = (o.patches or []) ++ [
./boehmgc-coroutine-sp-fallback.diff
# https://github.com/ivmai/bdwgc/pull/586
./boehmgc-traceable_allocator-public.diff
];
})
)
nlohmann_json
];
};
installScriptFor = tarballs:
with nixpkgsFor.x86_64-linux.native;
runCommand "installer-script"
{ buildInputs = [ nix ];
}
''
mkdir -p $out/nix-support
# Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix.
tarballPath() {
# Remove the store prefix
local path=''${1#${builtins.storeDir}/}
# Get the path relative to the derivation root
local rest=''${path#*/}
# Get the derivation hash
local drvHash=''${path%%-*}
echo "$drvHash/$rest"
}
substitute ${./scripts/install.in} $out/install \
${pkgs.lib.concatMapStrings
(tarball: let
inherit (tarball.stdenv.hostPlatform) system;
in '' \
--replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \
--replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \
''
)
tarballs
} --replace '@nixVersion@' ${version}
echo "file installer $out/install" >> $out/nix-support/hydra-build-products
'';
testNixVersions = pkgs: client: daemon: with commonDeps { inherit pkgs; }; with pkgs.lib; pkgs.stdenv.mkDerivation {
NIX_DAEMON_PACKAGE = daemon;
NIX_CLIENT_PACKAGE = client;
name =
"nix-tests"
+ optionalString
(versionAtLeast daemon.version "2.4pre20211005" &&
versionAtLeast client.version "2.4pre20211005")
"-${client.version}-against-${daemon.version}";
inherit version;
src = fileset.toSource {
root = ./.;
fileset = fileset.intersect baseFiles (fileset.unions [
configureFiles
topLevelBuildFiles
functionalTestFiles
]);
nixpkgsFor.x86_64-linux.native.callPackage ./scripts/installer.nix {
inherit tarballs;
};
VERSION_SUFFIX = versionSuffix;
testNixVersions = pkgs: client: daemon:
pkgs.callPackage ./package.nix {
pname =
"nix-tests"
+ lib.optionalString
(lib.versionAtLeast daemon.version "2.4pre20211005" &&
lib.versionAtLeast client.version "2.4pre20211005")
"-${client.version}-against-${daemon.version}";
nativeBuildInputs = nativeBuildDeps;
buildInputs = buildDeps ++ awsDeps ++ checkDeps;
propagatedBuildInputs = propagatedDeps;
inherit fileset;
enableParallelBuilding = true;
test-client = client;
test-daemon = daemon;
configureFlags =
testConfigureFlags # otherwise configure fails
++ [ "--disable-build" ];
dontBuild = true;
doInstallCheck = true;
doBuild = false;
};
installPhase = ''
mkdir -p $out
'';
installCheckPhase = ''
mkdir -p src/nix-channel
make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES
'';
binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix {
inherit nix;
};
binaryTarball = nix: pkgs:
let
inherit (pkgs) buildPackages;
inherit (pkgs) cacert;
installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; };
in
pkgs.runCommand "nix-binary-tarball-${version}"
{ #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
meta.description = "Distribution-independent Nix bootstrap binaries for ${pkgs.system}";
}
''
cp ${installerClosureInfo}/registration $TMPDIR/reginfo
cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
--subst-var-by nix ${nix} \
--subst-var-by cacert ${cacert}
substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
--subst-var-by nix ${nix} \
--subst-var-by cacert ${cacert}
substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
--subst-var-by nix ${nix} \
--subst-var-by cacert ${cacert}
substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
--subst-var-by nix ${nix} \
--subst-var-by cacert ${cacert}
if type -p shellcheck; then
# SC1090: Don't worry about not being able to find
# $nix/etc/profile.d/nix.sh
shellcheck --exclude SC1090 $TMPDIR/install
shellcheck $TMPDIR/create-darwin-volume.sh
shellcheck $TMPDIR/install-darwin-multi-user.sh
shellcheck $TMPDIR/install-systemd-multi-user.sh
# SC1091: Don't panic about not being able to source
# /etc/profile
# SC2002: Ignore "useless cat" "error", when loading
# .reginfo, as the cat is a much cleaner
# implementation, even though it is "useless"
# SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
# root's home directory
shellcheck --external-sources \
--exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
fi
chmod +x $TMPDIR/install
chmod +x $TMPDIR/create-darwin-volume.sh
chmod +x $TMPDIR/install-darwin-multi-user.sh
chmod +x $TMPDIR/install-systemd-multi-user.sh
chmod +x $TMPDIR/install-multi-user
dir=nix-${version}-${pkgs.system}
fn=$out/$dir.tar.xz
mkdir -p $out/nix-support
echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
tar cvfJ $fn \
--owner=0 --group=0 --mode=u+rw,uga+r \
--mtime='1970-01-01' \
--absolute-names \
--hard-dereference \
--transform "s,$TMPDIR/install,$dir/install," \
--transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
--transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
--transform "s,$NIX_STORE,$dir/store,S" \
$TMPDIR/install \
$TMPDIR/create-darwin-volume.sh \
$TMPDIR/install-darwin-multi-user.sh \
$TMPDIR/install-systemd-multi-user.sh \
$TMPDIR/install-multi-user \
$TMPDIR/reginfo \
$(cat ${installerClosureInfo}/store-paths)
'';
overlayFor = getStdenv: final: prev:
let currentStdenv = getStdenv final; in
let
stdenv = getStdenv final;
in
{
nixStable = prev.nix;
# Forward from the previous stage as we dont want it to pick the lowdown override
nixUnstable = prev.nixUnstable;
inherit (prev) nixUnstable;
nix =
with final;
with commonDeps {
inherit pkgs;
inherit (currentStdenv.hostPlatform) isStatic;
};
let
canRunInstalled = currentStdenv.buildPlatform.canExecute currentStdenv.hostPlatform;
in currentStdenv.mkDerivation (finalAttrs: {
name = "nix-${version}";
inherit version;
default-busybox-sandbox-shell = final.busybox.override {
useMusl = true;
enableStatic = true;
enableMinimal = true;
extraConfig = ''
CONFIG_FEATURE_FANCY_ECHO y
CONFIG_FEATURE_SH_MATH y
CONFIG_FEATURE_SH_MATH_64 y
src = nixSrc;
VERSION_SUFFIX = versionSuffix;
CONFIG_ASH y
CONFIG_ASH_OPTIMIZE_FOR_SIZE y
outputs = [ "out" "dev" "doc" ]
++ lib.optional (currentStdenv.hostPlatform != currentStdenv.buildPlatform) "check";
nativeBuildInputs = nativeBuildDeps;
buildInputs = buildDeps
# There have been issues building these dependencies
++ lib.optionals (currentStdenv.hostPlatform == currentStdenv.buildPlatform) awsDeps
++ lib.optionals finalAttrs.doCheck checkDeps;
propagatedBuildInputs = propagatedDeps;
disallowedReferences = [ boost ];
preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic)
''
# Copy libboost_context so we don't get all of Boost in our closure.
# https://github.com/NixOS/nixpkgs/issues/45462
mkdir -p $out/lib
cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
rm -f $out/lib/*.a
${lib.optionalString currentStdenv.hostPlatform.isLinux ''
chmod u+w $out/lib/*.so.*
patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
''}
${lib.optionalString currentStdenv.hostPlatform.isDarwin ''
for LIB in $out/lib/*.dylib; do
chmod u+w $LIB
install_name_tool -id $LIB $LIB
install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
done
install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
''}
'';
configureFlags = configureFlags ++
[ "--sysconfdir=/etc" ] ++
lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" ++
[ (lib.enableFeature finalAttrs.doCheck "tests") ] ++
lib.optionals finalAttrs.doCheck testConfigureFlags ++
lib.optional (!canRunInstalled) "--disable-doc-gen";
enableParallelBuilding = true;
makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1";
doCheck = true;
installFlags = "sysconfdir=$(out)/etc";
postInstall = ''
mkdir -p $doc/nix-support
echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
${lib.optionalString currentStdenv.hostPlatform.isStatic ''
mkdir -p $out/nix-support
echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
''}
${lib.optionalString currentStdenv.isDarwin ''
install_name_tool \
-change ${boost}/lib/libboost_context.dylib \
$out/lib/libboost_context.dylib \
$out/lib/libnixutil.dylib
''}
CONFIG_ASH_ALIAS y
CONFIG_ASH_BASH_COMPAT y
CONFIG_ASH_CMDCMD y
CONFIG_ASH_ECHO y
CONFIG_ASH_GETOPTS y
CONFIG_ASH_INTERNAL_GLOB y
CONFIG_ASH_JOB_CONTROL y
CONFIG_ASH_PRINTF y
CONFIG_ASH_TEST y
'';
};
doInstallCheck = finalAttrs.doCheck;
installCheckFlags = "sysconfdir=$(out)/etc";
installCheckTarget = "installcheck"; # work around buggy detection in stdenv
lowdown-nix = final.callPackage ./misc/lowdown.nix {
inherit lowdown-src stdenv;
};
separateDebugInfo = !currentStdenv.hostPlatform.isStatic;
strictDeps = true;
hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
passthru.perl-bindings = final.callPackage ./perl {
inherit fileset;
stdenv = currentStdenv;
};
meta.platforms = lib.platforms.unix ++ lib.platforms.windows;
meta.mainProgram = "nix";
libgit2-nix = final.libgit2.overrideAttrs (attrs: {
src = libgit2;
version = libgit2.lastModifiedDate;
cmakeFlags = attrs.cmakeFlags or []
++ [ "-DUSE_SSH=exec" ];
});
lowdown-nix = with final; currentStdenv.mkDerivation rec {
name = "lowdown-0.9.0";
boehmgc-nix = (final.boehmgc.override {
enableLargeConfig = true;
}).overrideAttrs(o: {
patches = (o.patches or []) ++ [
./boehmgc-coroutine-sp-fallback.diff
src = lowdown-src;
# https://github.com/ivmai/bdwgc/pull/586
./boehmgc-traceable_allocator-public.diff
];
});
outputs = [ "out" "bin" "dev" ];
changelog-d-nix = final.buildPackages.callPackage ./misc/changelog-d.nix { };
nativeBuildInputs = [ buildPackages.which ];
nix =
let
officialRelease = false;
versionSuffix =
if officialRelease
then ""
else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
in final.callPackage ./package.nix {
inherit
fileset
stdenv
versionSuffix
;
officialRelease = false;
boehmgc = final.boehmgc-nix;
libgit2 = final.libgit2-nix;
lowdown = final.lowdown-nix;
busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
changelog-d = final.changelog-d-nix;
} // {
# this is a proper separate downstream package, but put
# here also for back compat reasons.
perl-bindings = final.nix-perl-bindings;
};
nix-perl-bindings = final.callPackage ./perl {
inherit fileset stdenv;
};
configurePhase = ''
${if (currentStdenv.isDarwin && currentStdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""}
./configure \
PREFIX=${placeholder "dev"} \
BINDIR=${placeholder "bin"}/bin
'';
};
};
in {
# A Nixpkgs overlay that overrides the 'nix' and
@ -615,60 +292,17 @@
dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);
# Line coverage analysis.
coverage =
with nixpkgsFor.x86_64-linux.native;
with commonDeps { inherit pkgs; };
releaseTools.coverageAnalysis {
name = "nix-coverage-${version}";
src = nixSrc;
configureFlags = testConfigureFlags;
enableParallelBuilding = true;
nativeBuildInputs = nativeBuildDeps;
buildInputs = buildDeps ++ propagatedDeps ++ awsDeps ++ checkDeps;
dontInstall = false;
doInstallCheck = true;
installCheckTarget = "installcheck"; # work around buggy detection in stdenv
lcovFilter = [ "*/boost/*" "*-tab.*" ];
hardeningDisable = ["fortify"];
NIX_CFLAGS_COMPILE = "-DCOVERAGE=1";
};
coverage = nixpkgsFor.x86_64-linux.native.nix.override {
pname = "nix-coverage";
withCoverageChecks = true;
};
# API docs for Nix's unstable internal C++ interfaces.
internal-api-docs =
with nixpkgsFor.x86_64-linux.native;
with commonDeps { inherit pkgs; };
stdenv.mkDerivation {
pname = "nix-internal-api-docs";
inherit version;
src = nixSrc;
configureFlags = testConfigureFlags ++ internalApiDocsConfigureFlags;
nativeBuildInputs = nativeBuildDeps;
buildInputs = buildDeps ++ propagatedDeps
++ awsDeps ++ checkDeps ++ internalApiDocsDeps;
dontBuild = true;
installTargets = [ "internal-api-html" ];
postInstall = ''
mkdir -p $out/nix-support
echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products
'';
};
internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix {
inherit fileset;
doBuild = false;
enableInternalAPIDocs = true;
};
# System tests.
tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // {
@ -676,7 +310,9 @@
# Make sure that nix-env still produces the exact same result
# on a particular version of Nixpkgs.
evalNixpkgs =
with nixpkgsFor.x86_64-linux.native;
let
inherit (nixpkgsFor.x86_64-linux.native) runCommand nix;
in
runCommand "eval-nixos" { buildInputs = [ nix ]; }
''
type -p nix-env
@ -729,7 +365,7 @@
rl-next =
let pkgs = nixpkgsFor.${system}.native;
in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } ''
LANG=C.UTF-8 ${(commonDeps { inherit pkgs; }).changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out
LANG=C.UTF-8 ${pkgs.changelog-d-nix}/bin/changelog-d ${./doc/manual/rl-next} >$out
'';
} // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
dockerImage = self.hydraJobs.dockerImage.${system};
@ -768,47 +404,17 @@
stdenvs)));
devShells = let
makeShell = pkgs: stdenv:
let
canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
in
with commonDeps { inherit pkgs; };
stdenv.mkDerivation {
name = "nix";
makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; }).overrideAttrs (_: {
installFlags = "sysconfdir=$(out)/etc";
shellHook = ''
PATH=$prefix/bin:$PATH
unset PYTHONPATH
export MANPATH=$out/share/man:$MANPATH
outputs = [ "out" "dev" "doc" ]
++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "check";
nativeBuildInputs = nativeBuildDeps
++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear
++ lib.optional
(stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform)
pkgs.buildPackages.clang-tools
# We want changelog-d in the shell even if the current build doesn't need it
++ lib.optional (officialRelease || ! buildUnreleasedNotes) changelog-d
;
buildInputs = buildDeps ++ propagatedDeps
++ awsDeps ++ checkDeps ++ internalApiDocsDeps;
configureFlags = configureFlags
++ testConfigureFlags ++ internalApiDocsConfigureFlags
++ lib.optional (!canRunInstalled) "--disable-doc-gen";
enableParallelBuilding = true;
installFlags = "sysconfdir=$(out)/etc";
shellHook =
''
PATH=$prefix/bin:$PATH
unset PYTHONPATH
export MANPATH=$out/share/man:$MANPATH
# Make bash completion work.
XDG_DATA_DIRS+=:$out/share
'';
};
# Make bash completion work.
XDG_DATA_DIRS+=:$out/share
'';
});
in
forAllSystems (system:
let

22
misc/lowdown.nix Normal file
View file

@ -0,0 +1,22 @@
{ lib
, stdenv
, which
, lowdown-src
}:
stdenv.mkDerivation rec {
name = "lowdown-0.9.0";
src = lowdown-src;
outputs = [ "out" "bin" "dev" ];
nativeBuildInputs = [ which ];
configurePhase = ''
${lib.optionalString (stdenv.isDarwin && stdenv.isAarch64) "echo \"HAVE_SANDBOX_INIT=false\" > configure.local"}
./configure \
PREFIX=${placeholder "dev"} \
BINDIR=${placeholder "bin"}/bin
'';
}

384
package.nix Normal file
View file

@ -0,0 +1,384 @@
{ lib
, stdenv
, releaseTools
, autoconf-archive
, autoreconfHook
, aws-sdk-cpp
, boehmgc
, nlohmann_json
, bison
, boost
, brotli
, bzip2
, changelog-d
, curl
, editline
, fileset
, flex
, git
, gtest
, jq
, doxygen
, libarchive
, libcpuid
, libgit2
, libseccomp
, libsodium
, lowdown
, mdbook
, mdbook-linkcheck
, mercurial
, openssh
, openssl
, pkg-config
, rapidcheck
, sqlite
, util-linux
, xz
, busybox-sandbox-shell ? null
# Configuration Options
#:
# This probably seems like too many degrees of freedom, but it
# faithfully reflects how the underlying configure + make build system
# work. The top-level flake.nix will choose useful combinations of these
# options to CI.
, pname ? "nix"
, versionSuffix ? ""
, officialRelease ? false
# Whether to build Nix. Useful to skip for tasks like (a) just
# generating API docs or (b) testing existing pre-built versions of Nix
, doBuild ? true
# Run the unit tests as part of the build. See `installUnitTests` for an
# alternative to this.
, doCheck ? __forDefaults.canRunInstalled
# Run the functional tests as part of the build.
, doInstallCheck ? test-client != null || __forDefaults.canRunInstalled
# Check test coverage of Nix. Probably want to use with with at least
# one of `doCHeck` or `doInstallCheck` enabled.
, withCoverageChecks ? false
# Whether to build the regular manual
, enableManual ? __forDefaults.canRunInstalled
# Whether to compile `rl-next.md`, the release notes for the next
# not-yet-released version of Nix in the manul, from the individual
# change log entries in the directory.
, buildUnreleasedNotes ? false
# Whether to build the internal API docs, can be done separately from
# everything else.
, enableInternalAPIDocs ? false
# Whether to install unit tests. This is useful when cross compiling
# since we cannot run them natively during the build, but can do so
# later.
, installUnitTests ? __forDefaults.canRunInstalled
# For running the functional tests against a pre-built Nix. Probably
# want to use in conjunction with `doBuild = false;`.
, test-daemon ? null
, test-client ? null
# Not a real argument, just the only way to approximate let-binding some
# stuff for argument defaults.
, __forDefaults ? {
canRunInstalled = doBuild && stdenv.buildPlatform.canExecute stdenv.hostPlatform;
}
}:
let
version = lib.fileContents ./.version + versionSuffix;
# selected attributes with defaults, will be used to define some
# things which should instead be gotten via `finalAttrs` in order to
# work with overriding.
attrs = {
inherit doBuild doCheck doInstallCheck;
};
filesets = {
baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.;
configureFiles = fileset.unions [
./.version
./configure.ac
./m4
# TODO: do we really need README.md? It doesn't seem used in the build.
./README.md
];
topLevelBuildFiles = fileset.unions [
./local.mk
./Makefile
./Makefile.config.in
./mk
];
functionalTestFiles = fileset.unions [
./tests/functional
(fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
];
};
mkDerivation =
if withCoverageChecks
then
# TODO support `finalAttrs` args function in
# `releaseTools.coverageAnalysis`.
argsFun:
releaseTools.coverageAnalysis (let args = argsFun args; in args)
else stdenv.mkDerivation;
in
mkDerivation (finalAttrs: let
inherit (finalAttrs)
doCheck
doInstallCheck
;
doBuild = !finalAttrs.dontBuild;
# Either running the unit tests during the build, or installing them
# to be run later, requiresthe unit tests to be built.
buildUnitTests = doCheck || installUnitTests;
anySortOfTesting = buildUnitTests || doInstallCheck;
in {
inherit pname version;
src =
let
in
fileset.toSource {
root = ./.;
fileset = fileset.intersect filesets.baseFiles (fileset.unions ([
filesets.configureFiles
filesets.topLevelBuildFiles
./doc/internal-api
] ++ lib.optionals doBuild [
./boehmgc-coroutine-sp-fallback.diff
./doc
./misc
./precompiled-headers.h
./src
./tests/unit
./COPYING
./scripts/local.mk
] ++ lib.optionals anySortOfTesting [
filesets.functionalTestFiles
]));
};
VERSION_SUFFIX = versionSuffix;
outputs = [ "out" ]
++ lib.optional doBuild "dev"
# If we are doing just build or just docs, the one thing will use
# "out". We only need additional outputs if we are doing both.
++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs)) "doc"
++ lib.optional installUnitTests "check";
nativeBuildInputs = [
autoconf-archive
autoreconfHook
pkg-config
] ++ lib.optionals doBuild [
bison
flex
] ++ lib.optionals enableManual [
(lib.getBin lowdown)
mdbook
mdbook-linkcheck
] ++ lib.optionals (doInstallCheck || enableManual) [
jq # Also for custom mdBook preprocessor.
] ++ lib.optional stdenv.hostPlatform.isLinux util-linux
# Official releases don't have rl-next, so we don't need to compile a
# changelog
++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d
++ lib.optional enableInternalAPIDocs doxygen
;
buildInputs = lib.optionals doBuild [
boost
brotli
bzip2
curl
libarchive
libgit2
libsodium
openssl
sqlite
xz
] ++ lib.optionals (!stdenv.hostPlatform.isWindows) [
editline
lowdown
] ++ lib.optional stdenv.isLinux libseccomp
++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
# There have been issues building these dependencies
++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin))
(aws-sdk-cpp.override {
apis = ["s3" "transfer"];
customMemoryManagement = false;
})
;
propagatedBuildInputs = [
boehmgc
nlohmann_json
];
dontBuild = !attrs.doBuild;
doCheck = attrs.doCheck;
checkInputs = [
gtest
rapidcheck
];
nativeCheckInputs = [
git
mercurial
openssh
];
disallowedReferences = [ boost ];
preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) (
''
# Copy libboost_context so we don't get all of Boost in our closure.
# https://github.com/NixOS/nixpkgs/issues/45462
mkdir -p $out/lib
cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib
rm -f $out/lib/*.a
'' + lib.optionalString stdenv.hostPlatform.isLinux ''
chmod u+w $out/lib/*.so.*
patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
'' + lib.optionalString stdenv.hostPlatform.isDarwin ''
for LIB in $out/lib/*.dylib; do
chmod u+w $LIB
install_name_tool -id $LIB $LIB
install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
done
install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
''
);
configureFlags = [
"--sysconfdir=/etc"
(lib.enableFeature doBuild "build")
(lib.enableFeature anySortOfTesting "tests")
(lib.enableFeature enableInternalAPIDocs "internal-api-docs")
(lib.enableFeature enableManual "doc-gen")
(lib.enableFeature installUnitTests "install-unit-tests")
] ++ lib.optionals installUnitTests [
"--with-check-bin-dir=${builtins.placeholder "check"}/bin"
"--with-check-lib-dir=${builtins.placeholder "check"}/lib"
] ++ lib.optionals (doBuild && stdenv.isLinux) [
"--with-boost=${boost}/lib"
"--with-sandbox-shell=${busybox-sandbox-shell}/bin/busybox"
] ++ lib.optional (doBuild && stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux"))
"LDFLAGS=-fuse-ld=gold"
++ lib.optional (doBuild && stdenv.hostPlatform.isStatic) "--enable-embedded-sandbox-shell"
++ lib.optional buildUnitTests "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include";
enableParallelBuilding = true;
makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1";
installTargets = lib.optional doBuild "install"
++ lib.optional enableInternalAPIDocs "internal-api-html";
installFlags = "sysconfdir=$(out)/etc";
# In this case we are probably just running tests, and so there isn't
# anything to install, we just make an empty directory to signify tests
# succeeded.
installPhase = if finalAttrs.installTargets != [] then null else ''
mkdir -p $out
'';
postInstall = lib.optionalString doBuild (
''
mkdir -p $doc/nix-support
echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
'' + lib.optionalString stdenv.hostPlatform.isStatic ''
mkdir -p $out/nix-support
echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
'' + lib.optionalString stdenv.isDarwin ''
install_name_tool \
-change ${boost}/lib/libboost_context.dylib \
$out/lib/libboost_context.dylib \
$out/lib/libnixutil.dylib
''
) + lib.optionalString enableInternalAPIDocs ''
mkdir -p ''${!outputDoc}/nix-support
echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products
'';
doInstallCheck = attrs.doInstallCheck;
installCheckFlags = "sysconfdir=$(out)/etc";
# Work around buggy detection in stdenv.
installCheckTarget = "installcheck";
# Work around weird bug where it doesn't think there is a Makefile.
installCheckPhase = if (!doBuild && doInstallCheck) then ''
mkdir -p src/nix-channel
make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES
'' else null;
# Needed for tests if we are not doing a build, but testing existing
# built Nix.
preInstallCheck = lib.optionalString (! doBuild) ''
mkdir -p src/nix-channel
'';
separateDebugInfo = !stdenv.hostPlatform.isStatic;
# TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated
# to work with `strictDeps`.
strictDeps = !withCoverageChecks;
hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
meta = {
platforms = lib.platforms.unix ++ lib.platforms.windows;
mainProgram = "nix";
broken = !(lib.all (a: a) [
# We cannot run or install unit tests if we don't build them or
# Nix proper (which they depend on).
(installUnitTests -> doBuild)
(doCheck -> doBuild)
# We have to build the manual to build unreleased notes, as those
# are part of the manual
(buildUnreleasedNotes -> enableManual)
# The build process for the manual currently requires extracting
# data from the Nix executable we are trying to document.
(enableManual -> doBuild)
]);
};
} // lib.optionalAttrs withCoverageChecks {
lcovFilter = [ "*/boost/*" "*-tab.*" ];
hardeningDisable = ["fortify"];
NIX_CFLAGS_COMPILE = "-DCOVERAGE=1";
dontInstall = false;
} // lib.optionalAttrs (test-daemon != null) {
NIX_DAEMON_PACKAGE = test-daemon;
} // lib.optionalAttrs (test-client != null) {
NIX_CLIENT_PACKAGE = test-client;
})

View file

@ -0,0 +1,84 @@
{ runCommand
, system
, buildPackages
, cacert
, nix
}:
let
installerClosureInfo = buildPackages.closureInfo {
rootPaths = [ nix cacert ];
};
inherit (nix) version;
env = {
#nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
};
in
runCommand "nix-binary-tarball-${version}" env ''
cp ${installerClosureInfo}/registration $TMPDIR/reginfo
cp ${./create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
substitute ${./install-nix-from-closure.sh} $TMPDIR/install \
--subst-var-by nix ${nix} \
--subst-var-by cacert ${cacert}
substitute ${./install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
--subst-var-by nix ${nix} \
--subst-var-by cacert ${cacert}
substitute ${./install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
--subst-var-by nix ${nix} \
--subst-var-by cacert ${cacert}
substitute ${./install-multi-user.sh} $TMPDIR/install-multi-user \
--subst-var-by nix ${nix} \
--subst-var-by cacert ${cacert}
if type -p shellcheck; then
# SC1090: Don't worry about not being able to find
# $nix/etc/profile.d/nix.sh
shellcheck --exclude SC1090 $TMPDIR/install
shellcheck $TMPDIR/create-darwin-volume.sh
shellcheck $TMPDIR/install-darwin-multi-user.sh
shellcheck $TMPDIR/install-systemd-multi-user.sh
# SC1091: Don't panic about not being able to source
# /etc/profile
# SC2002: Ignore "useless cat" "error", when loading
# .reginfo, as the cat is a much cleaner
# implementation, even though it is "useless"
# SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
# root's home directory
shellcheck --external-sources \
--exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
fi
chmod +x $TMPDIR/install
chmod +x $TMPDIR/create-darwin-volume.sh
chmod +x $TMPDIR/install-darwin-multi-user.sh
chmod +x $TMPDIR/install-systemd-multi-user.sh
chmod +x $TMPDIR/install-multi-user
dir=nix-${version}-${system}
fn=$out/$dir.tar.xz
mkdir -p $out/nix-support
echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
tar cvfJ $fn \
--owner=0 --group=0 --mode=u+rw,uga+r \
--mtime='1970-01-01' \
--absolute-names \
--hard-dereference \
--transform "s,$TMPDIR/install,$dir/install," \
--transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
--transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
--transform "s,$NIX_STORE,$dir/store,S" \
$TMPDIR/install \
$TMPDIR/create-darwin-volume.sh \
$TMPDIR/install-darwin-multi-user.sh \
$TMPDIR/install-systemd-multi-user.sh \
$TMPDIR/install-multi-user \
$TMPDIR/reginfo \
$(cat ${installerClosureInfo}/store-paths)
''

36
scripts/installer.nix Normal file
View file

@ -0,0 +1,36 @@
{ lib
, runCommand
, nix
, tarballs
}:
runCommand "installer-script" {
buildInputs = [ nix ];
} ''
mkdir -p $out/nix-support
# Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix.
tarballPath() {
# Remove the store prefix
local path=''${1#${builtins.storeDir}/}
# Get the path relative to the derivation root
local rest=''${path#*/}
# Get the derivation hash
local drvHash=''${path%%-*}
echo "$drvHash/$rest"
}
substitute ${./install.in} $out/install \
${lib.concatMapStrings
(tarball: let
inherit (tarball.stdenv.hostPlatform) system;
in '' \
--replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \
--replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \
''
)
tarballs
} --replace '@nixVersion@' ${nix.version}
echo "file installer $out/install" >> $out/nix-support/hydra-build-products
''