From 2d5b1b24bf70a498e4c0b378704cfdb6471cc699 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Oct 2017 13:01:50 +0200 Subject: [PATCH] Pass lists/attrsets to bash as (associative) arrays --- src/libexpr/primops.cc | 6 +-- src/libstore/build.cc | 96 ++++++++++++++++++++++++++++++++++++-- src/libutil/util.cc | 10 ++++ src/libutil/util.hh | 6 +-- src/nix-build/nix-build.cc | 4 -- src/nix-store/nix-store.cc | 11 +---- tests/config.nix | 2 +- tests/local.mk | 3 +- tests/structured-attrs.nix | 47 +++++++++++++++++++ tests/structured-attrs.sh | 7 +++ 10 files changed, 166 insertions(+), 26 deletions(-) create mode 100644 tests/structured-attrs.nix create mode 100644 tests/structured-attrs.sh diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index fcd3f8efe..afd344286 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -713,7 +713,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo; Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); - drv.env["out"] = outPath; + if (!jsonObject) drv.env["out"] = outPath; drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, *outputHash); } @@ -724,7 +724,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * an empty value. This ensures that changes in the set of output names do get reflected in the hash. */ for (auto & i : outputs) { - drv.env[i] = ""; + if (!jsonObject) drv.env[i] = ""; drv.outputs[i] = DerivationOutput("", "", ""); } @@ -735,7 +735,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & i : drv.outputs) if (i.second.path == "") { Path outPath = state.store->makeOutputPath(i.first, h, drvName); - drv.env[i.first] = outPath; + if (!jsonObject) drv.env[i.first] = outPath; i.second.path = outPath; } } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 856d516d2..a7d418404 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -55,6 +56,8 @@ #include #endif +#include + namespace nix { @@ -2286,12 +2289,99 @@ void DerivationGoal::initEnv() } +static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); + + void DerivationGoal::writeStructuredAttrs() { - auto json = drv->env.find("__json"); - if (json == drv->env.end()) return; + auto jsonAttr = drv->env.find("__json"); + if (jsonAttr == drv->env.end()) return; - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json->second, inputRewrites)); + try { + + auto jsonStr = rewriteStrings(jsonAttr->second, inputRewrites); + + auto json = nlohmann::json::parse(jsonStr); + + /* Add an "outputs" object containing the output paths. */ + nlohmann::json outputs; + for (auto & i : drv->outputs) + outputs[i.first] = rewriteStrings(i.second.path, inputRewrites); + json["outputs"] = outputs; + + writeFile(tmpDir + "/.attrs.json", json.dump()); + + /* As a convenience to bash scripts, write a shell file that + maps all attributes that are representable in bash - + namely, strings, integers, nulls, Booleans, and arrays and + objects consisting entirely of those values. (So nested + arrays or objects are not supported.) */ + + auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional { + if (value.is_string()) + return shellEscape(value); + + if (value.is_number()) { + auto f = value.get(); + if (std::ceil(f) == f) + return std::to_string(value.get()); + } + + if (value.is_null()) + return "''"; + + if (value.is_boolean()) + return value.get() ? "1" : ""; + + return {}; + }; + + std::string jsonSh; + + for (auto i = json.begin(); i != json.end(); ++i) { + + if (!std::regex_match(i.key(), shVarName)) continue; + + auto & value = i.value(); + + auto s = handleSimpleType(value); + if (s) + jsonSh += fmt("declare %s=%s\n", i.key(), *s); + + else if (value.is_array()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += *s3; s2 += ' '; + } + + if (good) + jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); + } + + else if (value.is_object()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); + } + + if (good) + jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); + } + } + + writeFile(tmpDir + "/.attrs.sh", jsonSh); + + } catch (std::exception & e) { + throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what()); + } } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 3c98a61f9..9346d5dc4 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1142,6 +1142,16 @@ std::string toLower(const std::string & s) } +std::string shellEscape(const std::string & s) +{ + std::string r = "'"; + for (auto & i : s) + if (i == '\'') r += "'\\''"; else r += i; + r += '\''; + return r; +} + + void ignoreException() { try { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 6a66576e9..fccf5d854 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -352,10 +352,8 @@ bool hasSuffix(const string & s, const string & suffix); std::string toLower(const std::string & s); -/* Escape a string that contains octal-encoded escape codes such as - used in /etc/fstab and /proc/mounts (e.g. "foo\040bar" decodes to - "foo bar"). */ -string decodeOctalEscaped(const string & s); +/* Escape a string as a shell word. */ +std::string shellEscape(const std::string & s); /* Exception handling in destructors: print an error message, then diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 85137206f..8e56e5a46 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -196,10 +196,6 @@ void mainWrapped(int argc, char * * argv) interactive = false; auto execArgs = ""; - auto shellEscape = [](const string & s) { - return "'" + std::regex_replace(s, std::regex("'"), "'\\''") + "'"; - }; - // Überhack to support Perl. Perl examines the shebang and // executes it unless it contains the string "perl" or "indir", // or (undocumented) argv[0] does not contain "perl". Exploit diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 85bcbc22e..f6f276dd1 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -440,15 +440,6 @@ static void opQuery(Strings opFlags, Strings opArgs) } -static string shellEscape(const string & s) -{ - string r; - for (auto & i : s) - if (i == '\'') r += "'\\''"; else r += i; - return r; -} - - static void opPrintEnv(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); @@ -460,7 +451,7 @@ static void opPrintEnv(Strings opFlags, Strings opArgs) /* Print each environment variable in the derivation in a format that can be sourced by the shell. */ for (auto & i : drv.env) - cout << format("export %1%; %1%='%2%'\n") % i.first % shellEscape(i.second); + cout << format("export %1%; %1%=%2%\n") % i.first % shellEscape(i.second); /* Also output the arguments. This doesn't preserve whitespace in arguments. */ diff --git a/tests/config.nix b/tests/config.nix index 76388fdd5..6ba91065b 100644 --- a/tests/config.nix +++ b/tests/config.nix @@ -13,7 +13,7 @@ rec { derivation ({ inherit system; builder = shell; - args = ["-e" args.builder or (builtins.toFile "builder.sh" "eval \"$buildCommand\"")]; + args = ["-e" args.builder or (builtins.toFile "builder.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; PATH = path; } // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; diff --git a/tests/local.mk b/tests/local.mk index 19d6f1893..6160b04c2 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -14,7 +14,8 @@ nix_tests = \ placeholders.sh nix-shell.sh \ linux-sandbox.sh \ build-remote.sh \ - nar-index.sh + nar-index.sh \ + structured-attrs.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) diff --git a/tests/structured-attrs.nix b/tests/structured-attrs.nix new file mode 100644 index 000000000..2adc6b6c3 --- /dev/null +++ b/tests/structured-attrs.nix @@ -0,0 +1,47 @@ +with import ./config.nix; + +mkDerivation { + name = "structured"; + + __structuredAttrs = true; + + buildCommand = '' + set -x + + [[ $int = 123456789 ]] + [[ -z $float ]] + [[ -n $boolTrue ]] + [[ -z $boolFalse ]] + [[ -n ''${hardening[format]} ]] + [[ -z ''${hardening[fortify]} ]] + [[ ''${#buildInputs[@]} = 7 ]] + [[ ''${buildInputs[2]} = c ]] + [[ -v nothing ]] + [[ -z $nothing ]] + + mkdir ''${outputs[out]} + echo bar > $dest + ''; + + buildInputs = [ "a" "b" "c" 123 "'" "\"" null ]; + + hardening.format = true; + hardening.fortify = false; + + outer.inner = [ 1 2 3 ]; + + int = 123456789; + + float = 123.456; + + boolTrue = true; + boolFalse = false; + + nothing = null; + + dest = "${placeholder "out"}/foo"; + + "foo bar" = "BAD"; + "1foobar" = "BAD"; + "foo$" = "BAD"; +} diff --git a/tests/structured-attrs.sh b/tests/structured-attrs.sh new file mode 100644 index 000000000..9ba2672b6 --- /dev/null +++ b/tests/structured-attrs.sh @@ -0,0 +1,7 @@ +source common.sh + +clearStore + +outPath=$(nix-build structured-attrs.nix --no-out-link) + +[[ $(cat $outPath/foo) = bar ]]