From f07388bf985c2440413f398cf93d5f5840d1ec8c Mon Sep 17 00:00:00 2001 From: pennae Date: Mon, 15 Jan 2024 16:52:18 +0100 Subject: [PATCH 01/10] remove ParserFormals this is a proper subset of Formals anyway, so let's just use those and avoid the extra allocations and moves. --- src/libexpr/parser.y | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 60bcfebf9..b7b25854b 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -63,11 +63,6 @@ namespace nix { std::optional error; }; - struct ParserFormals { - std::vector formals; - bool ellipsis = false; - }; - } // using C a struct allows us to avoid having to define the special @@ -179,7 +174,7 @@ static void addAttr(ExprAttrs * attrs, AttrPath && attrPath, } -static Formals * toFormals(ParseData & data, ParserFormals * formals, +static Formals * validateFormals(ParseData & data, Formals * formals, PosIdx pos = noPos, Symbol arg = {}) { std::sort(formals->formals.begin(), formals->formals.end(), @@ -200,18 +195,13 @@ static Formals * toFormals(ParseData & data, ParserFormals * formals, .errPos = data.state.positions[duplicate->second] }); - Formals result; - result.ellipsis = formals->ellipsis; - result.formals = std::move(formals->formals); - - if (arg && result.has(arg)) + if (arg && formals->has(arg)) throw ParseError({ .msg = hintfmt("duplicate formal function argument '%1%'", data.symbols[arg]), .errPos = data.state.positions[pos] }); - delete formals; - return new Formals(std::move(result)); + return formals; } @@ -339,7 +329,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err nix::Expr * e; nix::ExprList * list; nix::ExprAttrs * attrs; - nix::ParserFormals * formals; + nix::Formals * formals; nix::Formal * formal; nix::NixInt n; nix::NixFloat nf; @@ -397,16 +387,16 @@ expr_function : ID ':' expr_function { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), 0, $3); } | '{' formals '}' ':' expr_function - { $$ = new ExprLambda(CUR_POS, toFormals(*data, $2), $5); } + { $$ = new ExprLambda(CUR_POS, validateFormals(*data, $2), $5); } | '{' formals '}' '@' ID ':' expr_function { auto arg = data->symbols.create($5); - $$ = new ExprLambda(CUR_POS, arg, toFormals(*data, $2, CUR_POS, arg), $7); + $$ = new ExprLambda(CUR_POS, arg, validateFormals(*data, $2, CUR_POS, arg), $7); } | ID '@' '{' formals '}' ':' expr_function { auto arg = data->symbols.create($1); - $$ = new ExprLambda(CUR_POS, arg, toFormals(*data, $4, CUR_POS, arg), $7); + $$ = new ExprLambda(CUR_POS, arg, validateFormals(*data, $4, CUR_POS, arg), $7); } | ASSERT expr ';' expr_function { $$ = new ExprAssert(CUR_POS, $2, $4); } @@ -650,11 +640,11 @@ formals : formal ',' formals { $$ = $3; $$->formals.emplace_back(*$1); delete $1; } | formal - { $$ = new ParserFormals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; } + { $$ = new Formals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; } | - { $$ = new ParserFormals; $$->ellipsis = false; } + { $$ = new Formals; $$->ellipsis = false; } | ELLIPSIS - { $$ = new ParserFormals; $$->ellipsis = true; } + { $$ = new Formals; $$->ellipsis = true; } ; formal From e8d9de967fe47a7f9324b0022a2ef50df59f419d Mon Sep 17 00:00:00 2001 From: pennae Date: Mon, 15 Jan 2024 16:52:18 +0100 Subject: [PATCH 02/10] simplify parse error reporting since nix doesn't use the bison `error` terminal anywhere any invocation of yyerror will immediately cause a failure. since we're *already* leaking tons of memory whatever little bit bison allocates internally doesn't much matter any more, and we'll be replacing the parser soon anyway. coincidentally this now also matches the error behavior of URIs when they are disabled or ~/ paths in pure eval mode, duplicate attr detection etc. --- src/libexpr/parser.y | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index b7b25854b..44fae6880 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -18,6 +18,7 @@ #include +#include "finally.hh" #include "util.hh" #include "users.hh" @@ -60,7 +61,6 @@ namespace nix { Expr * result; SourcePath basePath; PosTable::Origin origin; - std::optional error; }; } @@ -315,10 +315,10 @@ static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data) void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error) { - data->error = { + throw ParseError({ .msg = hintfmt(error), .errPos = data->state.positions[makeCurPos(*loc, data)] - }; + }); } @@ -689,11 +689,10 @@ Expr * EvalState::parse( }; yylex_init(&scanner); - yy_scan_buffer(text, length, scanner); - int res = yyparse(scanner, &data); - yylex_destroy(scanner); + Finally _destroy([&] { yylex_destroy(scanner); }); - if (res) throw ParseError(data.error.value()); + yy_scan_buffer(text, length, scanner); + yyparse(scanner, &data); data.result->bindVars(*this, staticEnv); From 1b09b80afac27c67157d4b315c237fa7bb9b8d08 Mon Sep 17 00:00:00 2001 From: pennae Date: Mon, 15 Jan 2024 16:52:18 +0100 Subject: [PATCH 03/10] make parser utility functions members of ParseData all of them need access to parser state in some way. make them members to allow this without fussing so much. --- src/libexpr/parser.y | 126 ++++++++++++++++++++++--------------------- 1 file changed, 66 insertions(+), 60 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 44fae6880..beb660e36 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -27,6 +27,15 @@ #include "eval-settings.hh" #include "globals.hh" +// using C a struct allows us to avoid having to define the special +// members that using string_view here would implicitly delete. +struct StringToken { + const char * p; + size_t l; + bool hasIndentation; + operator std::string_view() const { return {p, l}; } +}; + namespace nix { #define YYLTYPE ::nix::ParserLocation @@ -61,19 +70,18 @@ namespace nix { Expr * result; SourcePath basePath; PosTable::Origin origin; + + void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos); + void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos); + void addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos); + Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {}); + Expr * stripIndentation(const PosIdx pos, + std::vector>> && es); + PosIdx makeCurPos(const ParserLocation & loc); }; } -// using C a struct allows us to avoid having to define the special -// members that using string_view here would implicitly delete. -struct StringToken { - const char * p; - size_t l; - bool hasIndentation; - operator std::string_view() const { return {p, l}; } -}; - #define YY_DECL int yylex \ (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParseData * data) @@ -94,7 +102,7 @@ using namespace nix; namespace nix { -static void dupAttr(const EvalState & state, const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos) +void ParseData::dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos) { throw ParseError({ .msg = hintfmt("attribute '%1%' already defined at %2%", @@ -103,7 +111,7 @@ static void dupAttr(const EvalState & state, const AttrPath & attrPath, const Po }); } -static void dupAttr(const EvalState & state, Symbol attr, const PosIdx pos, const PosIdx prevPos) +void ParseData::dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos) { throw ParseError({ .msg = hintfmt("attribute '%1%' already defined at %2%", state.symbols[attr], state.positions[prevPos]), @@ -112,8 +120,7 @@ static void dupAttr(const EvalState & state, Symbol attr, const PosIdx pos, cons } -static void addAttr(ExprAttrs * attrs, AttrPath && attrPath, - Expr * e, const PosIdx pos, const nix::EvalState & state) +void ParseData::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos) { AttrPath::iterator i; // All attrpaths have at least one attr @@ -126,10 +133,10 @@ static void addAttr(ExprAttrs * attrs, AttrPath && attrPath, if (j != attrs->attrs.end()) { if (!j->second.inherited) { ExprAttrs * attrs2 = dynamic_cast(j->second.e); - if (!attrs2) dupAttr(state, attrPath, pos, j->second.pos); + if (!attrs2) dupAttr(attrPath, pos, j->second.pos); attrs = attrs2; } else - dupAttr(state, attrPath, pos, j->second.pos); + dupAttr(attrPath, pos, j->second.pos); } else { ExprAttrs * nested = new ExprAttrs; attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos); @@ -156,12 +163,12 @@ static void addAttr(ExprAttrs * attrs, AttrPath && attrPath, for (auto & ad : ae->attrs) { auto j2 = jAttrs->attrs.find(ad.first); if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. - dupAttr(state, ad.first, j2->second.pos, ad.second.pos); + dupAttr(ad.first, j2->second.pos, ad.second.pos); jAttrs->attrs.emplace(ad.first, ad.second); } jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end()); } else { - dupAttr(state, attrPath, pos, j->second.pos); + dupAttr(attrPath, pos, j->second.pos); } } else { // This attr path is not defined. Let's create it. @@ -174,8 +181,7 @@ static void addAttr(ExprAttrs * attrs, AttrPath && attrPath, } -static Formals * validateFormals(ParseData & data, Formals * formals, - PosIdx pos = noPos, Symbol arg = {}) +Formals * ParseData::validateFormals(Formals * formals, PosIdx pos, Symbol arg) { std::sort(formals->formals.begin(), formals->formals.end(), [] (const auto & a, const auto & b) { @@ -191,21 +197,21 @@ static Formals * validateFormals(ParseData & data, Formals * formals, } if (duplicate) throw ParseError({ - .msg = hintfmt("duplicate formal function argument '%1%'", data.symbols[duplicate->first]), - .errPos = data.state.positions[duplicate->second] + .msg = hintfmt("duplicate formal function argument '%1%'", symbols[duplicate->first]), + .errPos = state.positions[duplicate->second] }); if (arg && formals->has(arg)) throw ParseError({ - .msg = hintfmt("duplicate formal function argument '%1%'", data.symbols[arg]), - .errPos = data.state.positions[pos] + .msg = hintfmt("duplicate formal function argument '%1%'", symbols[arg]), + .errPos = state.positions[pos] }); return formals; } -static Expr * stripIndentation(const PosIdx pos, SymbolTable & symbols, +Expr * ParseData::stripIndentation(const PosIdx pos, std::vector>> && es) { if (es.empty()) return new ExprString(""); @@ -302,12 +308,12 @@ static Expr * stripIndentation(const PosIdx pos, SymbolTable & symbols, } -static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data) +PosIdx ParseData::makeCurPos(const ParserLocation & loc) { - return data->state.positions.add(data->origin, loc.first_line, loc.first_column); + return state.positions.add(origin, loc.first_line, loc.first_column); } -#define CUR_POS makeCurPos(*yylocp, data) +#define CUR_POS data->makeCurPos(*yylocp) } @@ -317,7 +323,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err { throw ParseError({ .msg = hintfmt(error), - .errPos = data->state.positions[makeCurPos(*loc, data)] + .errPos = data->state.positions[data->makeCurPos(*loc)] }); } @@ -387,16 +393,16 @@ expr_function : ID ':' expr_function { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), 0, $3); } | '{' formals '}' ':' expr_function - { $$ = new ExprLambda(CUR_POS, validateFormals(*data, $2), $5); } + { $$ = new ExprLambda(CUR_POS, data->validateFormals($2), $5); } | '{' formals '}' '@' ID ':' expr_function { auto arg = data->symbols.create($5); - $$ = new ExprLambda(CUR_POS, arg, validateFormals(*data, $2, CUR_POS, arg), $7); + $$ = new ExprLambda(CUR_POS, arg, data->validateFormals($2, CUR_POS, arg), $7); } | ID '@' '{' formals '}' ':' expr_function { auto arg = data->symbols.create($1); - $$ = new ExprLambda(CUR_POS, arg, validateFormals(*data, $4, CUR_POS, arg), $7); + $$ = new ExprLambda(CUR_POS, arg, data->validateFormals($4, CUR_POS, arg), $7); } | ASSERT expr ';' expr_function { $$ = new ExprAssert(CUR_POS, $2, $4); } @@ -423,21 +429,21 @@ expr_op | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {new ExprInt(0), $2}); } | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); } | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); } - | expr_op '<' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__lessThan")), {$1, $3}); } - | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__lessThan")), {$3, $1})); } - | expr_op '>' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__lessThan")), {$3, $1}); } - | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__lessThan")), {$1, $3})); } - | expr_op AND expr_op { $$ = new ExprOpAnd(makeCurPos(@2, data), $1, $3); } - | expr_op OR expr_op { $$ = new ExprOpOr(makeCurPos(@2, data), $1, $3); } - | expr_op IMPL expr_op { $$ = new ExprOpImpl(makeCurPos(@2, data), $1, $3); } - | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(makeCurPos(@2, data), $1, $3); } + | expr_op '<' expr_op { $$ = new ExprCall(data->makeCurPos(@2), new ExprVar(data->symbols.create("__lessThan")), {$1, $3}); } + | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(data->makeCurPos(@2), new ExprVar(data->symbols.create("__lessThan")), {$3, $1})); } + | expr_op '>' expr_op { $$ = new ExprCall(data->makeCurPos(@2), new ExprVar(data->symbols.create("__lessThan")), {$3, $1}); } + | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(data->makeCurPos(@2), new ExprVar(data->symbols.create("__lessThan")), {$1, $3})); } + | expr_op AND expr_op { $$ = new ExprOpAnd(data->makeCurPos(@2), $1, $3); } + | expr_op OR expr_op { $$ = new ExprOpOr(data->makeCurPos(@2), $1, $3); } + | expr_op IMPL expr_op { $$ = new ExprOpImpl(data->makeCurPos(@2), $1, $3); } + | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(data->makeCurPos(@2), $1, $3); } | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, std::move(*$3)); delete $3; } | expr_op '+' expr_op - { $$ = new ExprConcatStrings(makeCurPos(@2, data), false, new std::vector >({{makeCurPos(@1, data), $1}, {makeCurPos(@3, data), $3}})); } - | expr_op '-' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__sub")), {$1, $3}); } - | expr_op '*' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__mul")), {$1, $3}); } - | expr_op '/' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__div")), {$1, $3}); } - | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(makeCurPos(@2, data), $1, $3); } + { $$ = new ExprConcatStrings(data->makeCurPos(@2), false, new std::vector >({{data->makeCurPos(@1), $1}, {data->makeCurPos(@3), $3}})); } + | expr_op '-' expr_op { $$ = new ExprCall(data->makeCurPos(@2), new ExprVar(data->symbols.create("__sub")), {$1, $3}); } + | expr_op '*' expr_op { $$ = new ExprCall(data->makeCurPos(@2), new ExprVar(data->symbols.create("__mul")), {$1, $3}); } + | expr_op '/' expr_op { $$ = new ExprCall(data->makeCurPos(@2), new ExprVar(data->symbols.create("__div")), {$1, $3}); } + | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(data->makeCurPos(@2), $1, $3); } | expr_app ; @@ -476,12 +482,12 @@ expr_simple | FLOAT_LIT { $$ = new ExprFloat($1); } | '"' string_parts '"' { $$ = $2; } | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE { - $$ = stripIndentation(CUR_POS, data->symbols, std::move(*$2)); + $$ = data->stripIndentation(CUR_POS, std::move(*$2)); delete $2; } | path_start PATH_END | path_start string_parts_interpolated PATH_END { - $2->insert($2->begin(), {makeCurPos(@1, data), $1}); + $2->insert($2->begin(), {data->makeCurPos(@1), $1}); $$ = new ExprConcatStrings(CUR_POS, false, $2); } | SPATH { @@ -520,13 +526,13 @@ string_parts string_parts_interpolated : string_parts_interpolated STR - { $$ = $1; $1->emplace_back(makeCurPos(@2, data), new ExprString(std::string($2))); } - | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); } - | DOLLAR_CURLY expr '}' { $$ = new std::vector>; $$->emplace_back(makeCurPos(@1, data), $2); } + { $$ = $1; $1->emplace_back(data->makeCurPos(@2), new ExprString(std::string($2))); } + | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(data->makeCurPos(@2), $3); } + | DOLLAR_CURLY expr '}' { $$ = new std::vector>; $$->emplace_back(data->makeCurPos(@1), $2); } | STR DOLLAR_CURLY expr '}' { $$ = new std::vector>; - $$->emplace_back(makeCurPos(@1, data), new ExprString(std::string($1))); - $$->emplace_back(makeCurPos(@2, data), $3); + $$->emplace_back(data->makeCurPos(@1), new ExprString(std::string($1))); + $$->emplace_back(data->makeCurPos(@2), $3); } ; @@ -551,19 +557,19 @@ path_start ; ind_string_parts - : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $2); } - | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); } + : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(data->makeCurPos(@2), $2); } + | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(data->makeCurPos(@2), $3); } | { $$ = new std::vector>>; } ; binds - : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, std::move(*$2), $4, makeCurPos(@2, data), data->state); delete $2; } + : binds attrpath '=' expr ';' { $$ = $1; data->addAttr($$, std::move(*$2), $4, data->makeCurPos(@2)); delete $2; } | binds INHERIT attrs ';' { $$ = $1; for (auto & i : *$3) { if ($$->attrs.find(i.symbol) != $$->attrs.end()) - dupAttr(data->state, i.symbol, makeCurPos(@3, data), $$->attrs[i.symbol].pos); - auto pos = makeCurPos(@3, data); + data->dupAttr(i.symbol, data->makeCurPos(@3), $$->attrs[i.symbol].pos); + auto pos = data->makeCurPos(@3); $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true)); } delete $3; @@ -573,12 +579,12 @@ binds /* !!! Should ensure sharing of the expression in $4. */ for (auto & i : *$6) { if ($$->attrs.find(i.symbol) != $$->attrs.end()) - dupAttr(data->state, i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos); - $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data))); + data->dupAttr(i.symbol, data->makeCurPos(@6), $$->attrs[i.symbol].pos); + $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), data->makeCurPos(@6))); } delete $6; } - | { $$ = new ExprAttrs(makeCurPos(@0, data)); } + | { $$ = new ExprAttrs(data->makeCurPos(@0)); } ; attrs @@ -592,7 +598,7 @@ attrs } else throw ParseError({ .msg = hintfmt("dynamic attributes not allowed in inherit"), - .errPos = data->state.positions[makeCurPos(@2, data)] + .errPos = data->state.positions[data->makeCurPos(@2)] }); } | { $$ = new AttrPath; } From 007605616477f4f0d8a0064c375b1d3cf6188ac5 Mon Sep 17 00:00:00 2001 From: pennae Date: Mon, 15 Jan 2024 16:52:18 +0100 Subject: [PATCH 04/10] move ParseData to own header, rename to ParserState ParserState better describes what this struct really is. the parser really does modify its state (most notably position and symbol tables), so calling it that rather than obliquely "data" (which implies being input only) makes sense. --- src/libexpr/lexer.l | 17 +- src/libexpr/parser-state.hh | 262 +++++++++++++++++++++++ src/libexpr/parser.y | 413 +++++++----------------------------- 3 files changed, 339 insertions(+), 353 deletions(-) create mode 100644 src/libexpr/parser-state.hh diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 9addb3ae8..cfd61c90e 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -29,12 +29,7 @@ using namespace nix; namespace nix { -static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data) -{ - return data->state.positions.add(data->origin, loc.first_line, loc.first_column); -} - -#define CUR_POS makeCurPos(*yylloc, data) +#define CUR_POS state->makeCurPos(*yylloc) static void initLoc(YYLTYPE * loc) { @@ -153,7 +148,7 @@ or { return OR_KW; } } catch (const boost::bad_lexical_cast &) { throw ParseError({ .msg = hintfmt("invalid integer '%1%'", yytext), - .errPos = data->state.positions[CUR_POS], + .errPos = state->state.positions[CUR_POS], }); } return INT_LIT; @@ -163,7 +158,7 @@ or { return OR_KW; } if (errno != 0) throw ParseError({ .msg = hintfmt("invalid float '%1%'", yytext), - .errPos = data->state.positions[CUR_POS], + .errPos = state->state.positions[CUR_POS], }); return FLOAT_LIT; } @@ -186,7 +181,7 @@ or { return OR_KW; } /* It is impossible to match strings ending with '$' with one regex because trailing contexts are only valid at the end of a rule. (A sane but undocumented limitation.) */ - yylval->str = unescapeStr(data->symbols, yytext, yyleng); + yylval->str = unescapeStr(state->symbols, yytext, yyleng); return STR; } \$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } @@ -214,7 +209,7 @@ or { return OR_KW; } return IND_STR; } \'\'\\{ANY} { - yylval->str = unescapeStr(data->symbols, yytext + 2, yyleng - 2); + yylval->str = unescapeStr(state->symbols, yytext + 2, yyleng - 2); return IND_STR; } \$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } @@ -292,7 +287,7 @@ or { return OR_KW; } <> { throw ParseError({ .msg = hintfmt("path has a trailing slash"), - .errPos = data->state.positions[CUR_POS], + .errPos = state->state.positions[CUR_POS], }); } diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh new file mode 100644 index 000000000..b33311743 --- /dev/null +++ b/src/libexpr/parser-state.hh @@ -0,0 +1,262 @@ +#pragma once + +#include "eval.hh" + +namespace nix { + +// using C a struct allows us to avoid having to define the special +// members that using string_view here would implicitly delete. +struct StringToken { + const char * p; + size_t l; + bool hasIndentation; + operator std::string_view() const { return {p, l}; } +}; + +struct ParserLocation { + int first_line, first_column; + int last_line, last_column; + + // backup to recover from yyless(0) + int stashed_first_line, stashed_first_column; + int stashed_last_line, stashed_last_column; + + void stash() { + stashed_first_line = first_line; + stashed_first_column = first_column; + stashed_last_line = last_line; + stashed_last_column = last_column; + } + + void unstash() { + first_line = stashed_first_line; + first_column = stashed_first_column; + last_line = stashed_last_line; + last_column = stashed_last_column; + } +}; + +struct ParserState { + EvalState & state; + SymbolTable & symbols; + Expr * result; + SourcePath basePath; + PosTable::Origin origin; + + void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos); + void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos); + void addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos); + Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {}); + Expr * stripIndentation(const PosIdx pos, + std::vector>> && es); + PosIdx makeCurPos(const ParserLocation & loc); +}; + +inline void ParserState::dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos) +{ + throw ParseError({ + .msg = hintfmt("attribute '%1%' already defined at %2%", + showAttrPath(state.symbols, attrPath), state.positions[prevPos]), + .errPos = state.positions[pos] + }); +} + +inline void ParserState::dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos) +{ + throw ParseError({ + .msg = hintfmt("attribute '%1%' already defined at %2%", state.symbols[attr], state.positions[prevPos]), + .errPos = state.positions[pos] + }); +} + +inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos) +{ + AttrPath::iterator i; + // All attrpaths have at least one attr + assert(!attrPath.empty()); + // Checking attrPath validity. + // =========================== + for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { + if (i->symbol) { + ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); + if (j != attrs->attrs.end()) { + if (!j->second.inherited) { + ExprAttrs * attrs2 = dynamic_cast(j->second.e); + if (!attrs2) dupAttr(attrPath, pos, j->second.pos); + attrs = attrs2; + } else + dupAttr(attrPath, pos, j->second.pos); + } else { + ExprAttrs * nested = new ExprAttrs; + attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos); + attrs = nested; + } + } else { + ExprAttrs *nested = new ExprAttrs; + attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos)); + attrs = nested; + } + } + // Expr insertion. + // ========================== + if (i->symbol) { + ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); + if (j != attrs->attrs.end()) { + // This attr path is already defined. However, if both + // e and the expr pointed by the attr path are two attribute sets, + // we want to merge them. + // Otherwise, throw an error. + auto ae = dynamic_cast(e); + auto jAttrs = dynamic_cast(j->second.e); + if (jAttrs && ae) { + for (auto & ad : ae->attrs) { + auto j2 = jAttrs->attrs.find(ad.first); + if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. + dupAttr(ad.first, j2->second.pos, ad.second.pos); + jAttrs->attrs.emplace(ad.first, ad.second); + } + jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end()); + } else { + dupAttr(attrPath, pos, j->second.pos); + } + } else { + // This attr path is not defined. Let's create it. + attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos)); + e->setName(i->symbol); + } + } else { + attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); + } +} + +inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg) +{ + std::sort(formals->formals.begin(), formals->formals.end(), + [] (const auto & a, const auto & b) { + return std::tie(a.name, a.pos) < std::tie(b.name, b.pos); + }); + + std::optional> duplicate; + for (size_t i = 0; i + 1 < formals->formals.size(); i++) { + if (formals->formals[i].name != formals->formals[i + 1].name) + continue; + std::pair thisDup{formals->formals[i].name, formals->formals[i + 1].pos}; + duplicate = std::min(thisDup, duplicate.value_or(thisDup)); + } + if (duplicate) + throw ParseError({ + .msg = hintfmt("duplicate formal function argument '%1%'", symbols[duplicate->first]), + .errPos = state.positions[duplicate->second] + }); + + if (arg && formals->has(arg)) + throw ParseError({ + .msg = hintfmt("duplicate formal function argument '%1%'", symbols[arg]), + .errPos = state.positions[pos] + }); + + return formals; +} + +inline Expr * ParserState::stripIndentation(const PosIdx pos, + std::vector>> && es) +{ + if (es.empty()) return new ExprString(""); + + /* Figure out the minimum indentation. Note that by design + whitespace-only final lines are not taken into account. (So + the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */ + bool atStartOfLine = true; /* = seen only whitespace in the current line */ + size_t minIndent = 1000000; + size_t curIndent = 0; + for (auto & [i_pos, i] : es) { + auto * str = std::get_if(&i); + if (!str || !str->hasIndentation) { + /* Anti-quotations and escaped characters end the current start-of-line whitespace. */ + if (atStartOfLine) { + atStartOfLine = false; + if (curIndent < minIndent) minIndent = curIndent; + } + continue; + } + for (size_t j = 0; j < str->l; ++j) { + if (atStartOfLine) { + if (str->p[j] == ' ') + curIndent++; + else if (str->p[j] == '\n') { + /* Empty line, doesn't influence minimum + indentation. */ + curIndent = 0; + } else { + atStartOfLine = false; + if (curIndent < minIndent) minIndent = curIndent; + } + } else if (str->p[j] == '\n') { + atStartOfLine = true; + curIndent = 0; + } + } + } + + /* Strip spaces from each line. */ + auto * es2 = new std::vector>; + atStartOfLine = true; + size_t curDropped = 0; + size_t n = es.size(); + auto i = es.begin(); + const auto trimExpr = [&] (Expr * e) { + atStartOfLine = false; + curDropped = 0; + es2->emplace_back(i->first, e); + }; + const auto trimString = [&] (const StringToken & t) { + std::string s2; + for (size_t j = 0; j < t.l; ++j) { + if (atStartOfLine) { + if (t.p[j] == ' ') { + if (curDropped++ >= minIndent) + s2 += t.p[j]; + } + else if (t.p[j] == '\n') { + curDropped = 0; + s2 += t.p[j]; + } else { + atStartOfLine = false; + curDropped = 0; + s2 += t.p[j]; + } + } else { + s2 += t.p[j]; + if (t.p[j] == '\n') atStartOfLine = true; + } + } + + /* Remove the last line if it is empty and consists only of + spaces. */ + if (n == 1) { + std::string::size_type p = s2.find_last_of('\n'); + if (p != std::string::npos && s2.find_first_not_of(' ', p + 1) == std::string::npos) + s2 = std::string(s2, 0, p + 1); + } + + es2->emplace_back(i->first, new ExprString(std::move(s2))); + }; + for (; i != es.end(); ++i, --n) { + std::visit(overloaded { trimExpr, trimString }, i->second); + } + + /* If this is a single string, then don't do a concatenation. */ + if (es2->size() == 1 && dynamic_cast((*es2)[0].second)) { + auto *const result = (*es2)[0].second; + delete es2; + return result; + } + return new ExprConcatStrings(pos, true, es2); +} + +inline PosIdx ParserState::makeCurPos(const ParserLocation & loc) +{ + return state.positions.add(origin, loc.first_line, loc.first_column); +} + +} diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index beb660e36..7ce493df5 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -5,9 +5,9 @@ %defines /* %no-lines */ %parse-param { void * scanner } -%parse-param { nix::ParseData * data } +%parse-param { nix::ParserState * state } %lex-param { void * scanner } -%lex-param { nix::ParseData * data } +%lex-param { nix::ParserState * state } %expect 1 %expect-rr 1 @@ -26,64 +26,11 @@ #include "eval.hh" #include "eval-settings.hh" #include "globals.hh" - -// using C a struct allows us to avoid having to define the special -// members that using string_view here would implicitly delete. -struct StringToken { - const char * p; - size_t l; - bool hasIndentation; - operator std::string_view() const { return {p, l}; } -}; - -namespace nix { +#include "parser-state.hh" #define YYLTYPE ::nix::ParserLocation - struct ParserLocation - { - int first_line, first_column; - int last_line, last_column; - - // backup to recover from yyless(0) - int stashed_first_line, stashed_first_column; - int stashed_last_line, stashed_last_column; - - void stash() { - stashed_first_line = first_line; - stashed_first_column = first_column; - stashed_last_line = last_line; - stashed_last_column = last_column; - } - - void unstash() { - first_line = stashed_first_line; - first_column = stashed_first_column; - last_line = stashed_last_line; - last_column = stashed_last_column; - } - }; - - struct ParseData - { - EvalState & state; - SymbolTable & symbols; - Expr * result; - SourcePath basePath; - PosTable::Origin origin; - - void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos); - void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos); - void addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos); - Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {}); - Expr * stripIndentation(const PosIdx pos, - std::vector>> && es); - PosIdx makeCurPos(const ParserLocation & loc); - }; - -} - #define YY_DECL int yylex \ - (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParseData * data) + (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state) #endif @@ -98,232 +45,14 @@ YY_DECL; using namespace nix; - -namespace nix { +#define CUR_POS state->makeCurPos(*yylocp) -void ParseData::dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos) -{ - throw ParseError({ - .msg = hintfmt("attribute '%1%' already defined at %2%", - showAttrPath(state.symbols, attrPath), state.positions[prevPos]), - .errPos = state.positions[pos] - }); -} - -void ParseData::dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos) -{ - throw ParseError({ - .msg = hintfmt("attribute '%1%' already defined at %2%", state.symbols[attr], state.positions[prevPos]), - .errPos = state.positions[pos] - }); -} - - -void ParseData::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos) -{ - AttrPath::iterator i; - // All attrpaths have at least one attr - assert(!attrPath.empty()); - // Checking attrPath validity. - // =========================== - for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { - if (i->symbol) { - ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); - if (j != attrs->attrs.end()) { - if (!j->second.inherited) { - ExprAttrs * attrs2 = dynamic_cast(j->second.e); - if (!attrs2) dupAttr(attrPath, pos, j->second.pos); - attrs = attrs2; - } else - dupAttr(attrPath, pos, j->second.pos); - } else { - ExprAttrs * nested = new ExprAttrs; - attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos); - attrs = nested; - } - } else { - ExprAttrs *nested = new ExprAttrs; - attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos)); - attrs = nested; - } - } - // Expr insertion. - // ========================== - if (i->symbol) { - ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); - if (j != attrs->attrs.end()) { - // This attr path is already defined. However, if both - // e and the expr pointed by the attr path are two attribute sets, - // we want to merge them. - // Otherwise, throw an error. - auto ae = dynamic_cast(e); - auto jAttrs = dynamic_cast(j->second.e); - if (jAttrs && ae) { - for (auto & ad : ae->attrs) { - auto j2 = jAttrs->attrs.find(ad.first); - if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. - dupAttr(ad.first, j2->second.pos, ad.second.pos); - jAttrs->attrs.emplace(ad.first, ad.second); - } - jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end()); - } else { - dupAttr(attrPath, pos, j->second.pos); - } - } else { - // This attr path is not defined. Let's create it. - attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos)); - e->setName(i->symbol); - } - } else { - attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); - } -} - - -Formals * ParseData::validateFormals(Formals * formals, PosIdx pos, Symbol arg) -{ - std::sort(formals->formals.begin(), formals->formals.end(), - [] (const auto & a, const auto & b) { - return std::tie(a.name, a.pos) < std::tie(b.name, b.pos); - }); - - std::optional> duplicate; - for (size_t i = 0; i + 1 < formals->formals.size(); i++) { - if (formals->formals[i].name != formals->formals[i + 1].name) - continue; - std::pair thisDup{formals->formals[i].name, formals->formals[i + 1].pos}; - duplicate = std::min(thisDup, duplicate.value_or(thisDup)); - } - if (duplicate) - throw ParseError({ - .msg = hintfmt("duplicate formal function argument '%1%'", symbols[duplicate->first]), - .errPos = state.positions[duplicate->second] - }); - - if (arg && formals->has(arg)) - throw ParseError({ - .msg = hintfmt("duplicate formal function argument '%1%'", symbols[arg]), - .errPos = state.positions[pos] - }); - - return formals; -} - - -Expr * ParseData::stripIndentation(const PosIdx pos, - std::vector>> && es) -{ - if (es.empty()) return new ExprString(""); - - /* Figure out the minimum indentation. Note that by design - whitespace-only final lines are not taken into account. (So - the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */ - bool atStartOfLine = true; /* = seen only whitespace in the current line */ - size_t minIndent = 1000000; - size_t curIndent = 0; - for (auto & [i_pos, i] : es) { - auto * str = std::get_if(&i); - if (!str || !str->hasIndentation) { - /* Anti-quotations and escaped characters end the current start-of-line whitespace. */ - if (atStartOfLine) { - atStartOfLine = false; - if (curIndent < minIndent) minIndent = curIndent; - } - continue; - } - for (size_t j = 0; j < str->l; ++j) { - if (atStartOfLine) { - if (str->p[j] == ' ') - curIndent++; - else if (str->p[j] == '\n') { - /* Empty line, doesn't influence minimum - indentation. */ - curIndent = 0; - } else { - atStartOfLine = false; - if (curIndent < minIndent) minIndent = curIndent; - } - } else if (str->p[j] == '\n') { - atStartOfLine = true; - curIndent = 0; - } - } - } - - /* Strip spaces from each line. */ - auto * es2 = new std::vector>; - atStartOfLine = true; - size_t curDropped = 0; - size_t n = es.size(); - auto i = es.begin(); - const auto trimExpr = [&] (Expr * e) { - atStartOfLine = false; - curDropped = 0; - es2->emplace_back(i->first, e); - }; - const auto trimString = [&] (const StringToken & t) { - std::string s2; - for (size_t j = 0; j < t.l; ++j) { - if (atStartOfLine) { - if (t.p[j] == ' ') { - if (curDropped++ >= minIndent) - s2 += t.p[j]; - } - else if (t.p[j] == '\n') { - curDropped = 0; - s2 += t.p[j]; - } else { - atStartOfLine = false; - curDropped = 0; - s2 += t.p[j]; - } - } else { - s2 += t.p[j]; - if (t.p[j] == '\n') atStartOfLine = true; - } - } - - /* Remove the last line if it is empty and consists only of - spaces. */ - if (n == 1) { - std::string::size_type p = s2.find_last_of('\n'); - if (p != std::string::npos && s2.find_first_not_of(' ', p + 1) == std::string::npos) - s2 = std::string(s2, 0, p + 1); - } - - es2->emplace_back(i->first, new ExprString(std::move(s2))); - }; - for (; i != es.end(); ++i, --n) { - std::visit(overloaded { trimExpr, trimString }, i->second); - } - - /* If this is a single string, then don't do a concatenation. */ - if (es2->size() == 1 && dynamic_cast((*es2)[0].second)) { - auto *const result = (*es2)[0].second; - delete es2; - return result; - } - return new ExprConcatStrings(pos, true, es2); -} - - -PosIdx ParseData::makeCurPos(const ParserLocation & loc) -{ - return state.positions.add(origin, loc.first_line, loc.first_column); -} - -#define CUR_POS data->makeCurPos(*yylocp) - - -} - - -void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error) +void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error) { throw ParseError({ .msg = hintfmt(error), - .errPos = data->state.positions[data->makeCurPos(*loc)] + .errPos = state->state.positions[state->makeCurPos(*loc)] }); } @@ -339,13 +68,13 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err nix::Formal * formal; nix::NixInt n; nix::NixFloat nf; - StringToken id; // !!! -> Symbol - StringToken path; - StringToken uri; - StringToken str; + nix::StringToken id; // !!! -> Symbol + nix::StringToken path; + nix::StringToken uri; + nix::StringToken str; std::vector * attrNames; std::vector> * string_parts; - std::vector>> * ind_string_parts; + std::vector>> * ind_string_parts; } %type start expr expr_function expr_if expr_op @@ -385,24 +114,24 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err %% -start: expr { data->result = $1; }; +start: expr { state->result = $1; }; expr: expr_function; expr_function : ID ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), 0, $3); } + { $$ = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3); } | '{' formals '}' ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->validateFormals($2), $5); } + { $$ = new ExprLambda(CUR_POS, state->validateFormals($2), $5); } | '{' formals '}' '@' ID ':' expr_function { - auto arg = data->symbols.create($5); - $$ = new ExprLambda(CUR_POS, arg, data->validateFormals($2, CUR_POS, arg), $7); + auto arg = state->symbols.create($5); + $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7); } | ID '@' '{' formals '}' ':' expr_function { - auto arg = data->symbols.create($1); - $$ = new ExprLambda(CUR_POS, arg, data->validateFormals($4, CUR_POS, arg), $7); + auto arg = state->symbols.create($1); + $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7); } | ASSERT expr ';' expr_function { $$ = new ExprAssert(CUR_POS, $2, $4); } @@ -412,7 +141,7 @@ expr_function { if (!$2->dynamicAttrs.empty()) throw ParseError({ .msg = hintfmt("dynamic attributes not allowed in let"), - .errPos = data->state.positions[CUR_POS] + .errPos = state->state.positions[CUR_POS] }); $$ = new ExprLet($2, $4); } @@ -426,24 +155,24 @@ expr_if expr_op : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); } - | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {new ExprInt(0), $2}); } + | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(state->symbols.create("__sub")), {new ExprInt(0), $2}); } | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); } | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); } - | expr_op '<' expr_op { $$ = new ExprCall(data->makeCurPos(@2), new ExprVar(data->symbols.create("__lessThan")), {$1, $3}); } - | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(data->makeCurPos(@2), new ExprVar(data->symbols.create("__lessThan")), {$3, $1})); } - | expr_op '>' expr_op { $$ = new ExprCall(data->makeCurPos(@2), new ExprVar(data->symbols.create("__lessThan")), {$3, $1}); } - | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(data->makeCurPos(@2), new ExprVar(data->symbols.create("__lessThan")), {$1, $3})); } - | expr_op AND expr_op { $$ = new ExprOpAnd(data->makeCurPos(@2), $1, $3); } - | expr_op OR expr_op { $$ = new ExprOpOr(data->makeCurPos(@2), $1, $3); } - | expr_op IMPL expr_op { $$ = new ExprOpImpl(data->makeCurPos(@2), $1, $3); } - | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(data->makeCurPos(@2), $1, $3); } + | expr_op '<' expr_op { $$ = new ExprCall(state->makeCurPos(@2), new ExprVar(state->symbols.create("__lessThan")), {$1, $3}); } + | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->makeCurPos(@2), new ExprVar(state->symbols.create("__lessThan")), {$3, $1})); } + | expr_op '>' expr_op { $$ = new ExprCall(state->makeCurPos(@2), new ExprVar(state->symbols.create("__lessThan")), {$3, $1}); } + | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->makeCurPos(@2), new ExprVar(state->symbols.create("__lessThan")), {$1, $3})); } + | expr_op AND expr_op { $$ = new ExprOpAnd(state->makeCurPos(@2), $1, $3); } + | expr_op OR expr_op { $$ = new ExprOpOr(state->makeCurPos(@2), $1, $3); } + | expr_op IMPL expr_op { $$ = new ExprOpImpl(state->makeCurPos(@2), $1, $3); } + | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(state->makeCurPos(@2), $1, $3); } | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, std::move(*$3)); delete $3; } | expr_op '+' expr_op - { $$ = new ExprConcatStrings(data->makeCurPos(@2), false, new std::vector >({{data->makeCurPos(@1), $1}, {data->makeCurPos(@3), $3}})); } - | expr_op '-' expr_op { $$ = new ExprCall(data->makeCurPos(@2), new ExprVar(data->symbols.create("__sub")), {$1, $3}); } - | expr_op '*' expr_op { $$ = new ExprCall(data->makeCurPos(@2), new ExprVar(data->symbols.create("__mul")), {$1, $3}); } - | expr_op '/' expr_op { $$ = new ExprCall(data->makeCurPos(@2), new ExprVar(data->symbols.create("__div")), {$1, $3}); } - | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(data->makeCurPos(@2), $1, $3); } + { $$ = new ExprConcatStrings(state->makeCurPos(@2), false, new std::vector >({{state->makeCurPos(@1), $1}, {state->makeCurPos(@3), $3}})); } + | expr_op '-' expr_op { $$ = new ExprCall(state->makeCurPos(@2), new ExprVar(state->symbols.create("__sub")), {$1, $3}); } + | expr_op '*' expr_op { $$ = new ExprCall(state->makeCurPos(@2), new ExprVar(state->symbols.create("__mul")), {$1, $3}); } + | expr_op '/' expr_op { $$ = new ExprCall(state->makeCurPos(@2), new ExprVar(state->symbols.create("__div")), {$1, $3}); } + | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(state->makeCurPos(@2), $1, $3); } | expr_app ; @@ -466,7 +195,7 @@ expr_select | /* Backwards compatibility: because Nixpkgs has a rarely used function named ‘or’, allow stuff like ‘map or [...]’. */ expr_simple OR_KW - { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, data->symbols.create("or"))}); } + { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, state->symbols.create("or"))}); } | expr_simple ; @@ -476,25 +205,25 @@ expr_simple if ($1.l == s.size() && strncmp($1.p, s.data(), s.size()) == 0) $$ = new ExprPos(CUR_POS); else - $$ = new ExprVar(CUR_POS, data->symbols.create($1)); + $$ = new ExprVar(CUR_POS, state->symbols.create($1)); } | INT_LIT { $$ = new ExprInt($1); } | FLOAT_LIT { $$ = new ExprFloat($1); } | '"' string_parts '"' { $$ = $2; } | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE { - $$ = data->stripIndentation(CUR_POS, std::move(*$2)); + $$ = state->stripIndentation(CUR_POS, std::move(*$2)); delete $2; } | path_start PATH_END | path_start string_parts_interpolated PATH_END { - $2->insert($2->begin(), {data->makeCurPos(@1), $1}); + $2->insert($2->begin(), {state->makeCurPos(@1), $1}); $$ = new ExprConcatStrings(CUR_POS, false, $2); } | SPATH { std::string path($1.p + 1, $1.l - 2); $$ = new ExprCall(CUR_POS, - new ExprVar(data->symbols.create("__findFile")), - {new ExprVar(data->symbols.create("__nixPath")), + new ExprVar(state->symbols.create("__findFile")), + {new ExprVar(state->symbols.create("__nixPath")), new ExprString(std::move(path))}); } | URI { @@ -502,7 +231,7 @@ expr_simple if (noURLLiterals) throw ParseError({ .msg = hintfmt("URL literals are disabled"), - .errPos = data->state.positions[CUR_POS] + .errPos = state->state.positions[CUR_POS] }); $$ = new ExprString(std::string($1)); } @@ -510,7 +239,7 @@ expr_simple /* Let expressions `let {..., body = ...}' are just desugared into `(rec {..., body = ...}).body'. */ | LET '{' binds '}' - { $3->recursive = true; $$ = new ExprSelect(noPos, $3, data->symbols.create("body")); } + { $3->recursive = true; $$ = new ExprSelect(noPos, $3, state->symbols.create("body")); } | REC '{' binds '}' { $3->recursive = true; $$ = $3; } | '{' binds '}' @@ -526,23 +255,23 @@ string_parts string_parts_interpolated : string_parts_interpolated STR - { $$ = $1; $1->emplace_back(data->makeCurPos(@2), new ExprString(std::string($2))); } - | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(data->makeCurPos(@2), $3); } - | DOLLAR_CURLY expr '}' { $$ = new std::vector>; $$->emplace_back(data->makeCurPos(@1), $2); } + { $$ = $1; $1->emplace_back(state->makeCurPos(@2), new ExprString(std::string($2))); } + | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->makeCurPos(@2), $3); } + | DOLLAR_CURLY expr '}' { $$ = new std::vector>; $$->emplace_back(state->makeCurPos(@1), $2); } | STR DOLLAR_CURLY expr '}' { $$ = new std::vector>; - $$->emplace_back(data->makeCurPos(@1), new ExprString(std::string($1))); - $$->emplace_back(data->makeCurPos(@2), $3); + $$->emplace_back(state->makeCurPos(@1), new ExprString(std::string($1))); + $$->emplace_back(state->makeCurPos(@2), $3); } ; path_start : PATH { - Path path(absPath({$1.p, $1.l}, data->basePath.path.abs())); + Path path(absPath({$1.p, $1.l}, state->basePath.path.abs())); /* add back in the trailing '/' to the first segment */ if ($1.p[$1.l-1] == '/' && $1.l > 1) path += "/"; - $$ = new ExprPath(ref(data->state.rootFS), std::move(path)); + $$ = new ExprPath(ref(state->state.rootFS), std::move(path)); } | HPATH { if (evalSettings.pureEval) { @@ -552,24 +281,24 @@ path_start ); } Path path(getHome() + std::string($1.p + 1, $1.l - 1)); - $$ = new ExprPath(ref(data->state.rootFS), std::move(path)); + $$ = new ExprPath(ref(state->state.rootFS), std::move(path)); } ; ind_string_parts - : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(data->makeCurPos(@2), $2); } - | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(data->makeCurPos(@2), $3); } + : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(state->makeCurPos(@2), $2); } + | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->makeCurPos(@2), $3); } | { $$ = new std::vector>>; } ; binds - : binds attrpath '=' expr ';' { $$ = $1; data->addAttr($$, std::move(*$2), $4, data->makeCurPos(@2)); delete $2; } + : binds attrpath '=' expr ';' { $$ = $1; state->addAttr($$, std::move(*$2), $4, state->makeCurPos(@2)); delete $2; } | binds INHERIT attrs ';' { $$ = $1; for (auto & i : *$3) { if ($$->attrs.find(i.symbol) != $$->attrs.end()) - data->dupAttr(i.symbol, data->makeCurPos(@3), $$->attrs[i.symbol].pos); - auto pos = data->makeCurPos(@3); + state->dupAttr(i.symbol, state->makeCurPos(@3), $$->attrs[i.symbol].pos); + auto pos = state->makeCurPos(@3); $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true)); } delete $3; @@ -579,48 +308,48 @@ binds /* !!! Should ensure sharing of the expression in $4. */ for (auto & i : *$6) { if ($$->attrs.find(i.symbol) != $$->attrs.end()) - data->dupAttr(i.symbol, data->makeCurPos(@6), $$->attrs[i.symbol].pos); - $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), data->makeCurPos(@6))); + state->dupAttr(i.symbol, state->makeCurPos(@6), $$->attrs[i.symbol].pos); + $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), state->makeCurPos(@6))); } delete $6; } - | { $$ = new ExprAttrs(data->makeCurPos(@0)); } + | { $$ = new ExprAttrs(state->makeCurPos(@0)); } ; attrs - : attrs attr { $$ = $1; $1->push_back(AttrName(data->symbols.create($2))); } + : attrs attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($2))); } | attrs string_attr { $$ = $1; ExprString * str = dynamic_cast($2); if (str) { - $$->push_back(AttrName(data->symbols.create(str->s))); + $$->push_back(AttrName(state->symbols.create(str->s))); delete str; } else throw ParseError({ .msg = hintfmt("dynamic attributes not allowed in inherit"), - .errPos = data->state.positions[data->makeCurPos(@2)] + .errPos = state->state.positions[state->makeCurPos(@2)] }); } | { $$ = new AttrPath; } ; attrpath - : attrpath '.' attr { $$ = $1; $1->push_back(AttrName(data->symbols.create($3))); } + : attrpath '.' attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($3))); } | attrpath '.' string_attr { $$ = $1; ExprString * str = dynamic_cast($3); if (str) { - $$->push_back(AttrName(data->symbols.create(str->s))); + $$->push_back(AttrName(state->symbols.create(str->s))); delete str; } else $$->push_back(AttrName($3)); } - | attr { $$ = new std::vector; $$->push_back(AttrName(data->symbols.create($1))); } + | attr { $$ = new std::vector; $$->push_back(AttrName(state->symbols.create($1))); } | string_attr { $$ = new std::vector; ExprString *str = dynamic_cast($1); if (str) { - $$->push_back(AttrName(data->symbols.create(str->s))); + $$->push_back(AttrName(state->symbols.create(str->s))); delete str; } else $$->push_back(AttrName($1)); @@ -654,8 +383,8 @@ formals ; formal - : ID { $$ = new Formal{CUR_POS, data->symbols.create($1), 0}; } - | ID '?' expr { $$ = new Formal{CUR_POS, data->symbols.create($1), $3}; } + : ID { $$ = new Formal{CUR_POS, state->symbols.create($1), 0}; } + | ID '?' expr { $$ = new Formal{CUR_POS, state->symbols.create($1), $3}; } ; %% @@ -687,7 +416,7 @@ Expr * EvalState::parse( std::shared_ptr & staticEnv) { yyscan_t scanner; - ParseData data { + ParserState state { .state = *this, .symbols = symbols, .basePath = basePath, @@ -698,11 +427,11 @@ Expr * EvalState::parse( Finally _destroy([&] { yylex_destroy(scanner); }); yy_scan_buffer(text, length, scanner); - yyparse(scanner, &data); + yyparse(scanner, &state); - data.result->bindVars(*this, staticEnv); + state.result->bindVars(*this, staticEnv); - return data.result; + return state.result; } From 835a6c7bcfd0b22acc16f31de5fc7bb650d52017 Mon Sep 17 00:00:00 2001 From: pennae Date: Mon, 15 Jan 2024 16:52:18 +0100 Subject: [PATCH 05/10] rename ParserState::{makeCurPos -> at} most instances of this being used do not refer to the "current" position, sometimes not even to one reasonably close by. it could also be called `makePos` instead, but `at` seems clear in context. --- src/libexpr/lexer.l | 2 +- src/libexpr/parser-state.hh | 4 +-- src/libexpr/parser.y | 60 ++++++++++++++++++------------------- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index cfd61c90e..fae0e7a85 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -29,7 +29,7 @@ using namespace nix; namespace nix { -#define CUR_POS state->makeCurPos(*yylloc) +#define CUR_POS state->at(*yylloc) static void initLoc(YYLTYPE * loc) { diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index b33311743..167d3f4ae 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -49,7 +49,7 @@ struct ParserState { Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {}); Expr * stripIndentation(const PosIdx pos, std::vector>> && es); - PosIdx makeCurPos(const ParserLocation & loc); + PosIdx at(const ParserLocation & loc); }; inline void ParserState::dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos) @@ -254,7 +254,7 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos, return new ExprConcatStrings(pos, true, es2); } -inline PosIdx ParserState::makeCurPos(const ParserLocation & loc) +inline PosIdx ParserState::at(const ParserLocation & loc) { return state.positions.add(origin, loc.first_line, loc.first_column); } diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 7ce493df5..7763a72bc 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -45,14 +45,14 @@ YY_DECL; using namespace nix; -#define CUR_POS state->makeCurPos(*yylocp) +#define CUR_POS state->at(*yylocp) void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error) { throw ParseError({ .msg = hintfmt(error), - .errPos = state->state.positions[state->makeCurPos(*loc)] + .errPos = state->state.positions[state->at(*loc)] }); } @@ -158,21 +158,21 @@ expr_op | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(state->symbols.create("__sub")), {new ExprInt(0), $2}); } | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); } | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); } - | expr_op '<' expr_op { $$ = new ExprCall(state->makeCurPos(@2), new ExprVar(state->symbols.create("__lessThan")), {$1, $3}); } - | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->makeCurPos(@2), new ExprVar(state->symbols.create("__lessThan")), {$3, $1})); } - | expr_op '>' expr_op { $$ = new ExprCall(state->makeCurPos(@2), new ExprVar(state->symbols.create("__lessThan")), {$3, $1}); } - | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->makeCurPos(@2), new ExprVar(state->symbols.create("__lessThan")), {$1, $3})); } - | expr_op AND expr_op { $$ = new ExprOpAnd(state->makeCurPos(@2), $1, $3); } - | expr_op OR expr_op { $$ = new ExprOpOr(state->makeCurPos(@2), $1, $3); } - | expr_op IMPL expr_op { $$ = new ExprOpImpl(state->makeCurPos(@2), $1, $3); } - | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(state->makeCurPos(@2), $1, $3); } + | expr_op '<' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->symbols.create("__lessThan")), {$1, $3}); } + | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->at(@2), new ExprVar(state->symbols.create("__lessThan")), {$3, $1})); } + | expr_op '>' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->symbols.create("__lessThan")), {$3, $1}); } + | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->at(@2), new ExprVar(state->symbols.create("__lessThan")), {$1, $3})); } + | expr_op AND expr_op { $$ = new ExprOpAnd(state->at(@2), $1, $3); } + | expr_op OR expr_op { $$ = new ExprOpOr(state->at(@2), $1, $3); } + | expr_op IMPL expr_op { $$ = new ExprOpImpl(state->at(@2), $1, $3); } + | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(state->at(@2), $1, $3); } | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, std::move(*$3)); delete $3; } | expr_op '+' expr_op - { $$ = new ExprConcatStrings(state->makeCurPos(@2), false, new std::vector >({{state->makeCurPos(@1), $1}, {state->makeCurPos(@3), $3}})); } - | expr_op '-' expr_op { $$ = new ExprCall(state->makeCurPos(@2), new ExprVar(state->symbols.create("__sub")), {$1, $3}); } - | expr_op '*' expr_op { $$ = new ExprCall(state->makeCurPos(@2), new ExprVar(state->symbols.create("__mul")), {$1, $3}); } - | expr_op '/' expr_op { $$ = new ExprCall(state->makeCurPos(@2), new ExprVar(state->symbols.create("__div")), {$1, $3}); } - | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(state->makeCurPos(@2), $1, $3); } + { $$ = new ExprConcatStrings(state->at(@2), false, new std::vector >({{state->at(@1), $1}, {state->at(@3), $3}})); } + | expr_op '-' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->symbols.create("__sub")), {$1, $3}); } + | expr_op '*' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->symbols.create("__mul")), {$1, $3}); } + | expr_op '/' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->symbols.create("__div")), {$1, $3}); } + | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(state->at(@2), $1, $3); } | expr_app ; @@ -216,7 +216,7 @@ expr_simple } | path_start PATH_END | path_start string_parts_interpolated PATH_END { - $2->insert($2->begin(), {state->makeCurPos(@1), $1}); + $2->insert($2->begin(), {state->at(@1), $1}); $$ = new ExprConcatStrings(CUR_POS, false, $2); } | SPATH { @@ -255,13 +255,13 @@ string_parts string_parts_interpolated : string_parts_interpolated STR - { $$ = $1; $1->emplace_back(state->makeCurPos(@2), new ExprString(std::string($2))); } - | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->makeCurPos(@2), $3); } - | DOLLAR_CURLY expr '}' { $$ = new std::vector>; $$->emplace_back(state->makeCurPos(@1), $2); } + { $$ = $1; $1->emplace_back(state->at(@2), new ExprString(std::string($2))); } + | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); } + | DOLLAR_CURLY expr '}' { $$ = new std::vector>; $$->emplace_back(state->at(@1), $2); } | STR DOLLAR_CURLY expr '}' { $$ = new std::vector>; - $$->emplace_back(state->makeCurPos(@1), new ExprString(std::string($1))); - $$->emplace_back(state->makeCurPos(@2), $3); + $$->emplace_back(state->at(@1), new ExprString(std::string($1))); + $$->emplace_back(state->at(@2), $3); } ; @@ -286,19 +286,19 @@ path_start ; ind_string_parts - : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(state->makeCurPos(@2), $2); } - | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->makeCurPos(@2), $3); } + : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(state->at(@2), $2); } + | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); } | { $$ = new std::vector>>; } ; binds - : binds attrpath '=' expr ';' { $$ = $1; state->addAttr($$, std::move(*$2), $4, state->makeCurPos(@2)); delete $2; } + : binds attrpath '=' expr ';' { $$ = $1; state->addAttr($$, std::move(*$2), $4, state->at(@2)); delete $2; } | binds INHERIT attrs ';' { $$ = $1; for (auto & i : *$3) { if ($$->attrs.find(i.symbol) != $$->attrs.end()) - state->dupAttr(i.symbol, state->makeCurPos(@3), $$->attrs[i.symbol].pos); - auto pos = state->makeCurPos(@3); + state->dupAttr(i.symbol, state->at(@3), $$->attrs[i.symbol].pos); + auto pos = state->at(@3); $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true)); } delete $3; @@ -308,12 +308,12 @@ binds /* !!! Should ensure sharing of the expression in $4. */ for (auto & i : *$6) { if ($$->attrs.find(i.symbol) != $$->attrs.end()) - state->dupAttr(i.symbol, state->makeCurPos(@6), $$->attrs[i.symbol].pos); - $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), state->makeCurPos(@6))); + state->dupAttr(i.symbol, state->at(@6), $$->attrs[i.symbol].pos); + $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), state->at(@6))); } delete $6; } - | { $$ = new ExprAttrs(state->makeCurPos(@0)); } + | { $$ = new ExprAttrs(state->at(@0)); } ; attrs @@ -327,7 +327,7 @@ attrs } else throw ParseError({ .msg = hintfmt("dynamic attributes not allowed in inherit"), - .errPos = state->state.positions[state->makeCurPos(@2)] + .errPos = state->state.positions[state->at(@2)] }); } | { $$ = new AttrPath; } From e1aa585964c3d864ebff0030584f3349a539d615 Mon Sep 17 00:00:00 2001 From: pennae Date: Mon, 15 Jan 2024 16:52:18 +0100 Subject: [PATCH 06/10] slim down parser.y most EvalState and Expr members defined here could be elsewhere, where they'd be easier to maintain (not being embedded in a file with arcane syntax) and *somewhat* more faithfully placed according to the path of the file they're defined in. --- src/libexpr/eval.cc | 164 ++++++++++++++++++++++++++++++++++++++ src/libexpr/nixexpr.cc | 2 + src/libexpr/parser.y | 176 ----------------------------------------- 3 files changed, 166 insertions(+), 176 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 0659a2173..6eee7cdce 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -20,6 +20,8 @@ #include "gc-small-vector.hh" #include "url.hh" #include "fetch-to-store.hh" +#include "tarball.hh" +#include "flake/flakeref.hh" #include #include @@ -2636,6 +2638,168 @@ void EvalState::printStatistics() } +SourcePath resolveExprPath(SourcePath path) +{ + unsigned int followCount = 0, maxFollow = 1024; + + /* If `path' is a symlink, follow it. This is so that relative + path references work. */ + while (!path.path.isRoot()) { + // Basic cycle/depth limit to avoid infinite loops. + if (++followCount >= maxFollow) + throw Error("too many symbolic links encountered while traversing the path '%s'", path); + auto p = path.parent().resolveSymlinks() + path.baseName(); + if (p.lstat().type != InputAccessor::tSymlink) break; + path = {path.accessor, CanonPath(p.readLink(), path.path.parent().value_or(CanonPath::root))}; + } + + /* If `path' refers to a directory, append `/default.nix'. */ + if (path.resolveSymlinks().lstat().type == InputAccessor::tDirectory) + return path + "default.nix"; + + return path; +} + + +Expr * EvalState::parseExprFromFile(const SourcePath & path) +{ + return parseExprFromFile(path, staticBaseEnv); +} + + +Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr & staticEnv) +{ + auto buffer = path.resolveSymlinks().readFile(); + // readFile hopefully have left some extra space for terminators + buffer.append("\0\0", 2); + return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv); +} + + +Expr * EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr & staticEnv) +{ + auto s = make_ref(std::move(s_)); + s->append("\0\0", 2); + return parse(s->data(), s->size(), Pos::String{.source = s}, basePath, staticEnv); +} + + +Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath) +{ + return parseExprFromString(std::move(s), basePath, staticBaseEnv); +} + + +Expr * EvalState::parseStdin() +{ + //Activity act(*logger, lvlTalkative, "parsing standard input"); + auto buffer = drainFD(0); + // drainFD should have left some extra space for terminators + buffer.append("\0\0", 2); + auto s = make_ref(std::move(buffer)); + return parse(s->data(), s->size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv); +} + + +SourcePath EvalState::findFile(const std::string_view path) +{ + return findFile(searchPath, path); +} + + +SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos) +{ + for (auto & i : searchPath.elements) { + auto suffixOpt = i.prefix.suffixIfPotentialMatch(path); + + if (!suffixOpt) continue; + auto suffix = *suffixOpt; + + auto rOpt = resolveSearchPathPath(i.path); + if (!rOpt) continue; + auto r = *rOpt; + + Path res = suffix == "" ? r : concatStrings(r, "/", suffix); + if (pathExists(res)) return rootPath(CanonPath(canonPath(res))); + } + + if (hasPrefix(path, "nix/")) + return {corepkgsFS, CanonPath(path.substr(3))}; + + debugThrow(ThrownError({ + .msg = hintfmt(evalSettings.pureEval + ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" + : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", + path), + .errPos = positions[pos] + }), 0, 0); +} + + +std::optional EvalState::resolveSearchPathPath(const SearchPath::Path & value0, bool initAccessControl) +{ + auto & value = value0.s; + auto i = searchPathResolved.find(value); + if (i != searchPathResolved.end()) return i->second; + + std::optional res; + + if (EvalSettings::isPseudoUrl(value)) { + try { + auto storePath = fetchers::downloadTarball( + store, EvalSettings::resolvePseudoUrl(value), "source", false).storePath; + res = { store->toRealPath(storePath) }; + } catch (FileTransferError & e) { + logWarning({ + .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) + }); + } + } + + else if (hasPrefix(value, "flake:")) { + experimentalFeatureSettings.require(Xp::Flakes); + auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); + debug("fetching flake search path element '%s''", value); + auto storePath = flakeRef.resolve(store).fetchTree(store).first; + res = { store->toRealPath(storePath) }; + } + + else { + auto path = absPath(value); + + /* Allow access to paths in the search path. */ + if (initAccessControl) { + allowPath(path); + if (store->isInStore(path)) { + try { + StorePathSet closure; + store->computeFSClosure(store->toStorePath(path).first, closure); + for (auto & p : closure) + allowPath(p); + } catch (InvalidPath &) { } + } + } + + if (pathExists(path)) + res = { path }; + else { + logWarning({ + .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", value) + }); + res = std::nullopt; + } + } + + if (res) + debug("resolved search path element '%s' to '%s'", value, *res); + else + debug("failed to resolve search path element '%s'", value); + + searchPathResolved.emplace(value, res); + return res; +} + + std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const { throw TypeError({ diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 964de6351..6fe4ba81b 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -9,6 +9,8 @@ namespace nix { +unsigned long Expr::nrExprs = 0; + ExprBlackHole eBlackHole; // FIXME: remove, because *symbols* are abstract and do not have a single diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 7763a72bc..519d6b11f 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -389,25 +389,11 @@ formal %% - -#include -#include -#include -#include - #include "eval.hh" -#include "filetransfer.hh" -#include "tarball.hh" -#include "store-api.hh" -#include "flake/flake.hh" -#include "fs-input-accessor.hh" -#include "memory-input-accessor.hh" namespace nix { -unsigned long Expr::nrExprs = 0; - Expr * EvalState::parse( char * text, size_t length, @@ -435,166 +421,4 @@ Expr * EvalState::parse( } -SourcePath resolveExprPath(SourcePath path) -{ - unsigned int followCount = 0, maxFollow = 1024; - - /* If `path' is a symlink, follow it. This is so that relative - path references work. */ - while (!path.path.isRoot()) { - // Basic cycle/depth limit to avoid infinite loops. - if (++followCount >= maxFollow) - throw Error("too many symbolic links encountered while traversing the path '%s'", path); - auto p = path.parent().resolveSymlinks() + path.baseName(); - if (p.lstat().type != InputAccessor::tSymlink) break; - path = {path.accessor, CanonPath(p.readLink(), path.path.parent().value_or(CanonPath::root))}; - } - - /* If `path' refers to a directory, append `/default.nix'. */ - if (path.resolveSymlinks().lstat().type == InputAccessor::tDirectory) - return path + "default.nix"; - - return path; -} - - -Expr * EvalState::parseExprFromFile(const SourcePath & path) -{ - return parseExprFromFile(path, staticBaseEnv); -} - - -Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr & staticEnv) -{ - auto buffer = path.resolveSymlinks().readFile(); - // readFile hopefully have left some extra space for terminators - buffer.append("\0\0", 2); - return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv); -} - - -Expr * EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr & staticEnv) -{ - auto s = make_ref(std::move(s_)); - s->append("\0\0", 2); - return parse(s->data(), s->size(), Pos::String{.source = s}, basePath, staticEnv); -} - - -Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath) -{ - return parseExprFromString(std::move(s), basePath, staticBaseEnv); -} - - -Expr * EvalState::parseStdin() -{ - //Activity act(*logger, lvlTalkative, "parsing standard input"); - auto buffer = drainFD(0); - // drainFD should have left some extra space for terminators - buffer.append("\0\0", 2); - auto s = make_ref(std::move(buffer)); - return parse(s->data(), s->size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv); -} - - -SourcePath EvalState::findFile(const std::string_view path) -{ - return findFile(searchPath, path); -} - - -SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos) -{ - for (auto & i : searchPath.elements) { - auto suffixOpt = i.prefix.suffixIfPotentialMatch(path); - - if (!suffixOpt) continue; - auto suffix = *suffixOpt; - - auto rOpt = resolveSearchPathPath(i.path); - if (!rOpt) continue; - auto r = *rOpt; - - Path res = suffix == "" ? r : concatStrings(r, "/", suffix); - if (pathExists(res)) return rootPath(CanonPath(canonPath(res))); - } - - if (hasPrefix(path, "nix/")) - return {corepkgsFS, CanonPath(path.substr(3))}; - - debugThrow(ThrownError({ - .msg = hintfmt(evalSettings.pureEval - ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" - : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", - path), - .errPos = positions[pos] - }), 0, 0); -} - - -std::optional EvalState::resolveSearchPathPath(const SearchPath::Path & value0, bool initAccessControl) -{ - auto & value = value0.s; - auto i = searchPathResolved.find(value); - if (i != searchPathResolved.end()) return i->second; - - std::optional res; - - if (EvalSettings::isPseudoUrl(value)) { - try { - auto storePath = fetchers::downloadTarball( - store, EvalSettings::resolvePseudoUrl(value), "source", false).storePath; - res = { store->toRealPath(storePath) }; - } catch (FileTransferError & e) { - logWarning({ - .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) - }); - } - } - - else if (hasPrefix(value, "flake:")) { - experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); - debug("fetching flake search path element '%s''", value); - auto storePath = flakeRef.resolve(store).fetchTree(store).first; - res = { store->toRealPath(storePath) }; - } - - else { - auto path = absPath(value); - - /* Allow access to paths in the search path. */ - if (initAccessControl) { - allowPath(path); - if (store->isInStore(path)) { - try { - StorePathSet closure; - store->computeFSClosure(store->toStorePath(path).first, closure); - for (auto & p : closure) - allowPath(p); - } catch (InvalidPath &) { } - } - } - - if (pathExists(path)) - res = { path }; - else { - logWarning({ - .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", value) - }); - res = std::nullopt; - } - } - - if (res) - debug("resolved search path element '%s' to '%s'", value, *res); - else - debug("failed to resolve search path element '%s'", value); - - searchPathResolved.emplace(value, res); - return res; -} - - } From b596cc9e7960b9256bcd557334d81e9d555be5a2 Mon Sep 17 00:00:00 2001 From: pennae Date: Mon, 15 Jan 2024 16:52:18 +0100 Subject: [PATCH 07/10] decouple parser and EvalState there's no reason the parser itself should be doing semantic analysis like bindVars. split this bit apart (retaining the previous name in EvalState) and have the parser really do *only* parsing, decoupled from EvalState. --- src/libexpr/eval.cc | 16 ++++++++++++++++ src/libexpr/lexer.l | 6 +++--- src/libexpr/parser-state.hh | 17 +++++++++-------- src/libexpr/parser.y | 36 +++++++++++++++++++++++++----------- 4 files changed, 53 insertions(+), 22 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 6eee7cdce..b05ccfc85 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -22,6 +22,7 @@ #include "fetch-to-store.hh" #include "tarball.hh" #include "flake/flakeref.hh" +#include "parser-tab.hh" #include #include @@ -2800,6 +2801,21 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa } +Expr * EvalState::parse( + char * text, + size_t length, + Pos::Origin origin, + const SourcePath & basePath, + std::shared_ptr & staticEnv) +{ + auto result = parseExprFromBuf(text, length, origin, basePath, symbols, positions, rootFS); + + result->bindVars(*this, staticEnv); + + return result; +} + + std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const { throw TypeError({ diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index fae0e7a85..d7a0b5048 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -148,7 +148,7 @@ or { return OR_KW; } } catch (const boost::bad_lexical_cast &) { throw ParseError({ .msg = hintfmt("invalid integer '%1%'", yytext), - .errPos = state->state.positions[CUR_POS], + .errPos = state->positions[CUR_POS], }); } return INT_LIT; @@ -158,7 +158,7 @@ or { return OR_KW; } if (errno != 0) throw ParseError({ .msg = hintfmt("invalid float '%1%'", yytext), - .errPos = state->state.positions[CUR_POS], + .errPos = state->positions[CUR_POS], }); return FLOAT_LIT; } @@ -287,7 +287,7 @@ or { return OR_KW; } <> { throw ParseError({ .msg = hintfmt("path has a trailing slash"), - .errPos = state->state.positions[CUR_POS], + .errPos = state->positions[CUR_POS], }); } diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 167d3f4ae..6ab9fc962 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -37,11 +37,12 @@ struct ParserLocation { }; struct ParserState { - EvalState & state; SymbolTable & symbols; + PosTable & positions; Expr * result; SourcePath basePath; PosTable::Origin origin; + const ref rootFS; void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos); void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos); @@ -56,16 +57,16 @@ inline void ParserState::dupAttr(const AttrPath & attrPath, const PosIdx pos, co { throw ParseError({ .msg = hintfmt("attribute '%1%' already defined at %2%", - showAttrPath(state.symbols, attrPath), state.positions[prevPos]), - .errPos = state.positions[pos] + showAttrPath(symbols, attrPath), positions[prevPos]), + .errPos = positions[pos] }); } inline void ParserState::dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos) { throw ParseError({ - .msg = hintfmt("attribute '%1%' already defined at %2%", state.symbols[attr], state.positions[prevPos]), - .errPos = state.positions[pos] + .msg = hintfmt("attribute '%1%' already defined at %2%", symbols[attr], positions[prevPos]), + .errPos = positions[pos] }); } @@ -146,13 +147,13 @@ inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Sym if (duplicate) throw ParseError({ .msg = hintfmt("duplicate formal function argument '%1%'", symbols[duplicate->first]), - .errPos = state.positions[duplicate->second] + .errPos = positions[duplicate->second] }); if (arg && formals->has(arg)) throw ParseError({ .msg = hintfmt("duplicate formal function argument '%1%'", symbols[arg]), - .errPos = state.positions[pos] + .errPos = positions[pos] }); return formals; @@ -256,7 +257,7 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos, inline PosIdx ParserState::at(const ParserLocation & loc) { - return state.positions.add(origin, loc.first_line, loc.first_column); + return positions.add(origin, loc.first_line, loc.first_column); } } diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 519d6b11f..faf5e897f 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -32,6 +32,19 @@ #define YY_DECL int yylex \ (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state) +namespace nix { + +Expr * parseExprFromBuf( + char * text, + size_t length, + Pos::Origin origin, + const SourcePath & basePath, + SymbolTable & symbols, + PosTable & positions, + const ref rootFS); + +} + #endif } @@ -52,7 +65,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * { throw ParseError({ .msg = hintfmt(error), - .errPos = state->state.positions[state->at(*loc)] + .errPos = state->positions[state->at(*loc)] }); } @@ -141,7 +154,7 @@ expr_function { if (!$2->dynamicAttrs.empty()) throw ParseError({ .msg = hintfmt("dynamic attributes not allowed in let"), - .errPos = state->state.positions[CUR_POS] + .errPos = state->positions[CUR_POS] }); $$ = new ExprLet($2, $4); } @@ -231,7 +244,7 @@ expr_simple if (noURLLiterals) throw ParseError({ .msg = hintfmt("URL literals are disabled"), - .errPos = state->state.positions[CUR_POS] + .errPos = state->positions[CUR_POS] }); $$ = new ExprString(std::string($1)); } @@ -271,7 +284,7 @@ path_start /* add back in the trailing '/' to the first segment */ if ($1.p[$1.l-1] == '/' && $1.l > 1) path += "/"; - $$ = new ExprPath(ref(state->state.rootFS), std::move(path)); + $$ = new ExprPath(ref(state->rootFS), std::move(path)); } | HPATH { if (evalSettings.pureEval) { @@ -281,7 +294,7 @@ path_start ); } Path path(getHome() + std::string($1.p + 1, $1.l - 1)); - $$ = new ExprPath(ref(state->state.rootFS), std::move(path)); + $$ = new ExprPath(ref(state->rootFS), std::move(path)); } ; @@ -327,7 +340,7 @@ attrs } else throw ParseError({ .msg = hintfmt("dynamic attributes not allowed in inherit"), - .errPos = state->state.positions[state->at(@2)] + .errPos = state->positions[state->at(@2)] }); } | { $$ = new AttrPath; } @@ -394,19 +407,22 @@ formal namespace nix { -Expr * EvalState::parse( +Expr * parseExprFromBuf( char * text, size_t length, Pos::Origin origin, const SourcePath & basePath, - std::shared_ptr & staticEnv) + SymbolTable & symbols, + PosTable & positions, + const ref rootFS) { yyscan_t scanner; ParserState state { - .state = *this, .symbols = symbols, + .positions = positions, .basePath = basePath, .origin = {origin}, + .rootFS = rootFS, }; yylex_init(&scanner); @@ -415,8 +431,6 @@ Expr * EvalState::parse( yy_scan_buffer(text, length, scanner); yyparse(scanner, &state); - state.result->bindVars(*this, staticEnv); - return state.result; } From 09a1128d9e2ff0ae6176784938047350d6f8a782 Mon Sep 17 00:00:00 2001 From: pennae Date: Mon, 15 Jan 2024 16:52:18 +0100 Subject: [PATCH 08/10] don't repeatedly look up ast internal symbols these symbols are used a *lot*, so it makes sense to cache them. this mostly increases clarity of the code (however clear one may wish to call the parser desugaring here), but it also provides a small performance benefit. --- src/libexpr/eval.cc | 12 +++++++++++- src/libexpr/eval.hh | 2 ++ src/libexpr/nixexpr.hh | 5 +++++ src/libexpr/parser-state.hh | 1 + src/libexpr/parser.y | 31 +++++++++++++++++-------------- 5 files changed, 36 insertions(+), 15 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index b05ccfc85..dc9167144 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -419,6 +419,16 @@ EvalState::EvalState( , sPath(symbols.create("path")) , sPrefix(symbols.create("prefix")) , sOutputSpecified(symbols.create("outputSpecified")) + , exprSymbols{ + .sub = symbols.create("__sub"), + .lessThan = symbols.create("__lessThan"), + .mul = symbols.create("__mul"), + .div = symbols.create("__div"), + .or_ = symbols.create("or"), + .findFile = symbols.create("__findFile"), + .nixPath = symbols.create("__nixPath"), + .body = symbols.create("body") + } , repair(NoRepair) , emptyBindings(0) , rootFS( @@ -2808,7 +2818,7 @@ Expr * EvalState::parse( const SourcePath & basePath, std::shared_ptr & staticEnv) { - auto result = parseExprFromBuf(text, length, origin, basePath, symbols, positions, rootFS); + auto result = parseExprFromBuf(text, length, origin, basePath, symbols, positions, rootFS, exprSymbols); result->bindVars(*this, staticEnv); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 9141156b1..2368187b1 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -207,6 +207,8 @@ public: sPrefix, sOutputSpecified; + const Expr::AstSymbols exprSymbols; + /** * If set, force copying files to the Nix store even if they * already exist there. diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 3cd46ca27..b6189c2a9 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -140,6 +140,11 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) struct Expr { + struct AstSymbols { + Symbol sub, lessThan, mul, div, or_, findFile, nixPath, body; + }; + + static unsigned long nrExprs; Expr() { nrExprs++; diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 6ab9fc962..a5b932ae8 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -43,6 +43,7 @@ struct ParserState { SourcePath basePath; PosTable::Origin origin; const ref rootFS; + const Expr::AstSymbols & s; void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos); void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index faf5e897f..e95da37f7 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -41,7 +41,8 @@ Expr * parseExprFromBuf( const SourcePath & basePath, SymbolTable & symbols, PosTable & positions, - const ref rootFS); + const ref rootFS, + const Expr::AstSymbols & astSymbols); } @@ -168,13 +169,13 @@ expr_if expr_op : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); } - | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(state->symbols.create("__sub")), {new ExprInt(0), $2}); } + | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(state->s.sub), {new ExprInt(0), $2}); } | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); } | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); } - | expr_op '<' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->symbols.create("__lessThan")), {$1, $3}); } - | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->at(@2), new ExprVar(state->symbols.create("__lessThan")), {$3, $1})); } - | expr_op '>' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->symbols.create("__lessThan")), {$3, $1}); } - | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->at(@2), new ExprVar(state->symbols.create("__lessThan")), {$1, $3})); } + | expr_op '<' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$1, $3}); } + | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$3, $1})); } + | expr_op '>' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$3, $1}); } + | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$1, $3})); } | expr_op AND expr_op { $$ = new ExprOpAnd(state->at(@2), $1, $3); } | expr_op OR expr_op { $$ = new ExprOpOr(state->at(@2), $1, $3); } | expr_op IMPL expr_op { $$ = new ExprOpImpl(state->at(@2), $1, $3); } @@ -182,9 +183,9 @@ expr_op | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, std::move(*$3)); delete $3; } | expr_op '+' expr_op { $$ = new ExprConcatStrings(state->at(@2), false, new std::vector >({{state->at(@1), $1}, {state->at(@3), $3}})); } - | expr_op '-' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->symbols.create("__sub")), {$1, $3}); } - | expr_op '*' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->symbols.create("__mul")), {$1, $3}); } - | expr_op '/' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->symbols.create("__div")), {$1, $3}); } + | expr_op '-' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.sub), {$1, $3}); } + | expr_op '*' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.mul), {$1, $3}); } + | expr_op '/' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.div), {$1, $3}); } | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(state->at(@2), $1, $3); } | expr_app ; @@ -208,7 +209,7 @@ expr_select | /* Backwards compatibility: because Nixpkgs has a rarely used function named ‘or’, allow stuff like ‘map or [...]’. */ expr_simple OR_KW - { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, state->symbols.create("or"))}); } + { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, state->s.or_)}); } | expr_simple ; @@ -235,8 +236,8 @@ expr_simple | SPATH { std::string path($1.p + 1, $1.l - 2); $$ = new ExprCall(CUR_POS, - new ExprVar(state->symbols.create("__findFile")), - {new ExprVar(state->symbols.create("__nixPath")), + new ExprVar(state->s.findFile), + {new ExprVar(state->s.nixPath), new ExprString(std::move(path))}); } | URI { @@ -252,7 +253,7 @@ expr_simple /* Let expressions `let {..., body = ...}' are just desugared into `(rec {..., body = ...}).body'. */ | LET '{' binds '}' - { $3->recursive = true; $$ = new ExprSelect(noPos, $3, state->symbols.create("body")); } + { $3->recursive = true; $$ = new ExprSelect(noPos, $3, state->s.body); } | REC '{' binds '}' { $3->recursive = true; $$ = $3; } | '{' binds '}' @@ -414,7 +415,8 @@ Expr * parseExprFromBuf( const SourcePath & basePath, SymbolTable & symbols, PosTable & positions, - const ref rootFS) + const ref rootFS, + const Expr::AstSymbols & astSymbols) { yyscan_t scanner; ParserState state { @@ -423,6 +425,7 @@ Expr * parseExprFromBuf( .basePath = basePath, .origin = {origin}, .rootFS = rootFS, + .s = astSymbols, }; yylex_init(&scanner); From 80b84710b8c676620ed1e8bf8ff3bb1d5bc19b80 Mon Sep 17 00:00:00 2001 From: pennae <82953136+pennae@users.noreply.github.com> Date: Mon, 22 Jan 2024 15:15:53 +0100 Subject: [PATCH 09/10] Update src/libexpr/eval.cc Co-authored-by: John Ericson --- src/libexpr/eval.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index dc9167144..2330102c3 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -427,7 +427,7 @@ EvalState::EvalState( .or_ = symbols.create("or"), .findFile = symbols.create("__findFile"), .nixPath = symbols.create("__nixPath"), - .body = symbols.create("body") + .body = symbols.create("body"), } , repair(NoRepair) , emptyBindings(0) From 365b831e6f290c733da6879dae871dada343a1eb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 26 Jan 2024 23:11:31 -0500 Subject: [PATCH 10/10] Minor formatting tweaks --- src/libexpr/parser-state.hh | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index a5b932ae8..0a9f076dc 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -1,19 +1,25 @@ #pragma once +///@file #include "eval.hh" namespace nix { -// using C a struct allows us to avoid having to define the special -// members that using string_view here would implicitly delete. -struct StringToken { - const char * p; - size_t l; - bool hasIndentation; - operator std::string_view() const { return {p, l}; } +/** + * @note Storing a C-style `char *` and `size_t` allows us to avoid + * having to define the special members that using string_view here + * would implicitly delete. + */ +struct StringToken +{ + const char * p; + size_t l; + bool hasIndentation; + operator std::string_view() const { return {p, l}; } }; -struct ParserLocation { +struct ParserLocation +{ int first_line, first_column; int last_line, last_column; @@ -36,7 +42,8 @@ struct ParserLocation { } }; -struct ParserState { +struct ParserState +{ SymbolTable & symbols; PosTable & positions; Expr * result;