From e0a0ae0467fa8cdcc542f593b9d94283f04508ff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Mar 2020 14:03:28 +0200 Subject: [PATCH] Move fetchers from libstore to libfetchers --- Makefile | 1 + local.mk | 2 +- src/libexpr/common-eval-args.cc | 4 +-- src/libexpr/flake/flake.cc | 2 +- src/libexpr/flake/flakeref.cc | 7 ++--- src/libexpr/flake/flakeref.hh | 2 +- src/libexpr/flake/lockfile.cc | 3 +- src/libexpr/local.mk | 2 +- src/libexpr/parser.y | 2 +- src/libexpr/primops/fetchGit.cc | 8 ++--- src/libexpr/primops/fetchMercurial.cc | 11 ++++--- src/libexpr/primops/fetchTree.cc | 4 +-- .../fetchers => libfetchers}/attrs.cc | 0 .../fetchers => libfetchers}/attrs.hh | 0 .../fetchers => libfetchers}/cache.cc | 2 +- .../fetchers => libfetchers}/cache.hh | 3 +- .../fetchers => libfetchers}/fetchers.cc | 1 - .../fetchers => libfetchers}/fetchers.hh | 2 +- src/{libstore/fetchers => libfetchers}/git.cc | 6 ++-- .../fetchers => libfetchers}/github.cc | 6 ++-- .../fetchers => libfetchers}/indirect.cc | 2 -- src/libfetchers/local.mk | 9 ++++++ .../fetchers => libfetchers}/mercurial.cc | 6 ++-- .../fetchers => libfetchers}/registry.cc | 4 +-- .../fetchers => libfetchers}/registry.hh | 0 .../fetchers => libfetchers}/tarball.cc | 5 ++-- .../fetchers => libfetchers}/tree-info.hh | 0 src/libstore/fetchers/parse.hh | 30 ------------------- src/libstore/store-api.cc | 4 +-- .../fetchers/parse.cc => libutil/url.cc} | 5 ++-- .../fetchers/regex.hh => libutil/url.hh} | 27 ++++++++++++++++- src/nix-channel/nix-channel.cc | 2 +- src/nix/flake.cc | 4 +-- src/nix/installables.cc | 2 +- src/nix/local.mk | 2 +- 35 files changed, 80 insertions(+), 90 deletions(-) rename src/{libstore/fetchers => libfetchers}/attrs.cc (100%) rename src/{libstore/fetchers => libfetchers}/attrs.hh (100%) rename src/{libstore/fetchers => libfetchers}/cache.cc (99%) rename src/{libstore/fetchers => libfetchers}/cache.hh (92%) rename src/{libstore/fetchers => libfetchers}/fetchers.cc (99%) rename src/{libstore/fetchers => libfetchers}/fetchers.hh (99%) rename src/{libstore/fetchers => libfetchers}/git.cc (99%) rename src/{libstore/fetchers => libfetchers}/github.cc (98%) rename src/{libstore/fetchers => libfetchers}/indirect.cc (99%) create mode 100644 src/libfetchers/local.mk rename src/{libstore/fetchers => libfetchers}/mercurial.cc (98%) rename src/{libstore/fetchers => libfetchers}/registry.cc (98%) rename src/{libstore/fetchers => libfetchers}/registry.hh (100%) rename src/{libstore/fetchers => libfetchers}/tarball.cc (98%) rename src/{libstore/fetchers => libfetchers}/tree-info.hh (100%) delete mode 100644 src/libstore/fetchers/parse.hh rename src/{libstore/fetchers/parse.cc => libutil/url.cc} (98%) rename src/{libstore/fetchers/regex.hh => libutil/url.hh} (75%) diff --git a/Makefile b/Makefile index 469070533..e3057c36c 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ makefiles = \ nix-rust/local.mk \ src/libutil/local.mk \ src/libstore/local.mk \ + src/libfetchers/local.mk \ src/libmain/local.mk \ src/libexpr/local.mk \ src/nix/local.mk \ diff --git a/local.mk b/local.mk index a63cfd843..b57e4b061 100644 --- a/local.mk +++ b/local.mk @@ -6,7 +6,7 @@ dist-files += configure config.h.in perl/configure clean-files += Makefile.config -GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix -Wno-deprecated-declarations +GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr -I src/nix -Wno-deprecated-declarations $(foreach i, config.h $(wildcard src/lib*/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 9ebb6a835..bf5025147 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -3,8 +3,8 @@ #include "download.hh" #include "util.hh" #include "eval.hh" -#include "fetchers/registry.hh" -#include "fetchers/fetchers.hh" +#include "registry.hh" +#include "fetchers.hh" #include "flake/flakeref.hh" #include "store-api.hh" diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 9517e3645..59362c018 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -3,7 +3,7 @@ #include "primops.hh" #include "eval-inline.hh" #include "store-api.hh" -#include "fetchers/fetchers.hh" +#include "fetchers.hh" #include "finally.hh" namespace nix { diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 5c38f7ea5..de91f2eed 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -1,9 +1,8 @@ #include "flakeref.hh" #include "store-api.hh" -#include "fetchers/parse.hh" -#include "fetchers/fetchers.hh" -#include "fetchers/registry.hh" -#include "fetchers/regex.hh" +#include "url.hh" +#include "fetchers.hh" +#include "registry.hh" namespace nix { diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index d23a8f601..1dbb6e54d 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -2,7 +2,7 @@ #include "types.hh" #include "hash.hh" -#include "fetchers/fetchers.hh" +#include "fetchers.hh" #include diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index ee04ec64f..f73b1ac25 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -1,6 +1,5 @@ #include "lockfile.hh" #include "store-api.hh" -#include "fetchers/regex.hh" #include @@ -268,7 +267,7 @@ InputPath parseInputPath(std::string_view s) InputPath path; for (auto & elem : tokenizeString>(s, "/")) { - if (!std::regex_match(elem, fetchers::flakeIdRegex)) + if (!std::regex_match(elem, flakeIdRegex)) throw Error("invalid flake input path element '%s'", elem); path.push_back(elem); } diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index eb7243b29..85cb51f34 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -11,7 +11,7 @@ libexpr_SOURCES := \ $(d)/lexer-tab.cc \ $(d)/parser-tab.cc -libexpr_LIBS = libutil libstore libnixrust +libexpr_LIBS = libutil libstore libfetchers libnixrust libexpr_LDFLAGS = ifneq ($(OS), FreeBSD) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index dd5436c04..a30fb44b5 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -545,7 +545,7 @@ formal #include "eval.hh" #include "download.hh" -#include "fetchers/fetchers.hh" +#include "fetchers.hh" #include "store-api.hh" diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 638c14970..812de9d91 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -2,8 +2,8 @@ #include "eval-inline.hh" #include "store-api.hh" #include "hash.hh" -#include "fetchers/fetchers.hh" -#include "fetchers/parse.hh" +#include "fetchers.hh" +#include "url.hh" namespace nix { @@ -48,14 +48,14 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va if (evalSettings.pureEval && !rev) throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); - auto parsedUrl = fetchers::parseURL( + auto parsedUrl = parseURL( url.find("://") != std::string::npos ? "git+" + url : "git+file://" + url); if (ref) parsedUrl.query.insert_or_assign("ref", *ref); if (rev) parsedUrl.query.insert_or_assign("rev", rev->gitRev()); // FIXME: use name - auto input = inputFromURL(parsedUrl); + auto input = fetchers::inputFromURL(parsedUrl); auto [tree, input2] = input->fetchTree(state.store); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 5d6b65c3b..f18351646 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -1,9 +1,8 @@ #include "primops.hh" #include "eval-inline.hh" #include "store-api.hh" -#include "fetchers/fetchers.hh" -#include "fetchers/parse.hh" -#include "fetchers/regex.hh" +#include "fetchers.hh" +#include "url.hh" #include @@ -31,7 +30,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar // Ugly: unlike fetchGit, here the "rev" attribute can // be both a revision or a branch/tag name. auto value = state.forceStringNoCtx(*attr.value, *attr.pos); - if (std::regex_match(value, fetchers::revRegex)) + if (std::regex_match(value, revRegex)) rev = Hash(value, htSHA1); else ref = value; @@ -55,14 +54,14 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar if (evalSettings.pureEval && !rev) throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); - auto parsedUrl = fetchers::parseURL( + auto parsedUrl = parseURL( url.find("://") != std::string::npos ? "hg+" + url : "hg+file://" + url); if (rev) parsedUrl.query.insert_or_assign("rev", rev->gitRev()); if (ref) parsedUrl.query.insert_or_assign("ref", *ref); // FIXME: use name - auto input = inputFromURL(parsedUrl); + auto input = fetchers::inputFromURL(parsedUrl); auto [tree, input2] = input->fetchTree(state.store); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 8e8b48fc8..035586c25 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -1,8 +1,8 @@ #include "primops.hh" #include "eval-inline.hh" #include "store-api.hh" -#include "fetchers/fetchers.hh" -#include "fetchers/registry.hh" +#include "fetchers.hh" +#include "registry.hh" #include "download.hh" #include diff --git a/src/libstore/fetchers/attrs.cc b/src/libfetchers/attrs.cc similarity index 100% rename from src/libstore/fetchers/attrs.cc rename to src/libfetchers/attrs.cc diff --git a/src/libstore/fetchers/attrs.hh b/src/libfetchers/attrs.hh similarity index 100% rename from src/libstore/fetchers/attrs.hh rename to src/libfetchers/attrs.hh diff --git a/src/libstore/fetchers/cache.cc b/src/libfetchers/cache.cc similarity index 99% rename from src/libstore/fetchers/cache.cc rename to src/libfetchers/cache.cc index 14a84744a..e1c7f3dee 100644 --- a/src/libstore/fetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -1,4 +1,4 @@ -#include "fetchers/cache.hh" +#include "cache.hh" #include "sqlite.hh" #include "sync.hh" #include "store-api.hh" diff --git a/src/libstore/fetchers/cache.hh b/src/libfetchers/cache.hh similarity index 92% rename from src/libstore/fetchers/cache.hh rename to src/libfetchers/cache.hh index a25b05985..d76ab1233 100644 --- a/src/libstore/fetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -1,7 +1,6 @@ #pragma once -#include "types.hh" -#include "fetchers/fetchers.hh" +#include "fetchers.hh" namespace nix::fetchers { diff --git a/src/libstore/fetchers/fetchers.cc b/src/libfetchers/fetchers.cc similarity index 99% rename from src/libstore/fetchers/fetchers.cc rename to src/libfetchers/fetchers.cc index 25827ab7c..5a782a4fe 100644 --- a/src/libstore/fetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -1,5 +1,4 @@ #include "fetchers.hh" -#include "parse.hh" #include "store-api.hh" #include diff --git a/src/libstore/fetchers/fetchers.hh b/src/libfetchers/fetchers.hh similarity index 99% rename from src/libstore/fetchers/fetchers.hh rename to src/libfetchers/fetchers.hh index 5e33ec4ca..b75dcffa5 100644 --- a/src/libstore/fetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -5,7 +5,7 @@ #include "path.hh" #include "tree-info.hh" #include "attrs.hh" -#include "parse.hh" +#include "url.hh" #include diff --git a/src/libstore/fetchers/git.cc b/src/libfetchers/git.cc similarity index 99% rename from src/libstore/fetchers/git.cc rename to src/libfetchers/git.cc index f6b7820b8..e578ed731 100644 --- a/src/libstore/fetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1,10 +1,8 @@ -#include "fetchers/fetchers.hh" -#include "fetchers/cache.hh" -#include "fetchers/parse.hh" +#include "fetchers.hh" +#include "cache.hh" #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" -#include "regex.hh" #include diff --git a/src/libstore/fetchers/github.cc b/src/libfetchers/github.cc similarity index 98% rename from src/libstore/fetchers/github.cc rename to src/libfetchers/github.cc index 5e34ee051..505af8af1 100644 --- a/src/libstore/fetchers/github.cc +++ b/src/libfetchers/github.cc @@ -1,8 +1,6 @@ #include "download.hh" -#include "fetchers/cache.hh" -#include "fetchers/fetchers.hh" -#include "fetchers/parse.hh" -#include "fetchers/regex.hh" +#include "cache.hh" +#include "fetchers.hh" #include "globals.hh" #include "store-api.hh" diff --git a/src/libstore/fetchers/indirect.cc b/src/libfetchers/indirect.cc similarity index 99% rename from src/libstore/fetchers/indirect.cc rename to src/libfetchers/indirect.cc index 37e5afbc4..380b69fe0 100644 --- a/src/libstore/fetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -1,6 +1,4 @@ #include "fetchers.hh" -#include "parse.hh" -#include "regex.hh" namespace nix::fetchers { diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk new file mode 100644 index 000000000..4f3d4e85a --- /dev/null +++ b/src/libfetchers/local.mk @@ -0,0 +1,9 @@ +libraries += libfetchers + +libfetchers_NAME = libnixfetchers + +libfetchers_DIR := $(d) + +libfetchers_SOURCES := $(wildcard $(d)/*.cc) + +libfetchers_LIBS = libutil libstore libnixrust diff --git a/src/libstore/fetchers/mercurial.cc b/src/libfetchers/mercurial.cc similarity index 98% rename from src/libstore/fetchers/mercurial.cc rename to src/libfetchers/mercurial.cc index 6fb13391b..4b73fbcbe 100644 --- a/src/libstore/fetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -1,10 +1,8 @@ -#include "fetchers/fetchers.hh" -#include "fetchers/cache.hh" -#include "fetchers/parse.hh" +#include "fetchers.hh" +#include "cache.hh" #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" -#include "regex.hh" #include diff --git a/src/libstore/fetchers/registry.cc b/src/libfetchers/registry.cc similarity index 98% rename from src/libstore/fetchers/registry.cc rename to src/libfetchers/registry.cc index 69c80a5a9..34e63180c 100644 --- a/src/libstore/fetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,5 +1,5 @@ -#include "fetchers/registry.hh" -#include "fetchers/fetchers.hh" +#include "registry.hh" +#include "fetchers.hh" #include "util.hh" #include "globals.hh" #include "download.hh" diff --git a/src/libstore/fetchers/registry.hh b/src/libfetchers/registry.hh similarity index 100% rename from src/libstore/fetchers/registry.hh rename to src/libfetchers/registry.hh diff --git a/src/libstore/fetchers/tarball.cc b/src/libfetchers/tarball.cc similarity index 98% rename from src/libstore/fetchers/tarball.cc rename to src/libfetchers/tarball.cc index 55244b30e..b0a83001e 100644 --- a/src/libstore/fetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -1,6 +1,5 @@ -#include "fetchers/fetchers.hh" -#include "fetchers/parse.hh" -#include "fetchers/cache.hh" +#include "fetchers.hh" +#include "cache.hh" #include "download.hh" #include "globals.hh" #include "store-api.hh" diff --git a/src/libstore/fetchers/tree-info.hh b/src/libfetchers/tree-info.hh similarity index 100% rename from src/libstore/fetchers/tree-info.hh rename to src/libfetchers/tree-info.hh diff --git a/src/libstore/fetchers/parse.hh b/src/libstore/fetchers/parse.hh deleted file mode 100644 index 45d5182b0..000000000 --- a/src/libstore/fetchers/parse.hh +++ /dev/null @@ -1,30 +0,0 @@ -#pragma once - -#include "types.hh" - -namespace nix::fetchers { - -struct ParsedURL -{ - std::string url; - std::string base; // URL without query/fragment - std::string scheme; - std::optional authority; - std::string path; - std::map query; - std::string fragment; - - std::string to_string() const; - - bool operator ==(const ParsedURL & other) const; -}; - -MakeError(BadURL, Error); - -std::string percentDecode(std::string_view in); - -std::map decodeQuery(const std::string & query); - -ParsedURL parseURL(const std::string & url); - -} diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 66a9d9fb5..e5282bb30 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -6,7 +6,7 @@ #include "thread-pool.hh" #include "json.hh" #include "derivations.hh" -#include "fetchers/parse.hh" +#include "url.hh" #include @@ -867,7 +867,7 @@ std::pair splitUriAndParams(const std::string & uri_ Store::Params params; auto q = uri.find('?'); if (q != std::string::npos) { - params = fetchers::decodeQuery(uri.substr(q + 1)); + params = decodeQuery(uri.substr(q + 1)); uri = uri_.substr(0, q); } return {uri, params}; diff --git a/src/libstore/fetchers/parse.cc b/src/libutil/url.cc similarity index 98% rename from src/libstore/fetchers/parse.cc rename to src/libutil/url.cc index a5ad14c87..5d5328e5d 100644 --- a/src/libstore/fetchers/parse.cc +++ b/src/libutil/url.cc @@ -1,8 +1,7 @@ -#include "parse.hh" +#include "url.hh" #include "util.hh" -#include "regex.hh" -namespace nix::fetchers { +namespace nix { std::regex refRegex(refRegexS, std::regex::ECMAScript); std::regex revRegex(revRegexS, std::regex::ECMAScript); diff --git a/src/libstore/fetchers/regex.hh b/src/libutil/url.hh similarity index 75% rename from src/libstore/fetchers/regex.hh rename to src/libutil/url.hh index e0989edfc..1503023a2 100644 --- a/src/libstore/fetchers/regex.hh +++ b/src/libutil/url.hh @@ -1,8 +1,33 @@ #pragma once +#include "types.hh" + #include -namespace nix::fetchers { +namespace nix { + +struct ParsedURL +{ + std::string url; + std::string base; // URL without query/fragment + std::string scheme; + std::optional authority; + std::string path; + std::map query; + std::string fragment; + + std::string to_string() const; + + bool operator ==(const ParsedURL & other) const; +}; + +MakeError(BadURL, Error); + +std::string percentDecode(std::string_view in); + +std::map decodeQuery(const std::string & query); + +ParsedURL parseURL(const std::string & url); // URI stuff. const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 8b0937c53..17f7c149f 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -3,7 +3,7 @@ #include "download.hh" #include "store-api.hh" #include "legacy.hh" -#include "fetchers/fetchers.hh" +#include "fetchers.hh" #include #include diff --git a/src/nix/flake.cc b/src/nix/flake.cc index e5aa424b5..2cc61932d 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -9,8 +9,8 @@ #include "store-api.hh" #include "derivations.hh" #include "attr-path.hh" -#include "fetchers/fetchers.hh" -#include "fetchers/registry.hh" +#include "fetchers.hh" +#include "registry.hh" #include "json.hh" #include diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 64ea1e000..99bbe9769 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -10,7 +10,7 @@ #include "shared.hh" #include "flake/flake.hh" #include "flake/eval-cache.hh" -#include "fetchers/parse.hh" +#include "url.hh" #include #include diff --git a/src/nix/local.mk b/src/nix/local.mk index 6483000db..622f49019 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -15,7 +15,7 @@ nix_SOURCES := \ $(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-store/*.cc) \ -nix_LIBS = libexpr libmain libstore libutil libnixrust +nix_LIBS = libexpr libmain libfetchers libstore libutil libnixrust nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system