From 99bd12f0b18b1a2a94639134c49c478c9ab56b3b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 11 Dec 2023 22:36:08 +0100 Subject: [PATCH] fetchGit/fetchTree: Improve exportIgnore, submodule interaction Also fingerprint and some preparatory improvements. Testing is still not up to scratch because lots of logic is duplicated between the workdir and commit cases. --- src/libexpr/primops/fetchTree.cc | 16 ++++++---- src/libfetchers/fetchers.hh | 7 +++++ src/libfetchers/git-utils.cc | 43 +++++++++++++++++++++----- src/libfetchers/git.cc | 9 ++++-- tests/functional/fetchGitSubmodules.sh | 42 +++++++++++++++++++++++++ 5 files changed, 101 insertions(+), 16 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index c167444b0..7a4725334 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -116,11 +116,6 @@ static void fetchTree( attrs.emplace("type", type.value()); - if (params.isFetchGit) { - // Default value; user attrs are assigned later. - attrs.emplace("exportIgnore", Explicit{true}); - } - for (auto & attr : *args[0]->attrs) { if (attr.name == state.sType) continue; state.forceValue(*attr.value, attr.pos); @@ -144,6 +139,12 @@ static void fetchTree( state.symbols[attr.name], showType(*attr.value))); } + if (params.isFetchGit && !attrs.contains("exportIgnore")) { + // Default value; user attrs are assigned later. + // FIXME: exportIgnore := !submodules + attrs.emplace("exportIgnore", Explicit{true}); + } + if (!params.allowNameArgument) if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) state.debugThrowLastTrace(EvalError({ @@ -161,7 +162,10 @@ static void fetchTree( fetchers::Attrs attrs; attrs.emplace("type", "git"); attrs.emplace("url", fixGitURL(url)); - attrs.emplace("exportIgnore", Explicit{true}); + if (!attrs.contains("exportIgnore")) { + // FIXME: exportIgnore := !submodules + attrs.emplace("exportIgnore", Explicit{true}); + } input = fetchers::Input::fromAttrs(std::move(attrs)); } else { if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 5f3254b6d..036647830 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -187,6 +187,13 @@ struct InputScheme virtual bool isDirect(const Input & input) const { return true; } + /** + * A sufficiently unique string that can be used as a cache key to identify the `input`. + * + * Only known-equivalent inputs should return the same fingerprint. + * + * This is not a stable identifier between Nix versions, but not guaranteed to change either. + */ virtual std::optional getFingerprint(ref store, const Input & input) const { return std::nullopt; } }; diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index d218276b4..cd65e0fda 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -662,14 +662,45 @@ struct GitInputAccessor : InputAccessor struct GitExportIgnoreInputAccessor : FilteringInputAccessor { ref repo; + std::optional rev; - GitExportIgnoreInputAccessor(ref repo, ref next) + GitExportIgnoreInputAccessor(ref repo, ref next, std::optional rev) : FilteringInputAccessor(next, [&](const CanonPath & path) { return RestrictedPathError(fmt("'%s' does not exist because it was fetched with exportIgnore enabled", path)); }) , repo(repo) + , rev(rev) { } + bool gitAttrGet(const CanonPath & path, const char * attrName, const char * & valueOut) + { + std::string pathStr {path.rel()}; + const char * pathCStr = pathStr.c_str(); + + if (rev) { + git_attr_options opts = GIT_ATTR_OPTIONS_INIT; + opts.attr_commit_id = hashToOID(*rev); + // TODO: test that gitattributes from global and system are not used + // (ie more or less: home and etc - both of them!) + opts.flags = GIT_ATTR_CHECK_INCLUDE_COMMIT | GIT_ATTR_CHECK_NO_SYSTEM; + return git_attr_get_ext( + &valueOut, + *repo, + &opts, + pathCStr, + attrName + ); + } + else { + return git_attr_get( + &valueOut, + *repo, + GIT_ATTR_CHECK_INDEX_ONLY | GIT_ATTR_CHECK_NO_SYSTEM, + pathCStr, + attrName); + } + } + bool isExportIgnored(const CanonPath & path) { const char *exportIgnoreEntry = nullptr; @@ -677,11 +708,7 @@ struct GitExportIgnoreInputAccessor : FilteringInputAccessor { // > It will use index only for creating archives or for a bare repo // > (if an index has been specified for the bare repo). // -- https://github.com/libgit2/libgit2/blob/HEAD/include/git2/attr.h#L113C62-L115C48 - if (git_attr_get(&exportIgnoreEntry, - *repo, - GIT_ATTR_CHECK_INDEX_ONLY, - std::string(path.rel()).c_str(), - "export-ignore")) { + if (gitAttrGet(path, "export-ignore", exportIgnoreEntry)) { if (git_error_last()->klass == GIT_ENOTFOUND) return false; else @@ -711,7 +738,7 @@ ref GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore) auto self = ref(shared_from_this()); ref rawGitAccessor = getRawAccessor(rev); if (exportIgnore) { - return make_ref(self, rawGitAccessor); + return make_ref(self, rawGitAccessor, rev); } else { return rawGitAccessor; @@ -727,7 +754,7 @@ ref GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportI std::set { wd.files }, std::move(makeNotAllowedError)); if (exportIgnore) { - return make_ref(self, fileAccessor); + return make_ref(self, fileAccessor, std::nullopt); } else { return fileAccessor; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index d7818988f..10c0aef97 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -628,6 +628,7 @@ struct GitInputScheme : InputScheme if (submodule.branch != "") attrs.insert_or_assign("ref", submodule.branch); attrs.insert_or_assign("rev", submoduleRev.gitRev()); + attrs.insert_or_assign("exportIgnore", Explicit{ exportIgnore }); auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store); @@ -660,9 +661,11 @@ struct GitInputScheme : InputScheme auto repo = GitRepo::openRepo(CanonPath(repoInfo.url), false, false); + auto exportIgnore = getExportIgnoreAttr(input); + ref accessor = repo->getAccessor(repoInfo.workdirInfo, - getExportIgnoreAttr(input), + exportIgnore, makeNotAllowedError(repoInfo.url)); /* If the repo has submodules, return a mounted input accessor @@ -676,6 +679,8 @@ struct GitInputScheme : InputScheme fetchers::Attrs attrs; attrs.insert_or_assign("type", "git"); attrs.insert_or_assign("url", submodulePath.abs()); + attrs.insert_or_assign("exportIgnore", Explicit{ exportIgnore }); + auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store); @@ -747,7 +752,7 @@ struct GitInputScheme : InputScheme std::optional getFingerprint(ref store, const Input & input) const override { if (auto rev = input.getRev()) - return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : ""); + return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : ""); else return std::nullopt; } diff --git a/tests/functional/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh index 369cdc5db..1b425820e 100644 --- a/tests/functional/fetchGitSubmodules.sh +++ b/tests/functional/fetchGitSubmodules.sh @@ -118,3 +118,45 @@ cloneRepo=$TEST_ROOT/a/b/gitSubmodulesClone # NB /a/b to make the relative path git clone $rootRepo $cloneRepo pathIndirect=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath") [[ $pathIndirect = $pathWithRelative ]] + +# Test submodule export-ignore interaction +git -C $rootRepo/sub config user.email "foobar@example.com" +git -C $rootRepo/sub config user.name "Foobar" + +echo "/exclude-from-root export-ignore" >> $rootRepo/.gitattributes +echo nope > $rootRepo/exclude-from-root +git -C $rootRepo add .gitattributes exclude-from-root +git -C $rootRepo commit -m "Add export-ignore" + +echo "/exclude-from-sub export-ignore" >> $rootRepo/sub/.gitattributes +echo nope > $rootRepo/sub/exclude-from-sub +git -C $rootRepo/sub add .gitattributes exclude-from-sub +git -C $rootRepo/sub commit -m "Add export-ignore (sub)" + +git -C $rootRepo add sub +git -C $rootRepo commit -m "Update submodule" + +git -C $rootRepo status + +# exportIgnore can be used with submodules +pathWithExportIgnore=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; exportIgnore = true; }).outPath") +# find $pathWithExportIgnore +# git -C $rootRepo archive --format=tar HEAD | tar -t +# cp -a $rootRepo /tmp/rootRepo + +[[ -e $pathWithExportIgnore/sub/content ]] +[[ ! -e $pathWithExportIgnore/exclude-from-root ]] +[[ ! -e $pathWithExportIgnore/sub/exclude-from-sub ]] + +# exportIgnore can be explicitly disabled with submodules +pathWithoutExportIgnore=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; exportIgnore = false; }).outPath") +# find $pathWithoutExportIgnore + +[[ -e $pathWithoutExportIgnore/exclude-from-root ]] +[[ -e $pathWithoutExportIgnore/sub/exclude-from-sub ]] + +# exportIgnore defaults to false when submodules = true +pathWithSubmodules=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; }).outPath") + +[[ -e $pathWithoutExportIgnore/exclude-from-root ]] +[[ -e $pathWithoutExportIgnore/sub/exclude-from-sub ]]