fetchGit/fetchTree: Improve exportIgnore, submodule interaction

Also fingerprint and some preparatory improvements.

Testing is still not up to scratch because lots of logic is duplicated
between the workdir and commit cases.

src/libexpr/primops/fetchTree.cc

@@ -116,11 +116,6 @@ static void fetchTree(
 
         attrs.emplace("type", type.value());
 
-        if (params.isFetchGit) {
-            // Default value; user attrs are assigned later.
-            attrs.emplace("exportIgnore", Explicit<bool>{true});
-        }
-
         for (auto & attr : *args[0]->attrs) {
             if (attr.name == state.sType) continue;
             state.forceValue(*attr.value, attr.pos);
@@ -144,6 +139,12 @@ static void fetchTree(
                     state.symbols[attr.name], showType(*attr.value)));
         }
 
+        if (params.isFetchGit && !attrs.contains("exportIgnore")) {
+            // Default value; user attrs are assigned later.
+            // FIXME: exportIgnore := !submodules
+            attrs.emplace("exportIgnore", Explicit<bool>{true});
+        }
+
         if (!params.allowNameArgument)
             if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
                 state.debugThrowLastTrace(EvalError({
@@ -161,7 +162,10 @@ static void fetchTree(
             fetchers::Attrs attrs;
             attrs.emplace("type", "git");
             attrs.emplace("url", fixGitURL(url));
-            attrs.emplace("exportIgnore", Explicit<bool>{true});
+            if (!attrs.contains("exportIgnore")) {
+                // FIXME: exportIgnore := !submodules
+                attrs.emplace("exportIgnore", Explicit<bool>{true});
+            }
             input = fetchers::Input::fromAttrs(std::move(attrs));
         } else {
             if (!experimentalFeatureSettings.isEnabled(Xp::Flakes))

src/libfetchers/fetchers.hh

@@ -187,6 +187,13 @@ struct InputScheme
     virtual bool isDirect(const Input & input) const
     { return true; }
 
+    /**
+     * A sufficiently unique string that can be used as a cache key to identify the `input`.
+     *
+     * Only known-equivalent inputs should return the same fingerprint.
+     *
+     * This is not a stable identifier between Nix versions, but not guaranteed to change either.
+     */
     virtual std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const
     { return std::nullopt; }
 };

src/libfetchers/git-utils.cc

@@ -662,14 +662,45 @@ struct GitInputAccessor : InputAccessor
 
 struct GitExportIgnoreInputAccessor : FilteringInputAccessor {
     ref<GitRepoImpl> repo;
+    std::optional<Hash> rev;
 
-    GitExportIgnoreInputAccessor(ref<GitRepoImpl> repo, ref<InputAccessor> next)
+    GitExportIgnoreInputAccessor(ref<GitRepoImpl> repo, ref<InputAccessor> next, std::optional<Hash> rev)
         : FilteringInputAccessor(next, [&](const CanonPath & path) {
             return RestrictedPathError(fmt("'%s' does not exist because it was fetched with exportIgnore enabled", path));
         })
         , repo(repo)
+        , rev(rev)
     { }
 
+    bool gitAttrGet(const CanonPath & path, const char * attrName, const char * & valueOut)
+    {
+        std::string pathStr {path.rel()};
+        const char * pathCStr = pathStr.c_str();
+
+        if (rev) {
+            git_attr_options opts = GIT_ATTR_OPTIONS_INIT;
+            opts.attr_commit_id = hashToOID(*rev);
+            // TODO: test that gitattributes from global and system are not used
+            //       (ie more or less: home and etc - both of them!)
+            opts.flags = GIT_ATTR_CHECK_INCLUDE_COMMIT | GIT_ATTR_CHECK_NO_SYSTEM;
+            return git_attr_get_ext(
+                &valueOut,
+                *repo,
+                &opts,
+                pathCStr,
+                attrName
+                );
+        }
+        else {
+            return git_attr_get(
+                &valueOut,
+                *repo,
+                GIT_ATTR_CHECK_INDEX_ONLY | GIT_ATTR_CHECK_NO_SYSTEM,
+                pathCStr,
+                attrName);
+        }
+    }
+
     bool isExportIgnored(const CanonPath & path) {
         const char *exportIgnoreEntry = nullptr;
 
@@ -677,11 +708,7 @@ struct GitExportIgnoreInputAccessor : FilteringInputAccessor {
         // > It will use index only for creating archives or for a bare repo
         // > (if an index has been specified for the bare repo).
         // -- https://github.com/libgit2/libgit2/blob/HEAD/include/git2/attr.h#L113C62-L115C48
-        if (git_attr_get(&exportIgnoreEntry,
-                *repo,
-                GIT_ATTR_CHECK_INDEX_ONLY,
-                std::string(path.rel()).c_str(),
-                "export-ignore")) {
+        if (gitAttrGet(path, "export-ignore", exportIgnoreEntry)) {
             if (git_error_last()->klass == GIT_ENOTFOUND)
                 return false;
             else
@@ -711,7 +738,7 @@ ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore)
     auto self = ref<GitRepoImpl>(shared_from_this());
     ref<GitInputAccessor> rawGitAccessor = getRawAccessor(rev);
     if (exportIgnore) {
-        return make_ref<GitExportIgnoreInputAccessor>(self, rawGitAccessor);
+        return make_ref<GitExportIgnoreInputAccessor>(self, rawGitAccessor, rev);
     }
     else {
         return rawGitAccessor;
@@ -727,7 +754,7 @@ ref<InputAccessor> GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportI
                 std::set<CanonPath> { wd.files },
                 std::move(makeNotAllowedError));
     if (exportIgnore) {
-        return make_ref<GitExportIgnoreInputAccessor>(self, fileAccessor);
+        return make_ref<GitExportIgnoreInputAccessor>(self, fileAccessor, std::nullopt);
     }
     else {
         return fileAccessor;

src/libfetchers/git.cc

@@ -628,6 +628,7 @@ struct GitInputScheme : InputScheme
                 if (submodule.branch != "")
                     attrs.insert_or_assign("ref", submodule.branch);
                 attrs.insert_or_assign("rev", submoduleRev.gitRev());
+                attrs.insert_or_assign("exportIgnore", Explicit<bool>{ exportIgnore });
                 auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
                 auto [submoduleAccessor, submoduleInput2] =
                     submoduleInput.getAccessor(store);
@@ -660,9 +661,11 @@ struct GitInputScheme : InputScheme
 
         auto repo = GitRepo::openRepo(CanonPath(repoInfo.url), false, false);
 
+        auto exportIgnore = getExportIgnoreAttr(input);
+
         ref<InputAccessor> accessor =
             repo->getAccessor(repoInfo.workdirInfo,
-                getExportIgnoreAttr(input),
+                exportIgnore,
                 makeNotAllowedError(repoInfo.url));
 
         /* If the repo has submodules, return a mounted input accessor
@@ -676,6 +679,8 @@ struct GitInputScheme : InputScheme
                 fetchers::Attrs attrs;
                 attrs.insert_or_assign("type", "git");
                 attrs.insert_or_assign("url", submodulePath.abs());
+                attrs.insert_or_assign("exportIgnore", Explicit<bool>{ exportIgnore });
+
                 auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
                 auto [submoduleAccessor, submoduleInput2] =
                     submoduleInput.getAccessor(store);
@@ -747,7 +752,7 @@ struct GitInputScheme : InputScheme
     std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
     {
         if (auto rev = input.getRev())
-            return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : "");
+            return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "");
         else
             return std::nullopt;
     }

tests/functional/fetchGitSubmodules.sh

@@ -118,3 +118,45 @@ cloneRepo=$TEST_ROOT/a/b/gitSubmodulesClone # NB /a/b to make the relative path
 git clone $rootRepo $cloneRepo
 pathIndirect=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath")
 [[ $pathIndirect = $pathWithRelative ]]
+
+# Test submodule export-ignore interaction
+git -C $rootRepo/sub config user.email "foobar@example.com"
+git -C $rootRepo/sub config user.name "Foobar"
+
+echo "/exclude-from-root export-ignore" >> $rootRepo/.gitattributes
+echo nope > $rootRepo/exclude-from-root
+git -C $rootRepo add .gitattributes exclude-from-root
+git -C $rootRepo commit -m "Add export-ignore"
+
+echo "/exclude-from-sub export-ignore" >> $rootRepo/sub/.gitattributes
+echo nope > $rootRepo/sub/exclude-from-sub
+git -C $rootRepo/sub add .gitattributes exclude-from-sub
+git -C $rootRepo/sub commit -m "Add export-ignore (sub)"
+
+git -C $rootRepo add sub
+git -C $rootRepo commit -m "Update submodule"
+
+git -C $rootRepo status
+
+# exportIgnore can be used with submodules
+pathWithExportIgnore=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; exportIgnore = true; }).outPath")
+# find $pathWithExportIgnore
+# git -C $rootRepo archive --format=tar HEAD | tar -t
+# cp -a $rootRepo /tmp/rootRepo
+
+[[ -e $pathWithExportIgnore/sub/content ]]
+[[ ! -e $pathWithExportIgnore/exclude-from-root ]]
+[[ ! -e $pathWithExportIgnore/sub/exclude-from-sub ]]
+
+# exportIgnore can be explicitly disabled with submodules
+pathWithoutExportIgnore=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; exportIgnore = false; }).outPath")
+# find $pathWithoutExportIgnore
+
+[[ -e $pathWithoutExportIgnore/exclude-from-root ]]
+[[ -e $pathWithoutExportIgnore/sub/exclude-from-sub ]]
+
+# exportIgnore defaults to false when submodules = true
+pathWithSubmodules=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; }).outPath")
+
+[[ -e $pathWithoutExportIgnore/exclude-from-root ]]
+[[ -e $pathWithoutExportIgnore/sub/exclude-from-sub ]]