From bc00fa46472c56ccfddc2d6e81453be537d2e051 Mon Sep 17 00:00:00 2001 From: DavHau Date: Fri, 19 Jan 2024 15:59:15 +0700 Subject: [PATCH] fetchTree/fetchGit: re-enable shallow fetching Add several tests for git fetching: - shallow-cache-separation: can fetch the same repo shallowly and non-shallowly - shallow-ignore-ref: ensure that ref gets ignored when shallow=true is set - ssh-shallow: can fetch a git repo via ssh using shallow=1 --- src/libfetchers/git-utils.cc | 9 ++- src/libfetchers/git.cc | 16 ++++-- .../shallow-cache-separation/default.nix | 57 +++++++++++++++++++ .../test-cases/shallow-ignore-ref/default.nix | 40 +++++++++++++ .../test-cases/ssh-shallow/default.nix | 52 +++++++++++++++++ .../fetch-git/testsupport/gitea-repo.nix | 2 +- 6 files changed, 168 insertions(+), 8 deletions(-) create mode 100644 tests/nixos/fetch-git/test-cases/shallow-cache-separation/default.nix create mode 100644 tests/nixos/fetch-git/test-cases/shallow-ignore-ref/default.nix create mode 100644 tests/nixos/fetch-git/test-cases/ssh-shallow/default.nix diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 911c16c4b..382a363f0 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -387,13 +387,20 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this // then use code that was removed in this commit (see blame) auto dir = this->path; + Strings gitArgs; + if (shallow) { + gitArgs = { "-C", dir.abs(), "fetch", "--quiet", "--force", "--depth", "1", "--", url, refspec }; + } + else { + gitArgs = { "-C", dir.abs(), "fetch", "--quiet", "--force", "--", url, refspec }; + } runProgram(RunOptions { .program = "git", .searchPath = true, // FIXME: git stderr messes up our progress indicator, so // we're using --quiet for now. Should process its stderr. - .args = { "-C", path.abs(), "fetch", "--quiet", "--force", "--", url, refspec }, + .args = gitArgs, .input = {}, .isInteractive = true }); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 6ecb7a4ea..f9a1cb1bc 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -50,10 +50,12 @@ bool touchCacheFile(const Path & path, time_t touch_time) return lutimes(path.c_str(), times) == 0; } -Path getCachePath(std::string_view key) +Path getCachePath(std::string_view key, bool shallow) { - return getCacheDir() + "/nix/gitv3/" + - hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false); + return getCacheDir() + + "/nix/gitv3/" + + hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false) + + (shallow ? "-shallow" : ""); } // Returns the name of the HEAD branch. @@ -92,7 +94,8 @@ std::optional readHead(const Path & path) // Persist the HEAD ref from the remote repo in the local cached repo. bool storeCachedHead(const std::string & actualUrl, const std::string & headRef) { - Path cacheDir = getCachePath(actualUrl); + // set shallow=false as HEAD will never be queried for a shallow repo + Path cacheDir = getCachePath(actualUrl, false); try { runProgram("git", true, { "-C", cacheDir, "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef }); } catch (ExecError &e) { @@ -107,7 +110,8 @@ std::optional readHeadCached(const std::string & actualUrl) { // Create a cache path to store the branch of the HEAD ref. Append something // in front of the URL to prevent collision with the repository itself. - Path cacheDir = getCachePath(actualUrl); + // set shallow=false as HEAD will never be queried for a shallow repo + Path cacheDir = getCachePath(actualUrl, false); Path headRefFile = cacheDir + "/HEAD"; time_t now = time(0); @@ -508,7 +512,7 @@ struct GitInputScheme : InputScheme if (!input.getRev()) input.attrs.insert_or_assign("rev", GitRepo::openRepo(CanonPath(repoDir))->resolveRef(ref).gitRev()); } else { - Path cacheDir = getCachePath(repoInfo.url); + Path cacheDir = getCachePath(repoInfo.url, getShallowAttr(input)); repoDir = cacheDir; repoInfo.gitDir = "."; diff --git a/tests/nixos/fetch-git/test-cases/shallow-cache-separation/default.nix b/tests/nixos/fetch-git/test-cases/shallow-cache-separation/default.nix new file mode 100644 index 000000000..57561e74b --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/shallow-cache-separation/default.nix @@ -0,0 +1,57 @@ +{ + description = "can fetch the same repo shallowly and non-shallowly"; + script = '' + # create branch1 off of main + client.succeed(f""" + echo chiang-mai > {repo.path}/thailand \ + && {repo.git} add thailand \ + && {repo.git} commit -m 'commit1' \ + \ + && {repo.git} push origin --all + """) + + # save the revision + mainRev = client.succeed(f""" + {repo.git} rev-parse main + """).strip() + + # fetch shallowly + revCountShallow = client.succeed(f""" + nix eval --impure --expr ' + (builtins.fetchGit {{ + url = "{repo.remote}"; + rev = "{mainRev}"; + shallow = true; + }}).revCount + ' + """).strip() + # ensure the revCount is 0 + assert revCountShallow == "0", f"revCountShallow should be 0, but is {revCountShallow}" + + # fetch non-shallowly + revCountNonShallow = client.succeed(f""" + nix eval --impure --expr ' + (builtins.fetchGit {{ + url = "{repo.remote}"; + rev = "{mainRev}"; + shallow = false; + }}).revCount + ' + """).strip() + # ensure the revCount is 1 + assert revCountNonShallow == "1", f"revCountNonShallow should be 1, but is {revCountNonShallow}" + + # fetch shallowly again + revCountShallow2 = client.succeed(f""" + nix eval --impure --expr ' + (builtins.fetchGit {{ + url = "{repo.remote}"; + rev = "{mainRev}"; + shallow = true; + }}).revCount + ' + """).strip() + # ensure the revCount is 0 + assert revCountShallow2 == "0", f"revCountShallow2 should be 0, but is {revCountShallow2}" + ''; +} diff --git a/tests/nixos/fetch-git/test-cases/shallow-ignore-ref/default.nix b/tests/nixos/fetch-git/test-cases/shallow-ignore-ref/default.nix new file mode 100644 index 000000000..456ee8341 --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/shallow-ignore-ref/default.nix @@ -0,0 +1,40 @@ +{ + description = "ensure that ref gets ignored when shallow=true is set"; + script = '' + # create branch1 off of main + client.succeed(f""" + echo chiang-mai > {repo.path}/thailand \ + && {repo.git} add thailand \ + && {repo.git} commit -m 'commit1' \ + \ + && {repo.git} checkout -b branch1 main \ + && echo bangkok > {repo.path}/thailand \ + && {repo.git} add thailand \ + && {repo.git} commit -m 'commit2' \ + \ + && {repo.git} push origin --all + """) + + # save the revisions + mainRev = client.succeed(f""" + {repo.git} rev-parse main + """).strip() + branch1Rev = client.succeed(f""" + {repo.git} rev-parse branch1 + """).strip() + + # Ensure that ref gets ignored when fetching shallowly. + # This would fail if the ref was respected, as branch1Rev is not on main. + client.succeed(f""" + nix eval --impure --raw --expr ' + (builtins.fetchGit {{ + url = "{repo.remote}"; + rev = "{branch1Rev}"; + ref = "main"; + shallow = true; + }}) + ' + """) + + ''; +} diff --git a/tests/nixos/fetch-git/test-cases/ssh-shallow/default.nix b/tests/nixos/fetch-git/test-cases/ssh-shallow/default.nix new file mode 100644 index 000000000..979512af9 --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/ssh-shallow/default.nix @@ -0,0 +1,52 @@ +{ + description = "can fetch a git repo via ssh using shallow=1"; + script = '' + # add a file to the repo + client.succeed(f""" + echo chiang-mai > {repo.path}/thailand \ + && {repo.git} add thailand \ + && {repo.git} commit -m 'commit1' + """) + + # memoize the revision + rev1 = client.succeed(f""" + {repo.git} rev-parse HEAD + """).strip() + + # push to the server + client.succeed(f""" + {repo.git} push origin-ssh main + """) + + fetchGit_expr = f""" + builtins.fetchGit {{ + url = "{repo.remote_ssh}"; + rev = "{rev1}"; + shallow = true; + }} + """ + + # fetch the repo via nix + fetched1 = client.succeed(f""" + nix eval --impure --raw --expr '({fetchGit_expr}).outPath' + """) + + # check if the committed file is there + client.succeed(f""" + test -f {fetched1}/thailand + """) + + # check if the revision is the same + rev1_fetched = client.succeed(f""" + nix eval --impure --raw --expr '({fetchGit_expr}).rev' + """).strip() + assert rev1 == rev1_fetched, f"rev1: {rev1} != rev1_fetched: {rev1_fetched}" + + # check if revCount is 1 + revCount1 = client.succeed(f""" + nix eval --impure --expr '({fetchGit_expr}).revCount' + """).strip() + print(f"revCount1: {revCount1}") + assert revCount1 == '0', f"rev count is not 0 but {revCount1}" + ''; +} diff --git a/tests/nixos/fetch-git/testsupport/gitea-repo.nix b/tests/nixos/fetch-git/testsupport/gitea-repo.nix index a3ad65ca4..e9f4adcc1 100644 --- a/tests/nixos/fetch-git/testsupport/gitea-repo.nix +++ b/tests/nixos/fetch-git/testsupport/gitea-repo.nix @@ -72,4 +72,4 @@ in """) ''; }; -} \ No newline at end of file +}