diff options
65 files changed, 2541 insertions, 1522 deletions
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3622fec742..cc54824c38 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -458,6 +458,21 @@ jobs: - run: ci/install-dependencies.sh - run: ci/run-static-analysis.sh - run: ci/check-directional-formatting.bash + rust-analysis: + needs: ci-config + if: needs.ci-config.outputs.enabled == 'yes' + env: + jobname: RustAnalysis + CI_JOB_IMAGE: ubuntu:rolling + runs-on: ubuntu-latest + container: ubuntu:rolling + concurrency: + group: rust-analysis-${{ github.ref }} + cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} + steps: + - uses: actions/checkout@v4 + - run: ci/install-dependencies.sh + - run: ci/run-rust-checks.sh sparse: needs: ci-config if: needs.ci-config.outputs.enabled == 'yes' diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f7d57d1ee9..b419a84e2c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -161,7 +161,7 @@ test:mingw64: - saas-windows-medium-amd64 before_script: - *windows_before_script - - choco install -y git meson ninja + - choco install -y git meson ninja rust-ms - Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1 - refreshenv @@ -212,6 +212,17 @@ static-analysis: - ./ci/run-static-analysis.sh - ./ci/check-directional-formatting.bash +rust-analysis: + image: ubuntu:rolling + stage: analyze + needs: [ ] + variables: + jobname: RustAnalysis + before_script: + - ./ci/install-dependencies.sh + script: + - ./ci/run-rust-checks.sh + check-whitespace: image: ubuntu:latest stage: analyze diff --git a/Cargo.toml b/Cargo.toml index 45c9b34981..2f51bf5d5f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ name = "gitcore" version = "0.1.0" edition = "2018" +rust-version = "1.49.0" [lib] crate-type = ["staticlib"] diff --git a/Documentation/BreakingChanges.adoc b/Documentation/BreakingChanges.adoc index 90b53abcea..f814450d2f 100644 --- a/Documentation/BreakingChanges.adoc +++ b/Documentation/BreakingChanges.adoc @@ -295,6 +295,26 @@ The command will be removed. + cf. <xmqqa59i45wc.fsf@gitster.g> +* Support for `core.preferSymlinkRefs=true` has been deprecated and will be + removed in Git 3.0. Writing symbolic refs as symbolic links will be phased + out in favor of using plain files using the textual representation of + symbolic refs. ++ +Symbolic references were initially always stored as a symbolic link. This was +changed in 9b143c6e15 (Teach update-ref about a symbolic ref stored in a +textfile., 2005-09-25), where a new textual symref format was introduced to +store those symbolic refs in a plain file. In 9f0bb90d16 +(core.prefersymlinkrefs: use symlinks for .git/HEAD, 2006-05-02), the Git +project switched the default to use the textual symrefs in favor of symbolic +links. ++ +The migration away from symbolic links has happened almost 20 years ago by now, +and there is no known reason why one should prefer them nowadays. Furthermore, +symbolic links are not supported on some platforms. ++ +Note that only the writing side for such symbolic links is deprecated. Reading +such symbolic links is still supported for now. + == Superseded features that will not be deprecated Some features have gained newer replacements that aim to improve the design in diff --git a/Documentation/RelNotes/2.51.2.adoc b/Documentation/RelNotes/2.51.2.adoc new file mode 100644 index 0000000000..f0be60333a --- /dev/null +++ b/Documentation/RelNotes/2.51.2.adoc @@ -0,0 +1,45 @@ +Git 2.51.2 Release Notes +======================== + +In addition to fixes for an unfortunate regression introduced in Git +2.51.1 that caused "git diff --quiet -w" to be not so quiet when there +are additions, deletions and conflicts, this maintenance release merges +more fixes/improvements that have landed on the master front, primarily +to make the CI part of the system a bit more robust. + + +Fixes since Git 2.51.1 +---------------------- + + * Recently we attempted to improve "git diff -w --quiet" and friends + to handle cases where patch output would be suppressed, but it + introduced a bug that emits unnecessary output, which has been + corrected. + + * The code to squelch output from "git diff -w --name-status" + etc. for paths that "git diff -w -p" would have stayed silent + leaked output from dry-run patch generation, which has been + corrected. + + * Windows "real-time monitoring" interferes with the execution of + tests and affects negatively in both correctness and performance, + which has been disabled in Gitlab CI. + + * An earlier addition to "git diff --no-index A B" to limit the + output with pathspec after the two directories misbehaved when + these directories were given with a trailing slash, which has been + corrected. + + * The "--short" option of "git status" that meant output for humans + and "-z" option to show NUL delimited output format did not mix + well, and colored some but not all things. The command has been + updated to color all elements consistently in such a case. + + * Unicode width table update. + + * Recent OpenSSH creates the Unix domain socket to communicate with + ssh-agent under $HOME instead of /tmp, which causes our test to + fail doe to overly long pathname in our test environment, which has + been worked around by using "ssh-agent -T". + +Also contains various documentation updates, code cleanups and minor fixups. diff --git a/Documentation/RelNotes/2.52.0.adoc b/Documentation/RelNotes/2.52.0.adoc index 9785b9dac0..a86e2c09e0 100644 --- a/Documentation/RelNotes/2.52.0.adoc +++ b/Documentation/RelNotes/2.52.0.adoc @@ -57,6 +57,19 @@ UI, Workflows & Features * Show 'P'ipe command in "git add -p". + * "git sparse-checkout" subcommand learned a new "clean" action to + prune otherwise unused working-tree files that are outside the + areas of interest. + + * "git fast-import" is taught to handle signed tags, just like it + recently learned to handle signed commits, in different ways. + + * A new configuration variable commitGraph.changedPaths allows to + turn "--changed-paths" on by default for "git commit-graph". + + * "Symlink symref" has been added to the list of things that will + disappear at Git 3.0 boundary. + Performance, Internal Implementation, Development Support etc. -------------------------------------------------------------- @@ -104,7 +117,6 @@ Performance, Internal Implementation, Development Support etc. * Adjust to the way newer versions of cURL selectively enable tracing options, so that our tests can continue to work. - (merge 1b5a6bfff3 jk/curl-global-trace-components later to maint). * The clear_alloc_state() API function was not fully clearing the structure for reuse, but since nobody reuses it, replace it with a @@ -138,6 +150,14 @@ Performance, Internal Implementation, Development Support etc. * Build procedure for a few credential helpers (in contrib/) have been updated. + * CI improvements to handle the recent Rust integration better. + + * The code in "git repack" machinery has been cleaned up to prepare + for incremental update of midx files. + + * Two slightly different ways to get at "all the packfiles" in API + has been cleaned up. + Fixes since v2.51 ----------------- @@ -147,11 +167,9 @@ including security updates, are included in this release. * During interactive rebase, using 'drop' on a merge commit lead to an error, which was incorrect. - (merge 4d491ade8f js/rebase-i-allow-drop-on-a-merge later to maint). * "git refs migrate" to migrate the reflog entries from a refs backend to another had a handful of bugs squashed. - (merge 465eff81de ps/reflog-migrate-fixes later to maint). * "git remote rename origin upstream" failed to move origin/HEAD to upstream/HEAD when origin/HEAD is unborn and performed other @@ -164,11 +182,9 @@ including security updates, are included in this release. * "git push" had a code path that led to BUG() but it should have been a die(), as it is a response to a usual but invalid end-user action to attempt pushing an object that does not exist. - (merge dfbfc2221b dl/push-missing-object-error later to maint). * Various bugs about rename handling in "ort" merge strategy have been fixed. - (merge f6ecb603ff en/ort-rename-fixes later to maint). * "git jump" (in contrib/) fails to parse the diff header correctly when a file has a space in its name, which has been corrected. @@ -179,7 +195,6 @@ including security updates, are included in this release. the prefix from the output, and oddballs like "-" (stdin) did not work correctly because of it. Correct the set-up by undoing what the set-up sequence did to cwd and prefix. - (merge e1d3d61a45 jc/diff-no-index-in-subdir later to maint). * Various options to "git diff" that makes comparison ignore certain aspects of the differences (like "space changes are ignored", @@ -188,8 +203,6 @@ including security updates, are included in this release. (merge b55e6d36eb ly/diff-name-only-with-diff-from-content later to maint). * The above caused regressions, which has been corrected. - (merge 623f7af2 jk/diff-from-contents-fix later to maint). - (merge 3da4413d jc/diff-from-contents-fix later to maint). * Documentation for "git rebase" has been updated. (merge 3f7f2b0359 je/doc-rebase later to maint). @@ -197,13 +210,11 @@ including security updates, are included in this release. * The start_delayed_progress() function in the progress eye-candy API did not clear its internal state, making an initial delay value larger than 1 second ineffective, which has been corrected. - (merge 457534d041 js/progress-delay-fix later to maint). * The compatObjectFormat extension is used to hide an incomplete feature that is not yet usable for any purpose other than developing the feature further. Document it as such to discourage its use by mere mortals. - (merge 716d905792 bc/doc-compat-object-format-not-working later to maint). * "git log -L..." compared trees of multiple parents with the tree of the merge result in an unnecessarily inefficient way. @@ -213,7 +224,6 @@ including security updates, are included in this release. repository, especially a partially cloned one, "git fetch" may mistakenly think some objects we do have are missing, which has been corrected. - (merge 8f32a5a6c0 jk/fetch-check-graph-objects-fix later to maint). * "git fetch" can clobber a symref that is dangling when the remote-tracking HEAD is set to auto update, which has been @@ -225,20 +235,16 @@ including security updates, are included in this release. * Manual page for "gitk" is updated with the current maintainer's name. - (merge bcb20dda83 js/doc-gitk-history later to maint). * Update the instructions for using GGG in the MyFirstContribution document to say that a GitHub PR could be made against `git/git` instead of `gitgitgadget/git`. - (merge 37001cdbc4 ds/doc-ggg-pr-fork-clarify later to maint). * Makefile tried to run multiple "cargo build" which would not work very well; serialize their execution to work around this problem. - (merge 0eeacde50e da/cargo-serialize later to maint). * "git repack --path-walk" lost objects in some corner cases, which has been corrected. - (merge 93afe9b060 ds/path-walk-repack-fix later to maint). * "git ls-files <pathspec>..." should not necessarily have to expand the index fully if a sparsified directory is excluded by the @@ -249,15 +255,12 @@ including security updates, are included in this release. * Windows "real-time monitoring" interferes with the execution of tests and affects negatively in both correctness and performance, which has been disabled in Gitlab CI. - (merge 608cf5b793 ps/gitlab-ci-disable-windows-monitoring later to maint). * A broken or malicious "git fetch" can say that it has the same object for many many times, and the upload-pack serving it can exhaust memory storing them redundantly, which has been corrected. - (merge 88a2dc68c8 ps/upload-pack-oom-protection later to maint). * A corner case bug in "git log -L..." has been corrected. - (merge e3106998ff sg/line-log-boundary-fixes later to maint). * "git rev-parse --short" and friends failed to disambiguate two objects with object names that share common prefix longer than 32 @@ -267,7 +270,6 @@ including security updates, are included in this release. * Some among "git add -p" and friends ignored color.diff and/or color.ui configuration variables, which is an old regression, which has been corrected. - (merge 1092cd6435 jk/add-i-color later to maint). * "git subtree" (in contrib/) did not work correctly when splitting squashed subtrees, which has been improved. @@ -283,7 +285,6 @@ including security updates, are included in this release. * "git rebase -i" failed to clean-up the commit log message when the command commits the final one in a chain of "fixup" commands, which has been corrected. - (merge 82a0a73e15 pw/rebase-i-cleanup-fix later to maint). * There are double frees and leaks around setup_revisions() API used in "git stash show", which has been fixed, and setup_revisions() @@ -294,7 +295,6 @@ including security updates, are included in this release. * Deal more gracefully with directory / file conflicts when the files backend is used for ref storage, by failing only the ones that are involved in the conflict while allowing others. - (merge 948b2ab0d8 kn/refs-files-case-insensitive later to maint). * "git last-modified" operating in non-recursive mode used to trigger a BUG(), which has been corrected. @@ -307,16 +307,13 @@ including security updates, are included in this release. * The "do you still use it?" message given by a command that is deeply deprecated and allow us to suggest alternatives has been updated. - (merge 54a60e5b38 kh/you-still-use-whatchanged-fix later to maint). * Clang-format update to let our control macros be formatted the way we had them traditionally, e.g., "for_each_string_list_item()" without space before the parentheses. - (merge 3721541d35 jt/clang-format-foreach-wo-space-before-parenthesis later to maint). * A few places where a size_t value was cast to curl_off_t without checking has been updated to use the existing helper function. - (merge ecc5749578 js/curl-off-t-fixes later to maint). * "git reflog write" did not honor the configured user.name/email which has been corrected. @@ -328,7 +325,6 @@ including security updates, are included in this release. environment, but Ubuntu replaced with "sudo" with an implementation that lacks the feature. Work this around by reinstalling the original version. - (merge fddb484255 ps/ci-avoid-broken-sudo-on-ubuntu later to maint). * The reftable backend learned to sanity check its on-disk data more carefully. @@ -355,16 +351,13 @@ including security updates, are included in this release. output with pathspec after the two directories misbehaved when these directories were given with a trailing slash, which has been corrected. - (merge c0bec06cfe jk/diff-no-index-with-pathspec-fix later to maint). * The "--short" option of "git status" that meant output for humans and "-z" option to show NUL delimited output format did not mix well, and colored some but not all things. The command has been updated to color all elements consistently in such a case. - (merge 50927f4f68 jk/status-z-short-fix later to maint). * Unicode width table update. - (merge 330a54099e tb/unicode-width-table-17 later to maint). * GPG signing test set-up has been broken for a year, which has been corrected. @@ -374,41 +367,29 @@ including security updates, are included in this release. ssh-agent under $HOME instead of /tmp, which causes our test to fail doe to overly long pathname in our test environment, which has been worked around by using "ssh-agent -T". - (merge b7fb2194b9 ps/t7528-ssh-agent-uds-workaround later to maint). + + * strbuf_split*() to split a string into multiple strbufs is often a + wrong API to use. A few uses of it have been removed by + simplifying the code. + (merge 2ab72a16d9 ob/gpg-interface-cleanup later to maint). + + * "git shortlog" knows "--committer" and "--author" options, which + the command line completion (in contrib/) did not handle well, + which has been corrected. + (merge c568fa8e1c kf/log-shortlog-completion-fix later to maint). + + * "git bisect" command did not react correctly to "git bisect help" + and "git bisect unknown", which has been corrected. + (merge 2bb3a012f3 rz/bisect-help-unknown later to maint). * Other code cleanup, docfix, build fix, etc. - (merge 823d537fa7 kh/doc-git-log-markup-fix later to maint). - (merge cf7efa4f33 rj/t6137-cygwin-fix later to maint). (merge 529a60a885 ua/t1517-short-help-tests later to maint). (merge 22d421fed9 ac/deglobal-fmt-merge-log-config later to maint). - (merge 741f36c7d9 kr/clone-synopsis-fix later to maint). (merge a60203a015 dk/t7005-editor-updates later to maint). - (merge 7d4a5fef7d ds/doc-count-objects-fix later to maint). (merge 16684b6fae ps/reftable-libgit2-cleanup later to maint). - (merge f38786baa7 ja/asciidoc-doctor-verbatim-fixes later to maint). - (merge 374579c6d4 kh/doc-interpret-trailers-markup-fix later to maint). - (merge 44dce6541c kh/doc-config-typofix later to maint). - (merge 785628b173 js/doc-sending-patch-via-thunderbird later to maint). (merge e5c27bd3d8 je/doc-add later to maint). (merge 13296ac909 ps/object-store-midx-dedup-info later to maint). - (merge 2f4bf83ffc km/alias-doc-markup-fix later to maint). - (merge b0d97aac19 kh/doc-markup-fixes later to maint). (merge f9a6705d9a tc/t0450-harden later to maint). - (merge c25651aefd ds/midx-write-fixes later to maint). - (merge 069c15d256 rs/object-name-extend-abbrev-len-update later to maint). - (merge bf5c224537 mm/worktree-doc-typofix later to maint). - (merge 31397bc4f7 kh/doc-fast-import-markup-fix later to maint). - (merge ac7096723b jc/doc-includeif-hasconfig-remote-url-fix later to maint). - (merge fafc9b08b8 ag/doc-sendmail-gmail-example-update later to maint). (merge a66fc22bf9 rs/get-oid-with-flags-cleanup later to maint). - (merge e1d062e8ba ps/odb-clean-stale-wrappers later to maint). - (merge fdd21ba116 mh/doc-credential-url-prefix later to maint). - (merge 1c573a3451 en/doc-merge-tree-describe-merge-base later to maint). - (merge 84a6bf7965 ja/doc-markup-attached-paragraph-fix later to maint). - (merge 399694384b kh/doc-patch-id-markup-fix later to maint). (merge 15b8abde07 js/mingw-includes-cleanup later to maint). - (merge 3860985105 js/unreachable-workaround-for-no-symlink-head later to maint). - (merge b3ac6e737d kh/doc-continued-paragraph-fix later to maint). (merge 2cebca0582 tb/cat-file-objectmode-update later to maint). - (merge 96978d7545 js/ci-github-actions-update later to maint). - (merge 0c4f1346ca so/t2401-use-test-path-helpers later to maint). diff --git a/Documentation/config/commitgraph.adoc b/Documentation/config/commitgraph.adoc index 7f8c9d6638..70a56c53d2 100644 --- a/Documentation/config/commitgraph.adoc +++ b/Documentation/config/commitgraph.adoc @@ -8,6 +8,17 @@ commitGraph.maxNewFilters:: Specifies the default value for the `--max-new-filters` option of `git commit-graph write` (c.f., linkgit:git-commit-graph[1]). +commitGraph.changedPaths:: + If true, then `git commit-graph write` will compute and write + changed-path Bloom filters by default, equivalent to passing + `--changed-paths`. If false or unset, changed-paths Bloom filters will + be written during `git commit-graph write` only if the filters already + exist in the current commit-graph file. This matches the default + behavior of `git commit-graph write` without any `--[no-]changed-paths` + option. To rewrite a commit-graph file without any filters, use the + `--no-changed-paths` option. Command-line option `--[no-]changed-paths` + always takes precedence over this configuration. Defaults to unset. + commitGraph.readChangedPaths:: Deprecated. Equivalent to commitGraph.changedPathsVersion=-1 if true, and commitGraph.changedPathsVersion=0 if false. (If commitGraph.changedPathVersion diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index e2de270c86..11efad189e 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -290,6 +290,9 @@ core.preferSymlinkRefs:: and other symbolic reference files, use symbolic links. This is sometimes needed to work with old scripts that expect HEAD to be a symbolic link. ++ +This configuration is deprecated and will be removed in Git 3.0. Symbolic refs +will always be written as textual symrefs. core.alternateRefsCommand:: When advertising tips of available history from an alternate, use the shell to diff --git a/Documentation/git-commit-graph.adoc b/Documentation/git-commit-graph.adoc index e9558173c0..6d19026035 100644 --- a/Documentation/git-commit-graph.adoc +++ b/Documentation/git-commit-graph.adoc @@ -71,7 +71,7 @@ take a while on large repositories. It provides significant performance gains for getting history of a directory or a file with `git log -- <path>`. If this option is given, future commit-graph writes will automatically assume that this option was intended. Use `--no-changed-paths` to stop storing this -data. +data. `--changed-paths` is implied by config `commitGraph.changedPaths=true`. + With the `--max-new-filters=<n>` option, generate at most `n` new Bloom filters (if `--changed-paths` is specified). If `n` is `-1`, no limit is diff --git a/Documentation/git-fast-import.adoc b/Documentation/git-fast-import.adoc index 85ed7a7270..b74179a6c8 100644 --- a/Documentation/git-fast-import.adoc +++ b/Documentation/git-fast-import.adoc @@ -66,6 +66,11 @@ fast-import stream! This option is enabled automatically for remote-helpers that use the `import` capability, as they are already trusted to run their own code. +--signed-tags=(verbatim|warn-verbatim|warn-strip|strip|abort):: + Specify how to handle signed tags. Behaves in the same way + as the same option in linkgit:git-fast-export[1], except that + default is 'verbatim' (instead of 'abort'). + --signed-commits=(verbatim|warn-verbatim|warn-strip|strip|abort):: Specify how to handle signed commits. Behaves in the same way as the same option in linkgit:git-fast-export[1], except that diff --git a/Documentation/git-sparse-checkout.adoc b/Documentation/git-sparse-checkout.adoc index b5fe5da041..0d1618f161 100644 --- a/Documentation/git-sparse-checkout.adoc +++ b/Documentation/git-sparse-checkout.adoc @@ -9,7 +9,7 @@ git-sparse-checkout - Reduce your working tree to a subset of tracked files SYNOPSIS -------- [verse] -'git sparse-checkout' (init | list | set | add | reapply | disable | check-rules) [<options>] +'git sparse-checkout' (init | list | set | add | reapply | disable | check-rules | clean) [<options>] DESCRIPTION @@ -111,6 +111,37 @@ flags, with the same meaning as the flags from the `set` command, in order to change which sparsity mode you are using without needing to also respecify all sparsity paths. +'clean':: + Opportunistically remove files outside of the sparse-checkout + definition. This command requires cone mode to use recursive + directory matches to determine which files should be removed. A + file is considered for removal if it is contained within a tracked + directory that is outside of the sparse-checkout definition. ++ +Some special cases, such as merge conflicts or modified files outside of +the sparse-checkout definition could lead to keeping files that would +otherwise be removed. Resolve conflicts, stage modifications, and use +`git sparse-checkout reapply` in conjunction with `git sparse-checkout +clean` to resolve these cases. ++ +This command can be used to be sure the sparse index works efficiently, +though it does not require enabling the sparse index feature via the +`index.sparse=true` configuration. ++ +To prevent accidental deletion of worktree files, the `clean` subcommand +will not delete any files without the `-f` or `--force` option, unless +the `clean.requireForce` config option is set to `false`. ++ +The `--dry-run` option will list the directories that would be removed +without deleting them. Running in this mode can be helpful to predict the +behavior of the clean comand or to determine which kinds of files are left +in the sparse directories. ++ +The `--verbose` option will list every file within the directories that +are considered for removal. This option is helpful to determine if those +files are actually important or perhaps to explain why the directory is +still present despite the current sparse-checkout. + 'disable':: Disable the `core.sparseCheckout` config setting, and restore the working directory to include all files. diff --git a/Documentation/git-tag.adoc b/Documentation/git-tag.adoc index 0f7badc116..cea3202fdb 100644 --- a/Documentation/git-tag.adoc +++ b/Documentation/git-tag.adoc @@ -3,7 +3,7 @@ git-tag(1) NAME ---- -git-tag - Create, list, delete or verify a tag object signed with GPG +git-tag - Create, list, delete or verify tags SYNOPSIS @@ -38,15 +38,17 @@ and `-a`, `-s`, and `-u <key-id>` are absent, `-a` is implied. Otherwise, a tag reference that points directly at the given object (i.e., a lightweight tag) is created. -A GnuPG signed tag object will be created when `-s` or `-u -<key-id>` is used. When `-u <key-id>` is not used, the -committer identity for the current user is used to find the -GnuPG key for signing. The configuration variable `gpg.program` -is used to specify custom GnuPG binary. +A cryptographically signed tag object will be created when `-s` or +`-u <key-id>` is used. The signing backend (GPG, X.509, SSH, etc.) is +controlled by the `gpg.format` configuration variable, defaulting to +OpenPGP. When `-u <key-id>` is not used, the committer identity for +the current user is used to find the key for signing. The +configuration variable `gpg.program` is used to specify a custom +signing binary. Tag objects (created with `-a`, `-s`, or `-u`) are called "annotated" tags; they contain a creation date, the tagger name and e-mail, a -tagging message, and an optional GnuPG signature. Whereas a +tagging message, and an optional cryptographic signature. Whereas a "lightweight" tag is simply a name for an object (usually a commit object). @@ -64,10 +66,12 @@ OPTIONS `-s`:: `--sign`:: - Make a GPG-signed tag, using the default e-mail address's key. - The default behavior of tag GPG-signing is controlled by `tag.gpgSign` - configuration variable if it exists, or disabled otherwise. - See linkgit:git-config[1]. + Make a cryptographically signed tag, using the default signing + key. The signing backend used depends on the `gpg.format` + configuration variable. The default key is determined by the + backend. For GPG, it's based on the committer's email address, + while for SSH it may be a specific key file or agent + identity. See linkgit:git-config[1]. `--no-sign`:: Override `tag.gpgSign` configuration variable that is @@ -75,7 +79,10 @@ OPTIONS `-u <key-id>`:: `--local-user=<key-id>`:: - Make a GPG-signed tag, using the given key. + Make a cryptographically signed tag using the given key. The + format of the <key-id> and the backend used depend on the + `gpg.format` configuration variable. See + linkgit:git-config[1]. `-f`:: `--force`:: @@ -87,7 +94,7 @@ OPTIONS `-v`:: `--verify`:: - Verify the GPG signature of the given tag names. + Verify the cryptographic signature of the given tags. `-n<num>`:: _<num>_ specifies how many lines from the annotation, if any, @@ -235,12 +242,23 @@ it in the repository configuration as follows: ------------------------------------- [user] - signingKey = <gpg-key-id> + signingKey = <key-id> ------------------------------------- +The signing backend can be chosen via the `gpg.format` configuration +variable, which defaults to `openpgp`. See linkgit:git-config[1] +for a list of other supported formats. + +The path to the program used for each signing backend can be specified +with the `gpg.<format>.program` configuration variable. For the +`openpgp` backend, `gpg.program` can be used as a synonym for +`gpg.openpgp.program`. See linkgit:git-config[1] for details. + `pager.tag` is only respected when listing tags, i.e., when `-l` is used or implied. The default is to use a pager. -See linkgit:git-config[1]. + +See linkgit:git-config[1] for more details and other configuration +variables. DISCUSSION ---------- @@ -927,10 +927,17 @@ export PYTHON_PATH TEST_SHELL_PATH = $(SHELL_PATH) LIB_FILE = libgit.a + ifdef DEBUG -RUST_LIB = target/debug/libgitcore.a +RUST_TARGET_DIR = target/debug else -RUST_LIB = target/release/libgitcore.a +RUST_TARGET_DIR = target/release +endif + +ifeq ($(uname_S),Windows) +RUST_LIB = $(RUST_TARGET_DIR)/gitcore.lib +else +RUST_LIB = $(RUST_TARGET_DIR)/libgitcore.a endif GITLIBS = common-main.o $(LIB_FILE) @@ -1260,6 +1267,12 @@ LIB_OBJS += reftable/table.o LIB_OBJS += reftable/tree.o LIB_OBJS += reftable/writer.o LIB_OBJS += remote.o +LIB_OBJS += repack.o +LIB_OBJS += repack-cruft.o +LIB_OBJS += repack-filtered.o +LIB_OBJS += repack-geometry.o +LIB_OBJS += repack-midx.o +LIB_OBJS += repack-promisor.o LIB_OBJS += replace-object.o LIB_OBJS += repo-settings.o LIB_OBJS += repository.o @@ -1556,6 +1569,9 @@ ALL_LDFLAGS = $(LDFLAGS) $(LDFLAGS_APPEND) ifdef WITH_RUST BASIC_CFLAGS += -DWITH_RUST GITLIBS += $(RUST_LIB) +ifeq ($(uname_S),Windows) +EXTLIBS += -luserenv +endif endif ifdef SANITIZE diff --git a/builtin/bisect.c b/builtin/bisect.c index 8b8d870cd1..993caf545d 100644 --- a/builtin/bisect.c +++ b/builtin/bisect.c @@ -1453,9 +1453,13 @@ int cmd_bisect(int argc, if (!argc) usage_msg_opt(_("need a command"), git_bisect_usage, options); + if (!strcmp(argv[0], "help")) + usage_with_options(git_bisect_usage, options); + set_terms(&terms, "bad", "good"); get_terms(&terms); - if (check_and_set_terms(&terms, argv[0])) + if (check_and_set_terms(&terms, argv[0]) || + !one_of(argv[0], terms.term_good, terms.term_bad, NULL)) usage_msg_optf(_("unknown command: '%s'"), git_bisect_usage, options, argv[0]); res = bisect_state(&terms, argc, argv); diff --git a/builtin/cat-file.c b/builtin/cat-file.c index 5ca2ca3852..983ecec837 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -852,10 +852,9 @@ static void batch_each_object(struct batch_options *opt, if (bitmap && !for_each_bitmapped_object(bitmap, &opt->objects_filter, batch_one_object_bitmapped, &payload)) { - struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *pack; - for (pack = packfile_store_get_all_packs(packs); pack; pack = pack->next) { + repo_for_each_pack(the_repository, pack) { if (bitmap_index_contains_pack(bitmap, pack) || open_pack_index(pack)) continue; diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c index fe3ebaadad..d62005edc0 100644 --- a/builtin/commit-graph.c +++ b/builtin/commit-graph.c @@ -210,6 +210,8 @@ static int git_commit_graph_write_config(const char *var, const char *value, { if (!strcmp(var, "commitgraph.maxnewfilters")) write_opts.max_new_filters = git_config_int(var, value, ctx->kvi); + else if (!strcmp(var, "commitgraph.changedpaths")) + opts.enable_changed_paths = git_config_bool(var, value) ? 1 : -1; /* * No need to fall-back to 'git_default_config', since this was already * called in 'cmd_commit_graph()'. diff --git a/builtin/count-objects.c b/builtin/count-objects.c index f2f407c2a7..18f6e33b6f 100644 --- a/builtin/count-objects.c +++ b/builtin/count-objects.c @@ -122,7 +122,6 @@ int cmd_count_objects(int argc, count_loose, count_cruft, NULL, NULL); if (verbose) { - struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; unsigned long num_pack = 0; off_t size_pack = 0; @@ -130,7 +129,7 @@ int cmd_count_objects(int argc, struct strbuf pack_buf = STRBUF_INIT; struct strbuf garbage_buf = STRBUF_INIT; - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { if (!p->pack_local) continue; if (open_pack_index(p)) diff --git a/builtin/fast-export.c b/builtin/fast-export.c index dc2486f9a8..7adbc55f0d 100644 --- a/builtin/fast-export.c +++ b/builtin/fast-export.c @@ -931,9 +931,8 @@ static void handle_tag(const char *name, struct tag *tag) /* handle signed tags */ if (message) { - const char *signature = strstr(message, - "\n-----BEGIN PGP SIGNATURE-----\n"); - if (signature) + size_t sig_offset = parse_signed_buffer(message, message_size); + if (sig_offset < message_size) switch (signed_tag_mode) { case SIGN_ABORT: die("encountered signed tag %s; use " @@ -950,7 +949,7 @@ static void handle_tag(const char *name, struct tag *tag) oid_to_hex(&tag->object.oid)); /* fallthru */ case SIGN_STRIP: - message_size = signature + 1 - message; + message_size = sig_offset; break; } } diff --git a/builtin/fast-import.c b/builtin/fast-import.c index 606c6aea82..54d3e592c6 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -188,6 +188,7 @@ static int global_argc; static const char **global_argv; static const char *global_prefix; +static enum sign_mode signed_tag_mode = SIGN_VERBATIM; static enum sign_mode signed_commit_mode = SIGN_VERBATIM; /* Memory pools */ @@ -978,7 +979,7 @@ static int store_object( if (e->idx.offset) { duplicate_count_by_type[type]++; return 1; - } else if (find_oid_pack(&oid, packfile_store_get_all_packs(packs))) { + } else if (find_oid_pack(&oid, packfile_store_get_packs(packs))) { e->type = type; e->pack_id = MAX_PACK_ID; e->idx.offset = 1; /* just not zero! */ @@ -1179,7 +1180,7 @@ static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark) duplicate_count_by_type[OBJ_BLOB]++; truncate_pack(&checkpoint); - } else if (find_oid_pack(&oid, packfile_store_get_all_packs(packs))) { + } else if (find_oid_pack(&oid, packfile_store_get_packs(packs))) { e->type = OBJ_BLOB; e->pack_id = MAX_PACK_ID; e->idx.offset = 1; /* just not zero! */ @@ -2963,6 +2964,43 @@ static void parse_new_commit(const char *arg) b->last_commit = object_count_by_type[OBJ_COMMIT]; } +static void handle_tag_signature(struct strbuf *msg, const char *name) +{ + size_t sig_offset = parse_signed_buffer(msg->buf, msg->len); + + /* If there is no signature, there is nothing to do. */ + if (sig_offset >= msg->len) + return; + + switch (signed_tag_mode) { + + /* First, modes that don't change anything */ + case SIGN_ABORT: + die(_("encountered signed tag; use " + "--signed-tags=<mode> to handle it")); + case SIGN_WARN_VERBATIM: + warning(_("importing a tag signature verbatim for tag '%s'"), name); + /* fallthru */ + case SIGN_VERBATIM: + /* Nothing to do, the signature will be put into the imported tag. */ + break; + + /* Second, modes that remove the signature */ + case SIGN_WARN_STRIP: + warning(_("stripping a tag signature for tag '%s'"), name); + /* fallthru */ + case SIGN_STRIP: + /* Truncate the buffer to remove the signature */ + strbuf_setlen(msg, sig_offset); + break; + + /* Third, BUG */ + default: + BUG("invalid signed_tag_mode value %d from tag '%s'", + signed_tag_mode, name); + } +} + static void parse_new_tag(const char *arg) { static struct strbuf msg = STRBUF_INIT; @@ -3026,6 +3064,8 @@ static void parse_new_tag(const char *arg) /* tag payload/message */ parse_data(&msg, 0, NULL); + handle_tag_signature(&msg, t->name); + /* build the tag object */ strbuf_reset(&new_data); @@ -3546,6 +3586,9 @@ static int parse_one_option(const char *option) } else if (skip_prefix(option, "signed-commits=", &option)) { if (parse_sign_mode(option, &signed_commit_mode)) usagef(_("unknown --signed-commits mode '%s'"), option); + } else if (skip_prefix(option, "signed-tags=", &option)) { + if (parse_sign_mode(option, &signed_tag_mode)) + usagef(_("unknown --signed-tags mode '%s'"), option); } else if (!strcmp(option, "quiet")) { show_stats = 0; quiet = 1; diff --git a/builtin/fsck.c b/builtin/fsck.c index 8ee95e0d67..b1a650c673 100644 --- a/builtin/fsck.c +++ b/builtin/fsck.c @@ -867,20 +867,20 @@ static int mark_packed_for_connectivity(const struct object_id *oid, static int check_pack_rev_indexes(struct repository *r, int show_progress) { - struct packfile_store *packs = r->objects->packfiles; struct progress *progress = NULL; + struct packed_git *p; uint32_t pack_count = 0; int res = 0; if (show_progress) { - for (struct packed_git *p = packfile_store_get_all_packs(packs); p; p = p->next) + repo_for_each_pack(r, p) pack_count++; progress = start_delayed_progress(the_repository, "Verifying reverse pack-indexes", pack_count); pack_count = 0; } - for (struct packed_git *p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(r, p) { int load_error = load_pack_revindex_from_disk(p); if (load_error < 0) { @@ -1000,8 +1000,6 @@ int cmd_fsck(int argc, for_each_packed_object(the_repository, mark_packed_for_connectivity, NULL, 0); } else { - struct packfile_store *packs = the_repository->objects->packfiles; - odb_prepare_alternates(the_repository->objects); for (source = the_repository->objects->sources; source; source = source->next) fsck_source(source); @@ -1012,8 +1010,7 @@ int cmd_fsck(int argc, struct progress *progress = NULL; if (show_progress) { - for (p = packfile_store_get_all_packs(packs); p; - p = p->next) { + repo_for_each_pack(the_repository, p) { if (open_pack_index(p)) continue; total += p->num_objects; @@ -1022,8 +1019,8 @@ int cmd_fsck(int argc, progress = start_progress(the_repository, _("Checking objects"), total); } - for (p = packfile_store_get_all_packs(packs); p; - p = p->next) { + + repo_for_each_pack(the_repository, p) { /* verify gives error messages itself */ if (verify_pack(the_repository, p, fsck_obj_buffer, diff --git a/builtin/gc.c b/builtin/gc.c index e19e13d978..541d7471f1 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -487,10 +487,9 @@ static int too_many_loose_objects(struct gc_config *cfg) static struct packed_git *find_base_packs(struct string_list *packs, unsigned long limit) { - struct packfile_store *packfiles = the_repository->objects->packfiles; struct packed_git *p, *base = NULL; - for (p = packfile_store_get_all_packs(packfiles); p; p = p->next) { + repo_for_each_pack(the_repository, p) { if (!p->pack_local || p->is_cruft) continue; if (limit) { @@ -509,14 +508,13 @@ static struct packed_git *find_base_packs(struct string_list *packs, static int too_many_packs(struct gc_config *cfg) { - struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; - int cnt; + int cnt = 0; if (cfg->gc_auto_pack_limit <= 0) return 0; - for (cnt = 0, p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { if (!p->pack_local) continue; if (p->pack_keep) @@ -1425,9 +1423,9 @@ static int incremental_repack_auto_condition(struct gc_config *cfg UNUSED) if (incremental_repack_auto_limit < 0) return 1; - for (p = packfile_store_get_packs(the_repository->objects->packfiles); - count < incremental_repack_auto_limit && p; - p = p->next) { + repo_for_each_pack(the_repository, p) { + if (count >= incremental_repack_auto_limit) + break; if (!p->multi_pack_index) count++; } @@ -1494,7 +1492,7 @@ static off_t get_auto_pack_size(void) struct repository *r = the_repository; odb_reprepare(r->objects); - for (p = packfile_store_get_all_packs(r->objects->packfiles); p; p = p->next) { + repo_for_each_pack(r, p) { if (p->pack_size > max_size) { second_largest_size = max_size; max_size = p->pack_size; diff --git a/builtin/grep.c b/builtin/grep.c index 13841fbf00..53cccf2d25 100644 --- a/builtin/grep.c +++ b/builtin/grep.c @@ -1214,7 +1214,7 @@ int cmd_grep(int argc, if (recurse_submodules) repo_read_gitmodules(the_repository, 1); if (startup_info->have_repository) - (void)packfile_store_get_packs(the_repository->objects->packfiles); + packfile_store_prepare(the_repository->objects->packfiles); start_threads(&opt); } else { diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 5bdc44fb2d..b5454e5df1 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -3831,12 +3831,10 @@ static int pack_mtime_cmp(const void *_a, const void *_b) static void read_packs_list_from_stdin(struct rev_info *revs) { - struct packfile_store *packs = the_repository->objects->packfiles; struct strbuf buf = STRBUF_INIT; struct string_list include_packs = STRING_LIST_INIT_DUP; struct string_list exclude_packs = STRING_LIST_INIT_DUP; struct string_list_item *item = NULL; - struct packed_git *p; while (strbuf_getline(&buf, stdin) != EOF) { @@ -3856,7 +3854,7 @@ static void read_packs_list_from_stdin(struct rev_info *revs) string_list_sort(&exclude_packs); string_list_remove_duplicates(&exclude_packs, 0); - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { const char *pack_name = pack_basename(p); if ((item = string_list_lookup(&include_packs, pack_name))) @@ -4077,7 +4075,6 @@ static void enumerate_cruft_objects(void) static void enumerate_and_traverse_cruft_objects(struct string_list *fresh_packs) { - struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; struct rev_info revs; int ret; @@ -4107,7 +4104,7 @@ static void enumerate_and_traverse_cruft_objects(struct string_list *fresh_packs * Re-mark only the fresh packs as kept so that objects in * unknown packs do not halt the reachability traversal early. */ - for (p = packfile_store_get_all_packs(packs); p; p = p->next) + repo_for_each_pack(the_repository, p) p->pack_keep_in_core = 0; mark_pack_kept_in_core(fresh_packs, 1); @@ -4124,7 +4121,6 @@ static void enumerate_and_traverse_cruft_objects(struct string_list *fresh_packs static void read_cruft_objects(void) { - struct packfile_store *packs = the_repository->objects->packfiles; struct strbuf buf = STRBUF_INIT; struct string_list discard_packs = STRING_LIST_INIT_DUP; struct string_list fresh_packs = STRING_LIST_INIT_DUP; @@ -4145,7 +4141,7 @@ static void read_cruft_objects(void) string_list_sort(&discard_packs); string_list_sort(&fresh_packs); - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { const char *pack_name = pack_basename(p); struct string_list_item *item; @@ -4398,7 +4394,7 @@ static int has_sha1_pack_kept_or_nonlocal(const struct object_id *oid) struct packed_git *p; p = (last_found != (void *)1) ? last_found : - packfile_store_get_all_packs(packs); + packfile_store_get_packs(packs); while (p) { if ((!p->pack_local || p->pack_keep || @@ -4408,7 +4404,7 @@ static int has_sha1_pack_kept_or_nonlocal(const struct object_id *oid) return 1; } if (p == last_found) - p = packfile_store_get_all_packs(packs); + p = packfile_store_get_packs(packs); else p = p->next; if (p == last_found) @@ -4440,13 +4436,12 @@ static int loosened_object_can_be_discarded(const struct object_id *oid, static void loosen_unused_packed_objects(void) { - struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; uint32_t i; uint32_t loosened_objects_nr = 0; struct object_id oid; - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { if (!p->pack_local || p->pack_keep || p->pack_keep_in_core) continue; @@ -4747,13 +4742,12 @@ static void get_object_list(struct rev_info *revs, struct strvec *argv) static void add_extra_kept_packs(const struct string_list *names) { - struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; if (!names->nr) return; - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { const char *name = basename(p->pack_name); int i; @@ -5191,10 +5185,9 @@ int cmd_pack_objects(int argc, add_extra_kept_packs(&keep_pack_list); if (ignore_packed_keep_on_disk) { - struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; - for (p = packfile_store_get_all_packs(packs); p; p = p->next) + repo_for_each_pack(the_repository, p) if (p->pack_local && p->pack_keep) break; if (!p) /* no keep-able packs found */ @@ -5206,10 +5199,9 @@ int cmd_pack_objects(int argc, * want to unset "local" based on looking at packs, as * it also covers non-local objects */ - struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { if (!p->pack_local) { have_non_local_packs = 1; break; diff --git a/builtin/pack-redundant.c b/builtin/pack-redundant.c index 80743d8806..e4ecf774ca 100644 --- a/builtin/pack-redundant.c +++ b/builtin/pack-redundant.c @@ -566,29 +566,23 @@ static struct pack_list * add_pack(struct packed_git *p) static struct pack_list * add_pack_file(const char *filename) { - struct packfile_store *packs = the_repository->objects->packfiles; - struct packed_git *p = packfile_store_get_all_packs(packs); + struct packed_git *p; if (strlen(filename) < 40) die("Bad pack filename: %s", filename); - while (p) { + repo_for_each_pack(the_repository, p) if (strstr(p->pack_name, filename)) return add_pack(p); - p = p->next; - } die("Filename %s not found in packed_git", filename); } static void load_all(void) { - struct packfile_store *packs = the_repository->objects->packfiles; - struct packed_git *p = packfile_store_get_all_packs(packs); + struct packed_git *p; - while (p) { + repo_for_each_pack(the_repository, p) add_pack(p); - p = p->next; - } } int cmd_pack_redundant(int argc, const char **argv, const char *prefix UNUSED, struct repository *repo UNUSED) { diff --git a/builtin/repack.c b/builtin/repack.c index e8730808c5..cfdb4c0920 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -3,27 +3,18 @@ #include "builtin.h" #include "config.h" -#include "dir.h" #include "environment.h" -#include "gettext.h" -#include "hex.h" #include "parse-options.h" #include "path.h" #include "run-command.h" #include "server-info.h" -#include "strbuf.h" #include "string-list.h" -#include "strvec.h" #include "midx.h" #include "packfile.h" #include "prune-packed.h" -#include "odb.h" #include "promisor-remote.h" +#include "repack.h" #include "shallow.h" -#include "pack.h" -#include "pack-bitmap.h" -#include "refs.h" -#include "list-objects-filter-options.h" #define ALL_INTO_ONE 1 #define LOOSEN_UNREACHABLE 2 @@ -33,8 +24,6 @@ #define RETAIN_PACK 2 static int pack_everything; -static int delta_base_offset = 1; -static int pack_kept_objects = -1; static int write_bitmaps = -1; static int use_delta_islands; static int run_update_server_info = 1; @@ -53,31 +42,23 @@ static const char incremental_bitmap_conflict_error[] = N_( "--no-write-bitmap-index or disable the pack.writeBitmaps configuration." ); -struct pack_objects_args { - char *window; - char *window_memory; - char *depth; - char *threads; - unsigned long max_pack_size; - int no_reuse_delta; - int no_reuse_object; - int quiet; - int local; - int name_hash_version; - int path_walk; - struct list_objects_filter_options filter_options; +struct repack_config_ctx { + struct pack_objects_args *po_args; + struct pack_objects_args *cruft_po_args; }; static int repack_config(const char *var, const char *value, const struct config_context *ctx, void *cb) { - struct pack_objects_args *cruft_po_args = cb; + struct repack_config_ctx *repack_ctx = cb; + struct pack_objects_args *po_args = repack_ctx->po_args; + struct pack_objects_args *cruft_po_args = repack_ctx->cruft_po_args; if (!strcmp(var, "repack.usedeltabaseoffset")) { - delta_base_offset = git_config_bool(var, value); + po_args->delta_base_offset = git_config_bool(var, value); return 0; } if (!strcmp(var, "repack.packkeptobjects")) { - pack_kept_objects = git_config_bool(var, value); + po_args->pack_kept_objects = git_config_bool(var, value); return 0; } if (!strcmp(var, "repack.writebitmaps") || @@ -116,1138 +97,10 @@ static int repack_config(const char *var, const char *value, return git_default_config(var, value, ctx, cb); } -static void pack_objects_args_release(struct pack_objects_args *args) -{ - free(args->window); - free(args->window_memory); - free(args->depth); - free(args->threads); - list_objects_filter_release(&args->filter_options); -} - -struct existing_packs { - struct string_list kept_packs; - struct string_list non_kept_packs; - struct string_list cruft_packs; -}; - -#define EXISTING_PACKS_INIT { \ - .kept_packs = STRING_LIST_INIT_DUP, \ - .non_kept_packs = STRING_LIST_INIT_DUP, \ - .cruft_packs = STRING_LIST_INIT_DUP, \ -} - -static int has_existing_non_kept_packs(const struct existing_packs *existing) -{ - return existing->non_kept_packs.nr || existing->cruft_packs.nr; -} - -static void pack_mark_for_deletion(struct string_list_item *item) -{ - item->util = (void*)((uintptr_t)item->util | DELETE_PACK); -} - -static void pack_unmark_for_deletion(struct string_list_item *item) -{ - item->util = (void*)((uintptr_t)item->util & ~DELETE_PACK); -} - -static int pack_is_marked_for_deletion(struct string_list_item *item) -{ - return (uintptr_t)item->util & DELETE_PACK; -} - -static void pack_mark_retained(struct string_list_item *item) -{ - item->util = (void*)((uintptr_t)item->util | RETAIN_PACK); -} - -static int pack_is_retained(struct string_list_item *item) -{ - return (uintptr_t)item->util & RETAIN_PACK; -} - -static void mark_packs_for_deletion_1(struct string_list *names, - struct string_list *list) -{ - struct string_list_item *item; - const int hexsz = the_hash_algo->hexsz; - - for_each_string_list_item(item, list) { - char *sha1; - size_t len = strlen(item->string); - if (len < hexsz) - continue; - sha1 = item->string + len - hexsz; - - if (pack_is_retained(item)) { - pack_unmark_for_deletion(item); - } else if (!string_list_has_string(names, sha1)) { - /* - * Mark this pack for deletion, which ensures - * that this pack won't be included in a MIDX - * (if `--write-midx` was given) and that we - * will actually delete this pack (if `-d` was - * given). - */ - pack_mark_for_deletion(item); - } - } -} - -static void retain_cruft_pack(struct existing_packs *existing, - struct packed_git *cruft) -{ - struct strbuf buf = STRBUF_INIT; - struct string_list_item *item; - - strbuf_addstr(&buf, pack_basename(cruft)); - strbuf_strip_suffix(&buf, ".pack"); - - item = string_list_lookup(&existing->cruft_packs, buf.buf); - if (!item) - BUG("could not find cruft pack '%s'", pack_basename(cruft)); - - pack_mark_retained(item); - strbuf_release(&buf); -} - -static void mark_packs_for_deletion(struct existing_packs *existing, - struct string_list *names) - -{ - mark_packs_for_deletion_1(names, &existing->non_kept_packs); - mark_packs_for_deletion_1(names, &existing->cruft_packs); -} - -static void remove_redundant_pack(const char *dir_name, const char *base_name) -{ - struct strbuf buf = STRBUF_INIT; - struct odb_source *source = the_repository->objects->sources; - struct multi_pack_index *m = get_multi_pack_index(source); - strbuf_addf(&buf, "%s.pack", base_name); - if (m && source->local && midx_contains_pack(m, buf.buf)) - clear_midx_file(the_repository); - strbuf_insertf(&buf, 0, "%s/", dir_name); - unlink_pack_path(buf.buf, 1); - strbuf_release(&buf); -} - -static void remove_redundant_packs_1(struct string_list *packs) -{ - struct string_list_item *item; - for_each_string_list_item(item, packs) { - if (!pack_is_marked_for_deletion(item)) - continue; - remove_redundant_pack(packdir, item->string); - } -} - -static void remove_redundant_existing_packs(struct existing_packs *existing) -{ - remove_redundant_packs_1(&existing->non_kept_packs); - remove_redundant_packs_1(&existing->cruft_packs); -} - -static void existing_packs_release(struct existing_packs *existing) -{ - string_list_clear(&existing->kept_packs, 0); - string_list_clear(&existing->non_kept_packs, 0); - string_list_clear(&existing->cruft_packs, 0); -} - -/* - * Adds all packs hex strings (pack-$HASH) to either packs->non_kept - * or packs->kept based on whether each pack has a corresponding - * .keep file or not. Packs without a .keep file are not to be kept - * if we are going to pack everything into one file. - */ -static void collect_pack_filenames(struct existing_packs *existing, - const struct string_list *extra_keep) -{ - struct packfile_store *packs = the_repository->objects->packfiles; - struct packed_git *p; - struct strbuf buf = STRBUF_INIT; - - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { - int i; - const char *base; - - if (!p->pack_local) - continue; - - base = pack_basename(p); - - for (i = 0; i < extra_keep->nr; i++) - if (!fspathcmp(base, extra_keep->items[i].string)) - break; - - strbuf_reset(&buf); - strbuf_addstr(&buf, base); - strbuf_strip_suffix(&buf, ".pack"); - - if ((extra_keep->nr > 0 && i < extra_keep->nr) || p->pack_keep) - string_list_append(&existing->kept_packs, buf.buf); - else if (p->is_cruft) - string_list_append(&existing->cruft_packs, buf.buf); - else - string_list_append(&existing->non_kept_packs, buf.buf); - } - - string_list_sort(&existing->kept_packs); - string_list_sort(&existing->non_kept_packs); - string_list_sort(&existing->cruft_packs); - strbuf_release(&buf); -} - -static void prepare_pack_objects(struct child_process *cmd, - const struct pack_objects_args *args, - const char *out) -{ - strvec_push(&cmd->args, "pack-objects"); - if (args->window) - strvec_pushf(&cmd->args, "--window=%s", args->window); - if (args->window_memory) - strvec_pushf(&cmd->args, "--window-memory=%s", args->window_memory); - if (args->depth) - strvec_pushf(&cmd->args, "--depth=%s", args->depth); - if (args->threads) - strvec_pushf(&cmd->args, "--threads=%s", args->threads); - if (args->max_pack_size) - strvec_pushf(&cmd->args, "--max-pack-size=%lu", args->max_pack_size); - if (args->no_reuse_delta) - strvec_pushf(&cmd->args, "--no-reuse-delta"); - if (args->no_reuse_object) - strvec_pushf(&cmd->args, "--no-reuse-object"); - if (args->name_hash_version) - strvec_pushf(&cmd->args, "--name-hash-version=%d", args->name_hash_version); - if (args->path_walk) - strvec_pushf(&cmd->args, "--path-walk"); - if (args->local) - strvec_push(&cmd->args, "--local"); - if (args->quiet) - strvec_push(&cmd->args, "--quiet"); - if (delta_base_offset) - strvec_push(&cmd->args, "--delta-base-offset"); - strvec_push(&cmd->args, out); - cmd->git_cmd = 1; - cmd->out = -1; -} - -/* - * Write oid to the given struct child_process's stdin, starting it first if - * necessary. - */ -static int write_oid(const struct object_id *oid, - struct packed_git *pack UNUSED, - uint32_t pos UNUSED, void *data) -{ - struct child_process *cmd = data; - - if (cmd->in == -1) { - if (start_command(cmd)) - die(_("could not start pack-objects to repack promisor objects")); - } - - if (write_in_full(cmd->in, oid_to_hex(oid), the_hash_algo->hexsz) < 0 || - write_in_full(cmd->in, "\n", 1) < 0) - die(_("failed to feed promisor objects to pack-objects")); - return 0; -} - -static struct { - const char *name; - unsigned optional:1; -} exts[] = { - {".pack"}, - {".rev", 1}, - {".mtimes", 1}, - {".bitmap", 1}, - {".promisor", 1}, - {".idx"}, -}; - -struct generated_pack_data { - struct tempfile *tempfiles[ARRAY_SIZE(exts)]; -}; - -static struct generated_pack_data *populate_pack_exts(const char *name) -{ - struct stat statbuf; - struct strbuf path = STRBUF_INIT; - struct generated_pack_data *data = xcalloc(1, sizeof(*data)); - int i; - - for (i = 0; i < ARRAY_SIZE(exts); i++) { - strbuf_reset(&path); - strbuf_addf(&path, "%s-%s%s", packtmp, name, exts[i].name); - - if (stat(path.buf, &statbuf)) - continue; - - data->tempfiles[i] = register_tempfile(path.buf); - } - - strbuf_release(&path); - return data; -} - -static int has_pack_ext(const struct generated_pack_data *data, - const char *ext) -{ - int i; - for (i = 0; i < ARRAY_SIZE(exts); i++) { - if (strcmp(exts[i].name, ext)) - continue; - return !!data->tempfiles[i]; - } - BUG("unknown pack extension: '%s'", ext); -} - -static void repack_promisor_objects(const struct pack_objects_args *args, - struct string_list *names) -{ - struct child_process cmd = CHILD_PROCESS_INIT; - FILE *out; - struct strbuf line = STRBUF_INIT; - - prepare_pack_objects(&cmd, args, packtmp); - cmd.in = -1; - - /* - * NEEDSWORK: Giving pack-objects only the OIDs without any ordering - * hints may result in suboptimal deltas in the resulting pack. See if - * the OIDs can be sent with fake paths such that pack-objects can use a - * {type -> existing pack order} ordering when computing deltas instead - * of a {type -> size} ordering, which may produce better deltas. - */ - for_each_packed_object(the_repository, write_oid, &cmd, - FOR_EACH_OBJECT_PROMISOR_ONLY); - - if (cmd.in == -1) { - /* No packed objects; cmd was never started */ - child_process_clear(&cmd); - return; - } - - close(cmd.in); - - out = xfdopen(cmd.out, "r"); - while (strbuf_getline_lf(&line, out) != EOF) { - struct string_list_item *item; - char *promisor_name; - - if (line.len != the_hash_algo->hexsz) - die(_("repack: Expecting full hex object ID lines only from pack-objects.")); - item = string_list_append(names, line.buf); - - /* - * pack-objects creates the .pack and .idx files, but not the - * .promisor file. Create the .promisor file, which is empty. - * - * NEEDSWORK: fetch-pack sometimes generates non-empty - * .promisor files containing the ref names and associated - * hashes at the point of generation of the corresponding - * packfile, but this would not preserve their contents. Maybe - * concatenate the contents of all .promisor files instead of - * just creating a new empty file. - */ - promisor_name = mkpathdup("%s-%s.promisor", packtmp, - line.buf); - write_promisor_file(promisor_name, NULL, 0); - - item->util = populate_pack_exts(item->string); - - free(promisor_name); - } - - fclose(out); - if (finish_command(&cmd)) - die(_("could not finish pack-objects to repack promisor objects")); - strbuf_release(&line); -} - -struct pack_geometry { - struct packed_git **pack; - uint32_t pack_nr, pack_alloc; - uint32_t split; - - int split_factor; -}; - -static uint32_t geometry_pack_weight(struct packed_git *p) -{ - if (open_pack_index(p)) - die(_("cannot open index for %s"), p->pack_name); - return p->num_objects; -} - -static int geometry_cmp(const void *va, const void *vb) -{ - uint32_t aw = geometry_pack_weight(*(struct packed_git **)va), - bw = geometry_pack_weight(*(struct packed_git **)vb); - - if (aw < bw) - return -1; - if (aw > bw) - return 1; - return 0; -} - -static void init_pack_geometry(struct pack_geometry *geometry, - struct existing_packs *existing, - const struct pack_objects_args *args) -{ - struct packfile_store *packs = the_repository->objects->packfiles; - struct packed_git *p; - struct strbuf buf = STRBUF_INIT; - - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { - if (args->local && !p->pack_local) - /* - * When asked to only repack local packfiles we skip - * over any packfiles that are borrowed from alternate - * object directories. - */ - continue; - - if (!pack_kept_objects) { - /* - * Any pack that has its pack_keep bit set will - * appear in existing->kept_packs below, but - * this saves us from doing a more expensive - * check. - */ - if (p->pack_keep) - continue; - - /* - * The pack may be kept via the --keep-pack - * option; check 'existing->kept_packs' to - * determine whether to ignore it. - */ - strbuf_reset(&buf); - strbuf_addstr(&buf, pack_basename(p)); - strbuf_strip_suffix(&buf, ".pack"); - - if (string_list_has_string(&existing->kept_packs, buf.buf)) - continue; - } - if (p->is_cruft) - continue; - - ALLOC_GROW(geometry->pack, - geometry->pack_nr + 1, - geometry->pack_alloc); - - geometry->pack[geometry->pack_nr] = p; - geometry->pack_nr++; - } - - QSORT(geometry->pack, geometry->pack_nr, geometry_cmp); - strbuf_release(&buf); -} - -static void split_pack_geometry(struct pack_geometry *geometry) -{ - uint32_t i; - uint32_t split; - off_t total_size = 0; - - if (!geometry->pack_nr) { - geometry->split = geometry->pack_nr; - return; - } - - /* - * First, count the number of packs (in descending order of size) which - * already form a geometric progression. - */ - for (i = geometry->pack_nr - 1; i > 0; i--) { - struct packed_git *ours = geometry->pack[i]; - struct packed_git *prev = geometry->pack[i - 1]; - - if (unsigned_mult_overflows(geometry->split_factor, - geometry_pack_weight(prev))) - die(_("pack %s too large to consider in geometric " - "progression"), - prev->pack_name); - - if (geometry_pack_weight(ours) < - geometry->split_factor * geometry_pack_weight(prev)) - break; - } - - split = i; - - if (split) { - /* - * Move the split one to the right, since the top element in the - * last-compared pair can't be in the progression. Only do this - * when we split in the middle of the array (otherwise if we got - * to the end, then the split is in the right place). - */ - split++; - } - - /* - * Then, anything to the left of 'split' must be in a new pack. But, - * creating that new pack may cause packs in the heavy half to no longer - * form a geometric progression. - * - * Compute an expected size of the new pack, and then determine how many - * packs in the heavy half need to be joined into it (if any) to restore - * the geometric progression. - */ - for (i = 0; i < split; i++) { - struct packed_git *p = geometry->pack[i]; - - if (unsigned_add_overflows(total_size, geometry_pack_weight(p))) - die(_("pack %s too large to roll up"), p->pack_name); - total_size += geometry_pack_weight(p); - } - for (i = split; i < geometry->pack_nr; i++) { - struct packed_git *ours = geometry->pack[i]; - - if (unsigned_mult_overflows(geometry->split_factor, - total_size)) - die(_("pack %s too large to roll up"), ours->pack_name); - - if (geometry_pack_weight(ours) < - geometry->split_factor * total_size) { - if (unsigned_add_overflows(total_size, - geometry_pack_weight(ours))) - die(_("pack %s too large to roll up"), - ours->pack_name); - - split++; - total_size += geometry_pack_weight(ours); - } else - break; - } - - geometry->split = split; -} - -static struct packed_git *get_preferred_pack(struct pack_geometry *geometry) -{ - uint32_t i; - - if (!geometry) { - /* - * No geometry means either an all-into-one repack (in which - * case there is only one pack left and it is the largest) or an - * incremental one. - * - * If repacking incrementally, then we could check the size of - * all packs to determine which should be preferred, but leave - * this for later. - */ - return NULL; - } - if (geometry->split == geometry->pack_nr) - return NULL; - - /* - * The preferred pack is the largest pack above the split line. In - * other words, it is the largest pack that does not get rolled up in - * the geometric repack. - */ - for (i = geometry->pack_nr; i > geometry->split; i--) - /* - * A pack that is not local would never be included in a - * multi-pack index. We thus skip over any non-local packs. - */ - if (geometry->pack[i - 1]->pack_local) - return geometry->pack[i - 1]; - - return NULL; -} - -static void geometry_remove_redundant_packs(struct pack_geometry *geometry, - struct string_list *names, - struct existing_packs *existing) -{ - struct strbuf buf = STRBUF_INIT; - uint32_t i; - - for (i = 0; i < geometry->split; i++) { - struct packed_git *p = geometry->pack[i]; - if (string_list_has_string(names, hash_to_hex(p->hash))) - continue; - - strbuf_reset(&buf); - strbuf_addstr(&buf, pack_basename(p)); - strbuf_strip_suffix(&buf, ".pack"); - - if ((p->pack_keep) || - (string_list_has_string(&existing->kept_packs, buf.buf))) - continue; - - remove_redundant_pack(packdir, buf.buf); - } - - strbuf_release(&buf); -} - -static void free_pack_geometry(struct pack_geometry *geometry) -{ - if (!geometry) - return; - - free(geometry->pack); -} - -static int midx_has_unknown_packs(char **midx_pack_names, - size_t midx_pack_names_nr, - struct string_list *include, - struct pack_geometry *geometry, - struct existing_packs *existing) -{ - size_t i; - - string_list_sort(include); - - for (i = 0; i < midx_pack_names_nr; i++) { - const char *pack_name = midx_pack_names[i]; - - /* - * Determine whether or not each MIDX'd pack from the existing - * MIDX (if any) is represented in the new MIDX. For each pack - * in the MIDX, it must either be: - * - * - In the "include" list of packs to be included in the new - * MIDX. Note this function is called before the include - * list is populated with any cruft pack(s). - * - * - Below the geometric split line (if using pack geometry), - * indicating that the pack won't be included in the new - * MIDX, but its contents were rolled up as part of the - * geometric repack. - * - * - In the existing non-kept packs list (if not using pack - * geometry), and marked as non-deleted. - */ - if (string_list_has_string(include, pack_name)) { - continue; - } else if (geometry) { - struct strbuf buf = STRBUF_INIT; - uint32_t j; - - for (j = 0; j < geometry->split; j++) { - strbuf_reset(&buf); - strbuf_addstr(&buf, pack_basename(geometry->pack[j])); - strbuf_strip_suffix(&buf, ".pack"); - strbuf_addstr(&buf, ".idx"); - - if (!strcmp(pack_name, buf.buf)) { - strbuf_release(&buf); - break; - } - } - - strbuf_release(&buf); - - if (j < geometry->split) - continue; - } else { - struct string_list_item *item; - - item = string_list_lookup(&existing->non_kept_packs, - pack_name); - if (item && !pack_is_marked_for_deletion(item)) - continue; - } - - /* - * If we got to this point, the MIDX includes some pack that we - * don't know about. - */ - return 1; - } - - return 0; -} - -struct midx_snapshot_ref_data { - struct tempfile *f; - struct oidset seen; - int preferred; -}; - -static int midx_snapshot_ref_one(const char *refname UNUSED, - const char *referent UNUSED, - const struct object_id *oid, - int flag UNUSED, void *_data) -{ - struct midx_snapshot_ref_data *data = _data; - struct object_id peeled; - - if (!peel_iterated_oid(the_repository, oid, &peeled)) - oid = &peeled; - - if (oidset_insert(&data->seen, oid)) - return 0; /* already seen */ - - if (odb_read_object_info(the_repository->objects, oid, NULL) != OBJ_COMMIT) - return 0; - - fprintf(data->f->fp, "%s%s\n", data->preferred ? "+" : "", - oid_to_hex(oid)); - - return 0; -} - -static void midx_snapshot_refs(struct tempfile *f) -{ - struct midx_snapshot_ref_data data; - const struct string_list *preferred = bitmap_preferred_tips(the_repository); - - data.f = f; - data.preferred = 0; - oidset_init(&data.seen, 0); - - if (!fdopen_tempfile(f, "w")) - die(_("could not open tempfile %s for writing"), - get_tempfile_path(f)); - - if (preferred) { - struct string_list_item *item; - - data.preferred = 1; - for_each_string_list_item(item, preferred) - refs_for_each_ref_in(get_main_ref_store(the_repository), - item->string, - midx_snapshot_ref_one, &data); - data.preferred = 0; - } - - refs_for_each_ref(get_main_ref_store(the_repository), - midx_snapshot_ref_one, &data); - - if (close_tempfile_gently(f)) { - int save_errno = errno; - delete_tempfile(&f); - errno = save_errno; - die_errno(_("could not close refs snapshot tempfile")); - } - - oidset_clear(&data.seen); -} - -static void midx_included_packs(struct string_list *include, - struct existing_packs *existing, - char **midx_pack_names, - size_t midx_pack_names_nr, - struct string_list *names, - struct pack_geometry *geometry) -{ - struct string_list_item *item; - struct strbuf buf = STRBUF_INIT; - - for_each_string_list_item(item, &existing->kept_packs) { - strbuf_reset(&buf); - strbuf_addf(&buf, "%s.idx", item->string); - string_list_insert(include, buf.buf); - } - - for_each_string_list_item(item, names) { - strbuf_reset(&buf); - strbuf_addf(&buf, "pack-%s.idx", item->string); - string_list_insert(include, buf.buf); - } - - if (geometry->split_factor) { - uint32_t i; - - for (i = geometry->split; i < geometry->pack_nr; i++) { - struct packed_git *p = geometry->pack[i]; - - /* - * The multi-pack index never refers to packfiles part - * of an alternate object database, so we skip these. - * While git-multi-pack-index(1) would silently ignore - * them anyway, this allows us to skip executing the - * command completely when we have only non-local - * packfiles. - */ - if (!p->pack_local) - continue; - - strbuf_reset(&buf); - strbuf_addstr(&buf, pack_basename(p)); - strbuf_strip_suffix(&buf, ".pack"); - strbuf_addstr(&buf, ".idx"); - - string_list_insert(include, buf.buf); - } - } else { - for_each_string_list_item(item, &existing->non_kept_packs) { - if (pack_is_marked_for_deletion(item)) - continue; - - strbuf_reset(&buf); - strbuf_addf(&buf, "%s.idx", item->string); - string_list_insert(include, buf.buf); - } - } - - if (midx_must_contain_cruft || - midx_has_unknown_packs(midx_pack_names, midx_pack_names_nr, - include, geometry, existing)) { - /* - * If there are one or more unknown pack(s) present (see - * midx_has_unknown_packs() for what makes a pack - * "unknown") in the MIDX before the repack, keep them - * as they may be required to form a reachability - * closure if the MIDX is bitmapped. - * - * For example, a cruft pack can be required to form a - * reachability closure if the MIDX is bitmapped and one - * or more of the bitmap's selected commits reaches a - * once-cruft object that was later made reachable. - */ - for_each_string_list_item(item, &existing->cruft_packs) { - /* - * When doing a --geometric repack, there is no - * need to check for deleted packs, since we're - * by definition not doing an ALL_INTO_ONE - * repack (hence no packs will be deleted). - * Otherwise we must check for and exclude any - * packs which are enqueued for deletion. - * - * So we could omit the conditional below in the - * --geometric case, but doing so is unnecessary - * since no packs are marked as pending - * deletion (since we only call - * `mark_packs_for_deletion()` when doing an - * all-into-one repack). - */ - if (pack_is_marked_for_deletion(item)) - continue; - - strbuf_reset(&buf); - strbuf_addf(&buf, "%s.idx", item->string); - string_list_insert(include, buf.buf); - } - } else { - /* - * Modern versions of Git (with the appropriate - * configuration setting) will write new copies of - * once-cruft objects when doing a --geometric repack. - * - * If the MIDX has no cruft pack, new packs written - * during a --geometric repack will not rely on the - * cruft pack to form a reachability closure, so we can - * avoid including them in the MIDX in that case. - */ - ; - } - - strbuf_release(&buf); -} - -static int write_midx_included_packs(struct string_list *include, - struct pack_geometry *geometry, - struct string_list *names, - const char *refs_snapshot, - int show_progress, int write_bitmaps) -{ - struct child_process cmd = CHILD_PROCESS_INIT; - struct string_list_item *item; - struct packed_git *preferred = get_preferred_pack(geometry); - FILE *in; - int ret; - - if (!include->nr) - return 0; - - cmd.in = -1; - cmd.git_cmd = 1; - - strvec_push(&cmd.args, "multi-pack-index"); - strvec_pushl(&cmd.args, "write", "--stdin-packs", NULL); - - if (show_progress) - strvec_push(&cmd.args, "--progress"); - else - strvec_push(&cmd.args, "--no-progress"); - - if (write_bitmaps) - strvec_push(&cmd.args, "--bitmap"); - - if (preferred) - strvec_pushf(&cmd.args, "--preferred-pack=%s", - pack_basename(preferred)); - else if (names->nr) { - /* The largest pack was repacked, meaning that either - * one or two packs exist depending on whether the - * repository has a cruft pack or not. - * - * Select the non-cruft one as preferred to encourage - * pack-reuse among packs containing reachable objects - * over unreachable ones. - * - * (Note we could write multiple packs here if - * `--max-pack-size` was given, but any one of them - * will suffice, so pick the first one.) - */ - for_each_string_list_item(item, names) { - struct generated_pack_data *data = item->util; - if (has_pack_ext(data, ".mtimes")) - continue; - - strvec_pushf(&cmd.args, "--preferred-pack=pack-%s.pack", - item->string); - break; - } - } else { - /* - * No packs were kept, and no packs were written. The - * only thing remaining are .keep packs (unless - * --pack-kept-objects was given). - * - * Set the `--preferred-pack` arbitrarily here. - */ - ; - } - - if (refs_snapshot) - strvec_pushf(&cmd.args, "--refs-snapshot=%s", refs_snapshot); - - ret = start_command(&cmd); - if (ret) - return ret; - - in = xfdopen(cmd.in, "w"); - for_each_string_list_item(item, include) - fprintf(in, "%s\n", item->string); - fclose(in); - - return finish_command(&cmd); -} - -static void remove_redundant_bitmaps(struct string_list *include, - const char *packdir) -{ - struct strbuf path = STRBUF_INIT; - struct string_list_item *item; - size_t packdir_len; - - strbuf_addstr(&path, packdir); - strbuf_addch(&path, '/'); - packdir_len = path.len; - - /* - * Remove any pack bitmaps corresponding to packs which are now - * included in the MIDX. - */ - for_each_string_list_item(item, include) { - strbuf_addstr(&path, item->string); - strbuf_strip_suffix(&path, ".idx"); - strbuf_addstr(&path, ".bitmap"); - - if (unlink(path.buf) && errno != ENOENT) - warning_errno(_("could not remove stale bitmap: %s"), - path.buf); - - strbuf_setlen(&path, packdir_len); - } - strbuf_release(&path); -} - -static int finish_pack_objects_cmd(struct child_process *cmd, - struct string_list *names, - int local) -{ - FILE *out; - struct strbuf line = STRBUF_INIT; - - out = xfdopen(cmd->out, "r"); - while (strbuf_getline_lf(&line, out) != EOF) { - struct string_list_item *item; - - if (line.len != the_hash_algo->hexsz) - die(_("repack: Expecting full hex object ID lines only " - "from pack-objects.")); - /* - * Avoid putting packs written outside of the repository in the - * list of names. - */ - if (local) { - item = string_list_append(names, line.buf); - item->util = populate_pack_exts(line.buf); - } - } - fclose(out); - - strbuf_release(&line); - - return finish_command(cmd); -} - -static int write_filtered_pack(const struct pack_objects_args *args, - const char *destination, - const char *pack_prefix, - struct existing_packs *existing, - struct string_list *names) -{ - struct child_process cmd = CHILD_PROCESS_INIT; - struct string_list_item *item; - FILE *in; - int ret; - const char *caret; - const char *scratch; - int local = skip_prefix(destination, packdir, &scratch); - - prepare_pack_objects(&cmd, args, destination); - - strvec_push(&cmd.args, "--stdin-packs"); - - if (!pack_kept_objects) - strvec_push(&cmd.args, "--honor-pack-keep"); - for_each_string_list_item(item, &existing->kept_packs) - strvec_pushf(&cmd.args, "--keep-pack=%s", item->string); - - cmd.in = -1; - - ret = start_command(&cmd); - if (ret) - return ret; - - /* - * Here 'names' contains only the pack(s) that were just - * written, which is exactly the packs we want to keep. Also - * 'existing_kept_packs' already contains the packs in - * 'keep_pack_list'. - */ - in = xfdopen(cmd.in, "w"); - for_each_string_list_item(item, names) - fprintf(in, "^%s-%s.pack\n", pack_prefix, item->string); - for_each_string_list_item(item, &existing->non_kept_packs) - fprintf(in, "%s.pack\n", item->string); - for_each_string_list_item(item, &existing->cruft_packs) - fprintf(in, "%s.pack\n", item->string); - caret = pack_kept_objects ? "" : "^"; - for_each_string_list_item(item, &existing->kept_packs) - fprintf(in, "%s%s.pack\n", caret, item->string); - fclose(in); - - return finish_pack_objects_cmd(&cmd, names, local); -} - -static void combine_small_cruft_packs(FILE *in, size_t combine_cruft_below_size, - struct existing_packs *existing) -{ - struct packfile_store *packs = the_repository->objects->packfiles; - struct packed_git *p; - struct strbuf buf = STRBUF_INIT; - size_t i; - - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { - if (!(p->is_cruft && p->pack_local)) - continue; - - strbuf_reset(&buf); - strbuf_addstr(&buf, pack_basename(p)); - strbuf_strip_suffix(&buf, ".pack"); - - if (!string_list_has_string(&existing->cruft_packs, buf.buf)) - continue; - - if (p->pack_size < combine_cruft_below_size) { - fprintf(in, "-%s\n", pack_basename(p)); - } else { - retain_cruft_pack(existing, p); - fprintf(in, "%s\n", pack_basename(p)); - } - } - - for (i = 0; i < existing->non_kept_packs.nr; i++) - fprintf(in, "-%s.pack\n", - existing->non_kept_packs.items[i].string); - - strbuf_release(&buf); -} - -static int write_cruft_pack(const struct pack_objects_args *args, - const char *destination, - const char *pack_prefix, - const char *cruft_expiration, - unsigned long combine_cruft_below_size, - struct string_list *names, - struct existing_packs *existing) -{ - struct child_process cmd = CHILD_PROCESS_INIT; - struct string_list_item *item; - FILE *in; - int ret; - const char *scratch; - int local = skip_prefix(destination, packdir, &scratch); - - prepare_pack_objects(&cmd, args, destination); - - strvec_push(&cmd.args, "--cruft"); - if (cruft_expiration) - strvec_pushf(&cmd.args, "--cruft-expiration=%s", - cruft_expiration); - - strvec_push(&cmd.args, "--honor-pack-keep"); - strvec_push(&cmd.args, "--non-empty"); - - cmd.in = -1; - - ret = start_command(&cmd); - if (ret) - return ret; - - /* - * names has a confusing double use: it both provides the list - * of just-written new packs, and accepts the name of the cruft - * pack we are writing. - * - * By the time it is read here, it contains only the pack(s) - * that were just written, which is exactly the set of packs we - * want to consider kept. - * - * If `--expire-to` is given, the double-use served by `names` - * ensures that the pack written to `--expire-to` excludes any - * objects contained in the cruft pack. - */ - in = xfdopen(cmd.in, "w"); - for_each_string_list_item(item, names) - fprintf(in, "%s-%s.pack\n", pack_prefix, item->string); - if (combine_cruft_below_size && !cruft_expiration) { - combine_small_cruft_packs(in, combine_cruft_below_size, - existing); - } else { - for_each_string_list_item(item, &existing->non_kept_packs) - fprintf(in, "-%s.pack\n", item->string); - for_each_string_list_item(item, &existing->cruft_packs) - fprintf(in, "-%s.pack\n", item->string); - } - for_each_string_list_item(item, &existing->kept_packs) - fprintf(in, "%s.pack\n", item->string); - fclose(in); - - return finish_pack_objects_cmd(&cmd, names, local); -} - -static const char *find_pack_prefix(const char *packdir, const char *packtmp) -{ - const char *pack_prefix; - if (!skip_prefix(packtmp, packdir, &pack_prefix)) - die(_("pack prefix %s does not begin with objdir %s"), - packtmp, packdir); - if (*pack_prefix == '/') - pack_prefix++; - return pack_prefix; -} - int cmd_repack(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { struct child_process cmd = CHILD_PROCESS_INIT; struct string_list_item *item; @@ -1255,18 +108,17 @@ int cmd_repack(int argc, struct existing_packs existing = EXISTING_PACKS_INIT; struct pack_geometry geometry = { 0 }; struct tempfile *refs_snapshot = NULL; - int i, ext, ret; + int i, ret; int show_progress; - char **midx_pack_names = NULL; - size_t midx_pack_names_nr = 0; /* variables to be filled by option parsing */ + struct repack_config_ctx config_ctx; int delete_redundant = 0; const char *unpack_unreachable = NULL; int keep_unreachable = 0; struct string_list keep_pack_list = STRING_LIST_INIT_NODUP; - struct pack_objects_args po_args = { 0 }; - struct pack_objects_args cruft_po_args = { 0 }; + struct pack_objects_args po_args = PACK_OBJECTS_ARGS_INIT; + struct pack_objects_args cruft_po_args = PACK_OBJECTS_ARGS_INIT; int write_midx = 0; const char *cruft_expiration = NULL; const char *expire_to = NULL; @@ -1327,7 +179,7 @@ int cmd_repack(int argc, OPT_UNSIGNED(0, "max-pack-size", &po_args.max_pack_size, N_("maximum size of each packfile")), OPT_PARSE_LIST_OBJECTS_FILTER(&po_args.filter_options), - OPT_BOOL(0, "pack-kept-objects", &pack_kept_objects, + OPT_BOOL(0, "pack-kept-objects", &po_args.pack_kept_objects, N_("repack objects in packs marked with .keep")), OPT_STRING_LIST(0, "keep-pack", &keep_pack_list, N_("name"), N_("do not repack this pack")), @@ -1344,7 +196,11 @@ int cmd_repack(int argc, list_objects_filter_init(&po_args.filter_options); - repo_config(the_repository, repack_config, &cruft_po_args); + memset(&config_ctx, 0, sizeof(config_ctx)); + config_ctx.po_args = &po_args; + config_ctx.cruft_po_args = &cruft_po_args; + + repo_config(repo, repack_config, &config_ctx); argc = parse_options(argc, argv, prefix, builtin_repack_options, git_repack_usage, 0); @@ -1354,7 +210,7 @@ int cmd_repack(int argc, po_args.depth = xstrdup_or_null(opt_depth); po_args.threads = xstrdup_or_null(opt_threads); - if (delete_redundant && the_repository->repository_format_precious_objects) + if (delete_redundant && repo->repository_format_precious_objects) die(_("cannot delete packs in a precious-objects repo")); die_for_incompatible_opt3(unpack_unreachable || (pack_everything & LOOSEN_UNREACHABLE), "-A", @@ -1369,14 +225,14 @@ int cmd_repack(int argc, (!(pack_everything & ALL_INTO_ONE) || !is_bare_repository())) write_bitmaps = 0; } - if (pack_kept_objects < 0) - pack_kept_objects = write_bitmaps > 0 && !write_midx; + if (po_args.pack_kept_objects < 0) + po_args.pack_kept_objects = write_bitmaps > 0 && !write_midx; if (write_bitmaps && !(pack_everything & ALL_INTO_ONE) && !write_midx) die(_(incremental_bitmap_conflict_error)); if (write_bitmaps && po_args.local && - odb_has_alternates(the_repository->objects)) { + odb_has_alternates(repo->objects)) { /* * When asked to do a local repack, but we have * packfiles that are inherited from an alternate, then @@ -1391,26 +247,28 @@ int cmd_repack(int argc, if (write_midx && write_bitmaps) { struct strbuf path = STRBUF_INIT; - strbuf_addf(&path, "%s/%s_XXXXXX", repo_get_object_directory(the_repository), + strbuf_addf(&path, "%s/%s_XXXXXX", + repo_get_object_directory(repo), "bitmap-ref-tips"); refs_snapshot = xmks_tempfile(path.buf); - midx_snapshot_refs(refs_snapshot); + midx_snapshot_refs(repo, refs_snapshot); strbuf_release(&path); } - packdir = mkpathdup("%s/pack", repo_get_object_directory(the_repository)); + packdir = mkpathdup("%s/pack", repo_get_object_directory(repo)); packtmp_name = xstrfmt(".tmp-%d-pack", (int)getpid()); packtmp = mkpathdup("%s/%s", packdir, packtmp_name); - collect_pack_filenames(&existing, &keep_pack_list); + existing.repo = repo; + existing_packs_collect(&existing, &keep_pack_list); if (geometry.split_factor) { if (pack_everything) die(_("options '%s' and '%s' cannot be used together"), "--geometric", "-A/-a"); - init_pack_geometry(&geometry, &existing, &po_args); - split_pack_geometry(&geometry); + pack_geometry_init(&geometry, &existing, &po_args); + pack_geometry_split(&geometry); } prepare_pack_objects(&cmd, &po_args, packtmp); @@ -1418,8 +276,6 @@ int cmd_repack(int argc, show_progress = !po_args.quiet && isatty(2); strvec_push(&cmd.args, "--keep-true-parents"); - if (!pack_kept_objects) - strvec_push(&cmd.args, "--honor-pack-keep"); for (i = 0; i < keep_pack_list.nr; i++) strvec_pushf(&cmd.args, "--keep-pack=%s", keep_pack_list.items[i].string); @@ -1439,7 +295,7 @@ int cmd_repack(int argc, strvec_push(&cmd.args, "--reflog"); strvec_push(&cmd.args, "--indexed-objects"); } - if (repo_has_promisor_remote(the_repository)) + if (repo_has_promisor_remote(repo)) strvec_push(&cmd.args, "--exclude-promisor-objects"); if (!write_midx) { if (write_bitmaps > 0) @@ -1451,9 +307,9 @@ int cmd_repack(int argc, strvec_push(&cmd.args, "--delta-islands"); if (pack_everything & ALL_INTO_ONE) { - repack_promisor_objects(&po_args, &names); + repack_promisor_objects(repo, &po_args, &names, packtmp); - if (has_existing_non_kept_packs(&existing) && + if (existing_packs_has_non_kept(&existing) && delete_redundant && !(pack_everything & PACK_CRUFT)) { for_each_string_list_item(item, &names) { @@ -1515,9 +371,17 @@ int cmd_repack(int argc, fclose(in); } - ret = finish_pack_objects_cmd(&cmd, &names, 1); - if (ret) - goto cleanup; + { + struct write_pack_opts opts = { + .packdir = packdir, + .destination = packdir, + .packtmp = packtmp, + }; + ret = finish_pack_objects_cmd(repo->hash_algo, &opts, &cmd, + &names); + if (ret) + goto cleanup; + } if (!names.nr) { if (!po_args.quiet) @@ -1535,12 +399,17 @@ int cmd_repack(int argc, * midx_has_unknown_packs() will make the decision for * us. */ - if (!get_multi_pack_index(the_repository->objects->sources)) + if (!get_multi_pack_index(repo->objects->sources)) midx_must_contain_cruft = 1; } if (pack_everything & PACK_CRUFT) { - const char *pack_prefix = find_pack_prefix(packdir, packtmp); + struct write_pack_opts opts = { + .po_args = &cruft_po_args, + .destination = packtmp, + .packtmp = packtmp, + .packdir = packdir, + }; if (!cruft_po_args.window) cruft_po_args.window = xstrdup_or_null(po_args.window); @@ -1555,9 +424,10 @@ int cmd_repack(int argc, cruft_po_args.local = po_args.local; cruft_po_args.quiet = po_args.quiet; + cruft_po_args.delta_base_offset = po_args.delta_base_offset; + cruft_po_args.pack_kept_objects = 0; - ret = write_cruft_pack(&cruft_po_args, packtmp, pack_prefix, - cruft_expiration, + ret = write_cruft_pack(&opts, cruft_expiration, combine_cruft_below_size, &names, &existing); if (ret) @@ -1592,11 +462,8 @@ int cmd_repack(int argc, * pack, but rather removing all cruft packs from the * main repository regardless of size. */ - ret = write_cruft_pack(&cruft_po_args, expire_to, - pack_prefix, - NULL, - 0ul, - &names, + opts.destination = expire_to; + ret = write_cruft_pack(&opts, NULL, 0ul, &names, &existing); if (ret) goto cleanup; @@ -1604,99 +471,63 @@ int cmd_repack(int argc, } if (po_args.filter_options.choice) { - if (!filter_to) - filter_to = packtmp; - - ret = write_filtered_pack(&po_args, - filter_to, - find_pack_prefix(packdir, packtmp), - &existing, - &names); + struct write_pack_opts opts = { + .po_args = &po_args, + .destination = filter_to, + .packdir = packdir, + .packtmp = packtmp, + }; + + if (!opts.destination) + opts.destination = packtmp; + + ret = write_filtered_pack(&opts, &existing, &names); if (ret) goto cleanup; } string_list_sort(&names); - if (get_multi_pack_index(the_repository->objects->sources)) { - struct multi_pack_index *m = - get_multi_pack_index(the_repository->objects->sources); - - ALLOC_ARRAY(midx_pack_names, - m->num_packs + m->num_packs_in_base); - - for (; m; m = m->base_midx) - for (uint32_t i = 0; i < m->num_packs; i++) - midx_pack_names[midx_pack_names_nr++] = - xstrdup(m->pack_names[i]); - } - - close_object_store(the_repository->objects); + close_object_store(repo->objects); /* * Ok we have prepared all new packfiles. */ - for_each_string_list_item(item, &names) { - struct generated_pack_data *data = item->util; - - for (ext = 0; ext < ARRAY_SIZE(exts); ext++) { - char *fname; - - fname = mkpathdup("%s/pack-%s%s", - packdir, item->string, exts[ext].name); - - if (data->tempfiles[ext]) { - const char *fname_old = get_tempfile_path(data->tempfiles[ext]); - struct stat statbuffer; - - if (!stat(fname_old, &statbuffer)) { - statbuffer.st_mode &= ~(S_IWUSR | S_IWGRP | S_IWOTH); - chmod(fname_old, statbuffer.st_mode); - } - - if (rename_tempfile(&data->tempfiles[ext], fname)) - die_errno(_("renaming pack to '%s' failed"), fname); - } else if (!exts[ext].optional) - die(_("pack-objects did not write a '%s' file for pack %s-%s"), - exts[ext].name, packtmp, item->string); - else if (unlink(fname) < 0 && errno != ENOENT) - die_errno(_("could not unlink: %s"), fname); - - free(fname); - } - } + for_each_string_list_item(item, &names) + generated_pack_install(item->util, item->string, packdir, + packtmp); /* End of pack replacement. */ if (delete_redundant && pack_everything & ALL_INTO_ONE) - mark_packs_for_deletion(&existing, &names); + existing_packs_mark_for_deletion(&existing, &names); if (write_midx) { - struct string_list include = STRING_LIST_INIT_DUP; - midx_included_packs(&include, &existing, midx_pack_names, - midx_pack_names_nr, &names, &geometry); - - ret = write_midx_included_packs(&include, &geometry, &names, - refs_snapshot ? get_tempfile_path(refs_snapshot) : NULL, - show_progress, write_bitmaps > 0); - - if (!ret && write_bitmaps) - remove_redundant_bitmaps(&include, packdir); - - string_list_clear(&include, 0); + struct repack_write_midx_opts opts = { + .existing = &existing, + .geometry = &geometry, + .names = &names, + .refs_snapshot = refs_snapshot ? get_tempfile_path(refs_snapshot) : NULL, + .packdir = packdir, + .show_progress = show_progress, + .write_bitmaps = write_bitmaps > 0, + .midx_must_contain_cruft = midx_must_contain_cruft + }; + + ret = write_midx_included_packs(&opts); if (ret) goto cleanup; } - odb_reprepare(the_repository->objects); + odb_reprepare(repo->objects); if (delete_redundant) { int opts = 0; - remove_redundant_existing_packs(&existing); + existing_packs_remove_redundant(&existing, packdir); if (geometry.split_factor) - geometry_remove_redundant_packs(&geometry, &names, - &existing); + pack_geometry_remove_redundant(&geometry, &names, + &existing, packdir); if (show_progress) opts |= PRUNE_PACKED_VERBOSE; prune_packed_objects(opts); @@ -1704,18 +535,18 @@ int cmd_repack(int argc, if (!keep_unreachable && (!(pack_everything & LOOSEN_UNREACHABLE) || unpack_unreachable) && - is_repository_shallow(the_repository)) + is_repository_shallow(repo)) prune_shallow(PRUNE_QUICK); } if (run_update_server_info) - update_server_info(the_repository, 0); + update_server_info(repo, 0); if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX, 0)) { unsigned flags = 0; if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL, 0)) flags |= MIDX_WRITE_INCREMENTAL; - write_midx_file(the_repository->objects->sources, + write_midx_file(repo->objects->sources, NULL, NULL, flags); } @@ -1723,10 +554,7 @@ cleanup: string_list_clear(&keep_pack_list, 0); string_list_clear(&names, 1); existing_packs_release(&existing); - free_pack_geometry(&geometry); - for (size_t i = 0; i < midx_pack_names_nr; i++) - free(midx_pack_names[i]); - free(midx_pack_names); + pack_geometry_release(&geometry); pack_objects_args_release(&po_args); pack_objects_args_release(&cruft_po_args); diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index 8c333b3e2e..15d51e60a8 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -2,6 +2,7 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "builtin.h" +#include "abspath.h" #include "config.h" #include "dir.h" #include "environment.h" @@ -23,7 +24,7 @@ static const char *empty_base = ""; static char const * const builtin_sparse_checkout_usage[] = { - N_("git sparse-checkout (init | list | set | add | reapply | disable | check-rules) [<options>]"), + N_("git sparse-checkout (init | list | set | add | reapply | disable | check-rules | clean) [<options>]"), NULL }; @@ -204,12 +205,12 @@ static void clean_tracked_sparse_directories(struct repository *r) ensure_full_index(r->index); } -static int update_working_directory(struct pattern_list *pl) +static int update_working_directory(struct repository *r, + struct pattern_list *pl) { enum update_sparsity_result result; struct unpack_trees_options o; struct lock_file lock_file = LOCK_INIT; - struct repository *r = the_repository; struct pattern_list *old_pl; /* If no branch has been checked out, there are no updates to make. */ @@ -327,7 +328,8 @@ static void write_cone_to_file(FILE *fp, struct pattern_list *pl) string_list_clear(&sl, 0); } -static int write_patterns_and_update(struct pattern_list *pl) +static int write_patterns_and_update(struct repository *repo, + struct pattern_list *pl) { char *sparse_filename; FILE *fp; @@ -336,15 +338,15 @@ static int write_patterns_and_update(struct pattern_list *pl) sparse_filename = get_sparse_checkout_filename(); - if (safe_create_leading_directories(the_repository, sparse_filename)) + if (safe_create_leading_directories(repo, sparse_filename)) die(_("failed to create directory for sparse-checkout file")); hold_lock_file_for_update(&lk, sparse_filename, LOCK_DIE_ON_ERROR); - result = update_working_directory(pl); + result = update_working_directory(repo, pl); if (result) { rollback_lock_file(&lk); - update_working_directory(NULL); + update_working_directory(repo, NULL); goto out; } @@ -372,25 +374,26 @@ enum sparse_checkout_mode { MODE_CONE_PATTERNS = 2, }; -static int set_config(enum sparse_checkout_mode mode) +static int set_config(struct repository *repo, + enum sparse_checkout_mode mode) { /* Update to use worktree config, if not already. */ - if (init_worktree_config(the_repository)) { + if (init_worktree_config(repo)) { error(_("failed to initialize worktree config")); return 1; } - if (repo_config_set_worktree_gently(the_repository, + if (repo_config_set_worktree_gently(repo, "core.sparseCheckout", mode ? "true" : "false") || - repo_config_set_worktree_gently(the_repository, + repo_config_set_worktree_gently(repo, "core.sparseCheckoutCone", mode == MODE_CONE_PATTERNS ? "true" : "false")) return 1; if (mode == MODE_NO_PATTERNS) - return set_sparse_index_config(the_repository, 0); + return set_sparse_index_config(repo, 0); return 0; } @@ -410,7 +413,7 @@ static enum sparse_checkout_mode update_cone_mode(int *cone_mode) { return MODE_ALL_PATTERNS; } -static int update_modes(int *cone_mode, int *sparse_index) +static int update_modes(struct repository *repo, int *cone_mode, int *sparse_index) { int mode, record_mode; @@ -418,20 +421,20 @@ static int update_modes(int *cone_mode, int *sparse_index) record_mode = (*cone_mode != -1) || !core_apply_sparse_checkout; mode = update_cone_mode(cone_mode); - if (record_mode && set_config(mode)) + if (record_mode && set_config(repo, mode)) return 1; /* Set sparse-index/non-sparse-index mode if specified */ if (*sparse_index >= 0) { - if (set_sparse_index_config(the_repository, *sparse_index) < 0) + if (set_sparse_index_config(repo, *sparse_index) < 0) die(_("failed to modify sparse-index config")); /* force an index rewrite */ - repo_read_index(the_repository); - the_repository->index->updated_workdir = 1; + repo_read_index(repo); + repo->index->updated_workdir = 1; if (!*sparse_index) - ensure_full_index(the_repository->index); + ensure_full_index(repo->index); } return 0; @@ -448,7 +451,7 @@ static struct sparse_checkout_init_opts { } init_opts; static int sparse_checkout_init(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { struct pattern_list pl; char *sparse_filename; @@ -464,7 +467,7 @@ static int sparse_checkout_init(int argc, const char **argv, const char *prefix, }; setup_work_tree(); - repo_read_index(the_repository); + repo_read_index(repo); init_opts.cone_mode = -1; init_opts.sparse_index = -1; @@ -473,7 +476,7 @@ static int sparse_checkout_init(int argc, const char **argv, const char *prefix, builtin_sparse_checkout_init_options, builtin_sparse_checkout_init_usage, 0); - if (update_modes(&init_opts.cone_mode, &init_opts.sparse_index)) + if (update_modes(repo, &init_opts.cone_mode, &init_opts.sparse_index)) return 1; memset(&pl, 0, sizeof(pl)); @@ -485,14 +488,14 @@ static int sparse_checkout_init(int argc, const char **argv, const char *prefix, if (res >= 0) { free(sparse_filename); clear_pattern_list(&pl); - return update_working_directory(NULL); + return update_working_directory(repo, NULL); } - if (repo_get_oid(the_repository, "HEAD", &oid)) { + if (repo_get_oid(repo, "HEAD", &oid)) { FILE *fp; /* assume we are in a fresh repo, but update the sparse-checkout file */ - if (safe_create_leading_directories(the_repository, sparse_filename)) + if (safe_create_leading_directories(repo, sparse_filename)) die(_("unable to create leading directories of %s"), sparse_filename); fp = xfopen(sparse_filename, "w"); @@ -511,7 +514,7 @@ static int sparse_checkout_init(int argc, const char **argv, const char *prefix, add_pattern("!/*/", empty_base, 0, &pl, 0); pl.use_cone_patterns = init_opts.cone_mode; - return write_patterns_and_update(&pl); + return write_patterns_and_update(repo, &pl); } static void insert_recursive_pattern(struct pattern_list *pl, struct strbuf *path) @@ -674,7 +677,8 @@ static void add_patterns_literal(int argc, const char **argv, add_patterns_from_input(pl, argc, argv, use_stdin ? stdin : NULL); } -static int modify_pattern_list(struct strvec *args, int use_stdin, +static int modify_pattern_list(struct repository *repo, + struct strvec *args, int use_stdin, enum modify_type m) { int result; @@ -696,22 +700,23 @@ static int modify_pattern_list(struct strvec *args, int use_stdin, } if (!core_apply_sparse_checkout) { - set_config(MODE_ALL_PATTERNS); + set_config(repo, MODE_ALL_PATTERNS); core_apply_sparse_checkout = 1; changed_config = 1; } - result = write_patterns_and_update(pl); + result = write_patterns_and_update(repo, pl); if (result && changed_config) - set_config(MODE_NO_PATTERNS); + set_config(repo, MODE_NO_PATTERNS); clear_pattern_list(pl); free(pl); return result; } -static void sanitize_paths(struct strvec *args, +static void sanitize_paths(struct repository *repo, + struct strvec *args, const char *prefix, int skip_checks) { int i; @@ -752,7 +757,7 @@ static void sanitize_paths(struct strvec *args, for (i = 0; i < args->nr; i++) { struct cache_entry *ce; - struct index_state *index = the_repository->index; + struct index_state *index = repo->index; int pos = index_name_pos(index, args->v[i], strlen(args->v[i])); if (pos < 0) @@ -779,7 +784,7 @@ static struct sparse_checkout_add_opts { } add_opts; static int sparse_checkout_add(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { static struct option builtin_sparse_checkout_add_options[] = { OPT_BOOL_F(0, "skip-checks", &add_opts.skip_checks, @@ -796,7 +801,7 @@ static int sparse_checkout_add(int argc, const char **argv, const char *prefix, if (!core_apply_sparse_checkout) die(_("no sparse-checkout to add to")); - repo_read_index(the_repository); + repo_read_index(repo); argc = parse_options(argc, argv, prefix, builtin_sparse_checkout_add_options, @@ -804,9 +809,9 @@ static int sparse_checkout_add(int argc, const char **argv, const char *prefix, for (int i = 0; i < argc; i++) strvec_push(&patterns, argv[i]); - sanitize_paths(&patterns, prefix, add_opts.skip_checks); + sanitize_paths(repo, &patterns, prefix, add_opts.skip_checks); - ret = modify_pattern_list(&patterns, add_opts.use_stdin, ADD); + ret = modify_pattern_list(repo, &patterns, add_opts.use_stdin, ADD); strvec_clear(&patterns); return ret; @@ -825,7 +830,7 @@ static struct sparse_checkout_set_opts { } set_opts; static int sparse_checkout_set(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { int default_patterns_nr = 2; const char *default_patterns[] = {"/*", "!/*/", NULL}; @@ -847,7 +852,7 @@ static int sparse_checkout_set(int argc, const char **argv, const char *prefix, int ret; setup_work_tree(); - repo_read_index(the_repository); + repo_read_index(repo); set_opts.cone_mode = -1; set_opts.sparse_index = -1; @@ -856,7 +861,7 @@ static int sparse_checkout_set(int argc, const char **argv, const char *prefix, builtin_sparse_checkout_set_options, builtin_sparse_checkout_set_usage, 0); - if (update_modes(&set_opts.cone_mode, &set_opts.sparse_index)) + if (update_modes(repo, &set_opts.cone_mode, &set_opts.sparse_index)) return 1; /* @@ -870,10 +875,10 @@ static int sparse_checkout_set(int argc, const char **argv, const char *prefix, } else { for (int i = 0; i < argc; i++) strvec_push(&patterns, argv[i]); - sanitize_paths(&patterns, prefix, set_opts.skip_checks); + sanitize_paths(repo, &patterns, prefix, set_opts.skip_checks); } - ret = modify_pattern_list(&patterns, set_opts.use_stdin, REPLACE); + ret = modify_pattern_list(repo, &patterns, set_opts.use_stdin, REPLACE); strvec_clear(&patterns); return ret; @@ -891,7 +896,7 @@ static struct sparse_checkout_reapply_opts { static int sparse_checkout_reapply(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { static struct option builtin_sparse_checkout_reapply_options[] = { OPT_BOOL(0, "cone", &reapply_opts.cone_mode, @@ -912,12 +917,107 @@ static int sparse_checkout_reapply(int argc, const char **argv, builtin_sparse_checkout_reapply_options, builtin_sparse_checkout_reapply_usage, 0); - repo_read_index(the_repository); + repo_read_index(repo); - if (update_modes(&reapply_opts.cone_mode, &reapply_opts.sparse_index)) + if (update_modes(repo, &reapply_opts.cone_mode, &reapply_opts.sparse_index)) return 1; - return update_working_directory(NULL); + return update_working_directory(repo, NULL); +} + +static char const * const builtin_sparse_checkout_clean_usage[] = { + "git sparse-checkout clean [-n|--dry-run]", + NULL +}; + +static int list_file_iterator(const char *path, const void *data) +{ + const char *msg = data; + + printf(msg, path); + return 0; +} + +static void list_every_file_in_dir(const char *msg, + const char *directory) +{ + struct strbuf path = STRBUF_INIT; + + strbuf_addstr(&path, directory); + for_each_file_in_dir(&path, list_file_iterator, msg); + strbuf_release(&path); +} + +static const char *msg_remove = N_("Removing %s\n"); +static const char *msg_would_remove = N_("Would remove %s\n"); + +static int sparse_checkout_clean(int argc, const char **argv, + const char *prefix, + struct repository *repo) +{ + struct strbuf full_path = STRBUF_INIT; + const char *msg = msg_remove; + size_t worktree_len; + int force = 0, dry_run = 0, verbose = 0; + int require_force = 1; + + struct option builtin_sparse_checkout_clean_options[] = { + OPT__DRY_RUN(&dry_run, N_("dry run")), + OPT__FORCE(&force, N_("force"), PARSE_OPT_NOCOMPLETE), + OPT__VERBOSE(&verbose, N_("report each affected file, not just directories")), + OPT_END(), + }; + + setup_work_tree(); + if (!core_apply_sparse_checkout) + die(_("must be in a sparse-checkout to clean directories")); + if (!core_sparse_checkout_cone) + die(_("must be in a cone-mode sparse-checkout to clean directories")); + + argc = parse_options(argc, argv, prefix, + builtin_sparse_checkout_clean_options, + builtin_sparse_checkout_clean_usage, 0); + + repo_config_get_bool(repo, "clean.requireforce", &require_force); + if (require_force && !force && !dry_run) + die(_("for safety, refusing to clean without one of --force or --dry-run")); + + if (dry_run) + msg = msg_would_remove; + + if (repo_read_index(repo) < 0) + die(_("failed to read index")); + + if (convert_to_sparse(repo->index, SPARSE_INDEX_MEMORY_ONLY) || + repo->index->sparse_index == INDEX_EXPANDED) + die(_("failed to convert index to a sparse index; resolve merge conflicts and try again")); + + strbuf_addstr(&full_path, repo->worktree); + strbuf_addch(&full_path, '/'); + worktree_len = full_path.len; + + for (size_t i = 0; i < repo->index->cache_nr; i++) { + struct cache_entry *ce = repo->index->cache[i]; + if (!S_ISSPARSEDIR(ce->ce_mode)) + continue; + strbuf_setlen(&full_path, worktree_len); + strbuf_add(&full_path, ce->name, ce->ce_namelen); + + if (!is_directory(full_path.buf)) + continue; + + if (verbose) + list_every_file_in_dir(msg, ce->name); + else + printf(msg, ce->name); + + if (dry_run <= 0 && + remove_dir_recursively(&full_path, 0)) + warning_errno(_("failed to remove '%s'"), ce->name); + } + + strbuf_release(&full_path); + return 0; } static char const * const builtin_sparse_checkout_disable_usage[] = { @@ -927,7 +1027,7 @@ static char const * const builtin_sparse_checkout_disable_usage[] = { static int sparse_checkout_disable(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { static struct option builtin_sparse_checkout_disable_options[] = { OPT_END(), @@ -955,7 +1055,7 @@ static int sparse_checkout_disable(int argc, const char **argv, * are expecting to do that when disabling sparse-checkout. */ give_advice_on_expansion = 0; - repo_read_index(the_repository); + repo_read_index(repo); memset(&pl, 0, sizeof(pl)); hashmap_init(&pl.recursive_hashmap, pl_hashmap_cmp, NULL, 0); @@ -966,13 +1066,13 @@ static int sparse_checkout_disable(int argc, const char **argv, add_pattern("/*", empty_base, 0, &pl, 0); prepare_repo_settings(the_repository); - the_repository->settings.sparse_index = 0; + repo->settings.sparse_index = 0; - if (update_working_directory(&pl)) + if (update_working_directory(repo, &pl)) die(_("error while refreshing working directory")); clear_pattern_list(&pl); - return set_config(MODE_NO_PATTERNS); + return set_config(repo, MODE_NO_PATTERNS); } static char const * const builtin_sparse_checkout_check_rules_usage[] = { @@ -987,14 +1087,17 @@ static struct sparse_checkout_check_rules_opts { char *rules_file; } check_rules_opts; -static int check_rules(struct pattern_list *pl, int null_terminated) { +static int check_rules(struct repository *repo, + struct pattern_list *pl, + int null_terminated) +{ struct strbuf line = STRBUF_INIT; struct strbuf unquoted = STRBUF_INIT; char *path; int line_terminator = null_terminated ? 0 : '\n'; strbuf_getline_fn getline_fn = null_terminated ? strbuf_getline_nul : strbuf_getline; - the_repository->index->sparse_checkout_patterns = pl; + repo->index->sparse_checkout_patterns = pl; while (!getline_fn(&line, stdin)) { path = line.buf; if (!null_terminated && line.buf[0] == '"') { @@ -1006,7 +1109,7 @@ static int check_rules(struct pattern_list *pl, int null_terminated) { path = unquoted.buf; } - if (path_in_sparse_checkout(path, the_repository->index)) + if (path_in_sparse_checkout(path, repo->index)) write_name_quoted(path, stdout, line_terminator); } strbuf_release(&line); @@ -1016,7 +1119,7 @@ static int check_rules(struct pattern_list *pl, int null_terminated) { } static int sparse_checkout_check_rules(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { static struct option builtin_sparse_checkout_check_rules_options[] = { OPT_BOOL('z', NULL, &check_rules_opts.null_termination, @@ -1055,7 +1158,7 @@ static int sparse_checkout_check_rules(int argc, const char **argv, const char * free(sparse_filename); } - ret = check_rules(&pl, check_rules_opts.null_termination); + ret = check_rules(repo, &pl, check_rules_opts.null_termination); clear_pattern_list(&pl); free(check_rules_opts.rules_file); return ret; @@ -1073,6 +1176,7 @@ int cmd_sparse_checkout(int argc, OPT_SUBCOMMAND("set", &fn, sparse_checkout_set), OPT_SUBCOMMAND("add", &fn, sparse_checkout_add), OPT_SUBCOMMAND("reapply", &fn, sparse_checkout_reapply), + OPT_SUBCOMMAND("clean", &fn, sparse_checkout_clean), OPT_SUBCOMMAND("disable", &fn, sparse_checkout_disable), OPT_SUBCOMMAND("check-rules", &fn, sparse_checkout_check_rules), OPT_END(), @@ -1084,8 +1188,8 @@ int cmd_sparse_checkout(int argc, repo_config(the_repository, git_default_config, NULL); - prepare_repo_settings(the_repository); - the_repository->settings.command_requires_full_index = 0; + prepare_repo_settings(repo); + repo->settings.command_requires_full_index = 0; return fn(argc, argv, prefix, repo); } diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index a8dcd9b9bc..50628ee2dd 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -10,6 +10,8 @@ begin_group "Install dependencies" P4WHENCE=https://cdist2.perforce.com/perforce/r23.2 LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION JGITWHENCE=https://repo1.maven.org/maven2/org/eclipse/jgit/org.eclipse.jgit.pgm/6.8.0.202311291450-r/org.eclipse.jgit.pgm-6.8.0.202311291450-r.sh +CARGO_MSRV_VERSION=0.18.4 +CARGO_MSRV_WHENCE=https://github.com/foresterre/cargo-msrv/releases/download/v$CARGO_MSRV_VERSION/cargo-msrv-x86_64-unknown-linux-musl-v$CARGO_MSRV_VERSION.tgz # Make sudo a no-op and execute the command directly when running as root. # While using sudo would be fine on most platforms when we are root already, @@ -129,21 +131,28 @@ esac case "$jobname" in ClangFormat) - sudo apt-get -q update sudo apt-get -q -y install clang-format ;; StaticAnalysis) - sudo apt-get -q update sudo apt-get -q -y install coccinelle libcurl4-openssl-dev libssl-dev \ libexpat-dev gettext make ;; +RustAnalysis) + sudo apt-get -q -y install rustup + rustup default stable + rustup component add clippy rustfmt + + wget -q "$CARGO_MSRV_WHENCE" -O "cargo-msvc.tgz" + sudo mkdir -p "$CUSTOM_PATH" + sudo tar -xf "cargo-msvc.tgz" --strip-components=1 \ + --directory "$CUSTOM_PATH" --wildcards "*/cargo-msrv" + sudo chmod a+x "$CUSTOM_PATH/cargo-msrv" + ;; sparse) - sudo apt-get -q update -q sudo apt-get -q -y install libssl-dev libcurl4-openssl-dev \ libexpat-dev gettext zlib1g-dev sparse ;; Documentation) - sudo apt-get -q update sudo apt-get -q -y install asciidoc xmlto docbook-xsl-ns make test -n "$ALREADY_HAVE_ASCIIDOCTOR" || diff --git a/ci/run-rust-checks.sh b/ci/run-rust-checks.sh new file mode 100755 index 0000000000..b5ad9e8dc6 --- /dev/null +++ b/ci/run-rust-checks.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +. ${0%/*}/lib.sh + +set +x + +if ! group "Check Rust formatting" cargo fmt --all --check +then + RET=1 +fi + +if ! group "Check for common Rust mistakes" cargo clippy --all-targets --all-features -- -Dwarnings +then + RET=1 +fi + +if ! group "Check for minimum required Rust version" cargo msrv verify +then + RET=1 +fi + +exit $RET diff --git a/connected.c b/connected.c index b288a18b17..79403108dd 100644 --- a/connected.c +++ b/connected.c @@ -74,10 +74,9 @@ int check_connected(oid_iterate_fn fn, void *cb_data, */ odb_reprepare(the_repository->objects); do { - struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { if (!p->pack_promisor) continue; if (find_pack_entry_one(oid, p)) diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash index e3d88b0672..73abea31b4 100644 --- a/contrib/completion/git-completion.bash +++ b/contrib/completion/git-completion.bash @@ -2218,7 +2218,7 @@ __git_log_gitk_options=" " # Options that go well for log and shortlog (not gitk) __git_log_shortlog_options=" - --author= --committer= --grep= + --author= --grep= --exclude= --all-match --invert-grep " # Options accepted by log and show @@ -2296,6 +2296,7 @@ __git_complete_log_opts () $__git_log_shortlog_options $__git_log_gitk_options $__git_log_show_options + --committer= --root --topo-order --date-order --reverse --follow --full-diff --abbrev-commit --no-abbrev-commit --abbrev= @@ -3229,7 +3230,7 @@ _git_shortlog () __gitcomp " $__git_log_common_options $__git_log_shortlog_options - --numbered --summary --email + --committer --numbered --summary --email " return ;; @@ -1351,6 +1351,9 @@ static void emit_diff_symbol_from_struct(struct diff_options *o, int len = eds->len; unsigned flags = eds->flags; + if (o->dry_run) + return; + switch (s) { case DIFF_SYMBOL_NO_LF_EOF: context = diff_get_color_opt(o, DIFF_CONTEXT); @@ -4420,7 +4423,7 @@ static void run_external_diff(const struct external_diff *pgm, { struct child_process cmd = CHILD_PROCESS_INIT; struct diff_queue_struct *q = &diff_queued_diff; - int quiet = !(o->output_format & DIFF_FORMAT_PATCH); + int quiet = !(o->output_format & DIFF_FORMAT_PATCH) || o->dry_run; int rc; /* @@ -4615,7 +4618,8 @@ static void run_diff_cmd(const struct external_diff *pgm, p->status == DIFF_STATUS_RENAMED) o->found_changes = 1; } else { - fprintf(o->file, "* Unmerged path %s\n", name); + if (!o->dry_run) + fprintf(o->file, "* Unmerged path %s\n", name); o->found_changes = 1; } } @@ -30,6 +30,7 @@ #include "read-cache-ll.h" #include "setup.h" #include "sparse-index.h" +#include "strbuf.h" #include "submodule-config.h" #include "symlinks.h" #include "trace2.h" @@ -87,6 +88,33 @@ struct dirent *readdir_skip_dot_and_dotdot(DIR *dirp) return e; } +int for_each_file_in_dir(struct strbuf *path, file_iterator fn, const void *data) +{ + struct dirent *e; + int res = 0; + size_t baselen = path->len; + DIR *dir = opendir(path->buf); + + if (!dir) + return 0; + + while (!res && (e = readdir_skip_dot_and_dotdot(dir)) != NULL) { + unsigned char dtype = get_dtype(e, path, 0); + strbuf_setlen(path, baselen); + strbuf_addstr(path, e->d_name); + + if (dtype == DT_REG) { + res = fn(path->buf, data); + } else if (dtype == DT_DIR) { + strbuf_addch(path, '/'); + res = for_each_file_in_dir(path, fn, data); + } + } + + closedir(dir); + return res; +} + int count_slashes(const char *s) { int cnt = 0; @@ -537,6 +537,20 @@ int get_sparse_checkout_patterns(struct pattern_list *pl); int remove_dir_recursively(struct strbuf *path, int flag); /* + * This function pointer type is called on each file discovered in + * for_each_file_in_dir. The iteration stops if this method returns + * non-zero. + */ +typedef int (*file_iterator)(const char *path, const void *data); + +struct strbuf; +/* + * Given a directory path, recursively visit each file within, including + * within subdirectories. + */ +int for_each_file_in_dir(struct strbuf *path, file_iterator fn, const void *data); + +/* * Tries to remove the path, along with leading empty directories so long as * those empty directories are not startup_info->original_cwd. Ignores * ENOENT. diff --git a/gpg-interface.c b/gpg-interface.c index 2f4f0e32cb..d1e88da8c1 100644 --- a/gpg-interface.c +++ b/gpg-interface.c @@ -821,8 +821,7 @@ static char *get_ssh_key_fingerprint(const char *signing_key) struct child_process ssh_keygen = CHILD_PROCESS_INIT; int ret = -1; struct strbuf fingerprint_stdout = STRBUF_INIT; - struct strbuf **fingerprint; - char *fingerprint_ret; + char *fingerprint_ret, *begin, *delim; const char *literal_key = NULL; /* @@ -845,13 +844,17 @@ static char *get_ssh_key_fingerprint(const char *signing_key) die_errno(_("failed to get the ssh fingerprint for key '%s'"), signing_key); - fingerprint = strbuf_split_max(&fingerprint_stdout, ' ', 3); - if (!fingerprint[1]) - die_errno(_("failed to get the ssh fingerprint for key '%s'"), + begin = fingerprint_stdout.buf; + delim = strchr(begin, ' '); + if (!delim) + die(_("failed to get the ssh fingerprint for key %s"), signing_key); - - fingerprint_ret = strbuf_detach(fingerprint[1], NULL); - strbuf_list_free(fingerprint); + begin = delim + 1; + delim = strchr(begin, ' '); + if (!delim) + die(_("failed to get the ssh fingerprint for key %s"), + signing_key); + fingerprint_ret = xmemdupz(begin, delim - begin); strbuf_release(&fingerprint_stdout); return fingerprint_ret; } @@ -862,12 +865,12 @@ static char *get_default_ssh_signing_key(void) struct child_process ssh_default_key = CHILD_PROCESS_INIT; int ret = -1; struct strbuf key_stdout = STRBUF_INIT, key_stderr = STRBUF_INIT; - struct strbuf **keys; char *key_command = NULL; const char **argv; int n; char *default_key = NULL; const char *literal_key = NULL; + char *begin, *new_line, *first_line; if (!ssh_default_key_command) die(_("either user.signingkey or gpg.ssh.defaultKeyCommand needs to be configured")); @@ -884,19 +887,24 @@ static char *get_default_ssh_signing_key(void) &key_stderr, 0); if (!ret) { - keys = strbuf_split_max(&key_stdout, '\n', 2); - if (keys[0] && is_literal_ssh_key(keys[0]->buf, &literal_key)) { + begin = key_stdout.buf; + new_line = strchr(begin, '\n'); + if (new_line) + first_line = xmemdupz(begin, new_line - begin); + else + first_line = xstrdup(begin); + if (is_literal_ssh_key(first_line, &literal_key)) { /* * We only use `is_literal_ssh_key` here to check validity * The prefix will be stripped when the key is used. */ - default_key = strbuf_detach(keys[0], NULL); + default_key = first_line; } else { + free(first_line); warning(_("gpg.ssh.defaultKeyCommand succeeded but returned no keys: %s %s"), key_stderr.buf, key_stdout.buf); } - strbuf_list_free(keys); } else { warning(_("gpg.ssh.defaultKeyCommand failed: %s %s"), key_stderr.buf, key_stdout.buf); diff --git a/http-backend.c b/http-backend.c index 9084058f1e..52f0483dd3 100644 --- a/http-backend.c +++ b/http-backend.c @@ -603,19 +603,18 @@ static void get_head(struct strbuf *hdr, char *arg UNUSED) static void get_info_packs(struct strbuf *hdr, char *arg UNUSED) { size_t objdirlen = strlen(repo_get_object_directory(the_repository)); - struct packfile_store *packs = the_repository->objects->packfiles; struct strbuf buf = STRBUF_INIT; struct packed_git *p; size_t cnt = 0; select_getanyfile(hdr); - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { if (p->pack_local) cnt++; } strbuf_grow(&buf, cnt * 53 + 2); - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { if (p->pack_local) strbuf_addf(&buf, "P %s\n", p->pack_name + objdirlen + 6); } @@ -2416,7 +2416,6 @@ static char *fetch_pack_index(unsigned char *hash, const char *base_url) static int fetch_and_setup_pack_index(struct packed_git **packs_head, unsigned char *sha1, const char *base_url) { - struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *new_pack, *p; char *tmp_idx = NULL; int ret; @@ -2425,7 +2424,7 @@ static int fetch_and_setup_pack_index(struct packed_git **packs_head, * If we already have the pack locally, no need to fetch its index or * even add it to list; we already have all of its objects. */ - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { if (hasheq(p->hash, sha1, the_repository->hash_algo)) return 0; } diff --git a/meson.build b/meson.build index cee9424475..2b763f7c53 100644 --- a/meson.build +++ b/meson.build @@ -463,6 +463,12 @@ libgit_sources = [ 'reftable/tree.c', 'reftable/writer.c', 'remote.c', + 'repack.c', + 'repack-cruft.c', + 'repack-filtered.c', + 'repack-geometry.c', + 'repack-midx.c', + 'repack-promisor.c', 'replace-object.c', 'repo-settings.c', 'repository.c', @@ -1708,6 +1714,10 @@ rust_option = get_option('rust').disable_auto_if(not cargo.found()) if rust_option.allowed() subdir('src') libgit_c_args += '-DWITH_RUST' + + if host_machine.system() == 'windows' + libgit_dependencies += compiler.find_library('userenv') + endif else libgit_sources += [ 'varint.c', diff --git a/object-name.c b/object-name.c index f6902e140d..766c757042 100644 --- a/object-name.c +++ b/object-name.c @@ -213,9 +213,11 @@ static void find_short_packed_object(struct disambiguate_state *ds) unique_in_midx(m, ds); } - for (p = packfile_store_get_packs(ds->repo->objects->packfiles); p && !ds->ambiguous; - p = p->next) + repo_for_each_pack(ds->repo, p) { + if (ds->ambiguous) + break; unique_in_pack(p, ds); + } } static int finish_object_disambiguation(struct disambiguate_state *ds, @@ -805,7 +807,7 @@ static void find_abbrev_len_packed(struct min_abbrev_data *mad) find_abbrev_len_for_midx(m, mad); } - for (p = packfile_store_get_packs(mad->repo->objects->packfiles); p; p = p->next) + repo_for_each_pack(mad->repo, p) find_abbrev_len_for_pack(p, mad); } diff --git a/pack-bitmap.c b/pack-bitmap.c index ac71035d77..291e1a9cf4 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -664,7 +664,7 @@ static int open_pack_bitmap(struct repository *r, struct packed_git *p; int ret = -1; - for (p = packfile_store_get_all_packs(r->objects->packfiles); p; p = p->next) { + repo_for_each_pack(r, p) { if (open_pack_bitmap_1(bitmap_git, p) == 0) { ret = 0; /* @@ -3347,6 +3347,7 @@ static int verify_bitmap_file(const struct git_hash_algo *algop, int verify_bitmap_files(struct repository *r) { struct odb_source *source; + struct packed_git *p; int res = 0; odb_prepare_alternates(r->objects); @@ -3362,8 +3363,7 @@ int verify_bitmap_files(struct repository *r) free(midx_bitmap_name); } - for (struct packed_git *p = packfile_store_get_all_packs(r->objects->packfiles); - p; p = p->next) { + repo_for_each_pack(r, p) { char *pack_bitmap_name = pack_bitmap_filename(p); res |= verify_bitmap_file(r->hash_algo, pack_bitmap_name); free(pack_bitmap_name); diff --git a/pack-objects.c b/pack-objects.c index 9d6ee72569..48510dd343 100644 --- a/pack-objects.c +++ b/pack-objects.c @@ -87,7 +87,6 @@ struct object_entry *packlist_find(struct packing_data *pdata, static void prepare_in_pack_by_idx(struct packing_data *pdata) { - struct packfile_store *packs = pdata->repo->objects->packfiles; struct packed_git **mapping, *p; int cnt = 0, nr = 1U << OE_IN_PACK_BITS; @@ -97,13 +96,13 @@ static void prepare_in_pack_by_idx(struct packing_data *pdata) * (i.e. in_pack_idx also zero) should return NULL. */ mapping[cnt++] = NULL; - for (p = packfile_store_get_all_packs(packs); p; p = p->next, cnt++) { + repo_for_each_pack(pdata->repo, p) { if (cnt == nr) { free(mapping); return; } p->index = cnt; - mapping[cnt] = p; + mapping[cnt++] = p; } pdata->in_pack_by_idx = mapping; } diff --git a/packfile.c b/packfile.c index 5a7caec292..1ae2b2fe1e 100644 --- a/packfile.c +++ b/packfile.c @@ -1030,12 +1030,6 @@ void packfile_store_reprepare(struct packfile_store *store) struct packed_git *packfile_store_get_packs(struct packfile_store *store) { packfile_store_prepare(store); - return store->packs; -} - -struct packed_git *packfile_store_get_all_packs(struct packfile_store *store) -{ - packfile_store_prepare(store); for (struct odb_source *source = store->odb->sources; source; source = source->next) { struct multi_pack_index *m = source->midx; @@ -2105,7 +2099,7 @@ struct packed_git **kept_pack_cache(struct repository *r, unsigned flags) * covers, one kept and one not kept, but the midx returns only * the non-kept version. */ - for (p = packfile_store_get_all_packs(r->objects->packfiles); p; p = p->next) { + repo_for_each_pack(r, p) { if ((p->pack_keep && (flags & ON_DISK_KEEP_PACKS)) || (p->pack_keep_in_core && (flags & IN_CORE_KEEP_PACKS))) { ALLOC_GROW(packs, nr + 1, alloc); @@ -2202,7 +2196,7 @@ int for_each_packed_object(struct repository *repo, each_packed_object_fn cb, int r = 0; int pack_errors = 0; - for (p = packfile_store_get_all_packs(repo->objects->packfiles); p; p = p->next) { + repo_for_each_pack(repo, p) { if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local) continue; if ((flags & FOR_EACH_OBJECT_PROMISOR_ONLY) && diff --git a/packfile.h b/packfile.h index e7a5792b6c..c9d0b93446 100644 --- a/packfile.h +++ b/packfile.h @@ -137,16 +137,18 @@ void packfile_store_add_pack(struct packfile_store *store, struct packed_git *pack); /* - * Get packs managed by the given store. Does not load the MIDX or any packs - * referenced by it. + * Load and iterate through all packs of the given repository. This helper + * function will yield packfiles from all object sources connected to the + * repository. */ -struct packed_git *packfile_store_get_packs(struct packfile_store *store); +#define repo_for_each_pack(repo, p) \ + for (p = packfile_store_get_packs(repo->objects->packfiles); p; p = p->next) /* * Get all packs managed by the given store, including packfiles that are * referenced by multi-pack indices. */ -struct packed_git *packfile_store_get_all_packs(struct packfile_store *store); +struct packed_git *packfile_store_get_packs(struct packfile_store *store); /* * Get all packs in most-recently-used order. diff --git a/refs/files-backend.c b/refs/files-backend.c index 8d7007f4aa..054cf42f4e 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -2113,20 +2113,35 @@ static int commit_ref_update(struct files_ref_store *refs, return 0; } -#ifdef NO_SYMLINK_HEAD +#if defined(NO_SYMLINK_HEAD) || defined(WITH_BREAKING_CHANGES) #define create_ref_symlink(a, b) (-1) #else static int create_ref_symlink(struct ref_lock *lock, const char *target) { + static int warn_once = 1; + char *ref_path; int ret = -1; - char *ref_path = get_locked_file_path(&lock->lk); + ref_path = get_locked_file_path(&lock->lk); unlink(ref_path); ret = symlink(target, ref_path); free(ref_path); if (ret) fprintf(stderr, "no symlink - falling back to symbolic ref\n"); + + if (warn_once) + warning(_("'core.preferSymlinkRefs=true' is nominated for removal.\n" + "hint: The use of symbolic links for symbolic refs is deprecated\n" + "hint: and will be removed in Git 3.0. The configuration that\n" + "hint: tells Git to use them is thus going away. You can unset\n" + "hint: it with:\n" + "hint:\n" + "hint:\tgit config unset core.preferSymlinkRefs\n" + "hint:\n" + "hint: Git will then use the textual symref format instead.")); + warn_once = 0; + return ret; } #endif diff --git a/repack-cruft.c b/repack-cruft.c new file mode 100644 index 0000000000..0653e88792 --- /dev/null +++ b/repack-cruft.c @@ -0,0 +1,98 @@ +#include "git-compat-util.h" +#include "repack.h" +#include "packfile.h" +#include "repository.h" +#include "run-command.h" + +static void combine_small_cruft_packs(FILE *in, off_t combine_cruft_below_size, + struct existing_packs *existing) +{ + struct packed_git *p; + struct strbuf buf = STRBUF_INIT; + size_t i; + + repo_for_each_pack(existing->repo, p) { + if (!(p->is_cruft && p->pack_local)) + continue; + + strbuf_reset(&buf); + strbuf_addstr(&buf, pack_basename(p)); + strbuf_strip_suffix(&buf, ".pack"); + + if (!string_list_has_string(&existing->cruft_packs, buf.buf)) + continue; + + if (p->pack_size < combine_cruft_below_size) { + fprintf(in, "-%s\n", pack_basename(p)); + } else { + existing_packs_retain_cruft(existing, p); + fprintf(in, "%s\n", pack_basename(p)); + } + } + + for (i = 0; i < existing->non_kept_packs.nr; i++) + fprintf(in, "-%s.pack\n", + existing->non_kept_packs.items[i].string); + + strbuf_release(&buf); +} + +int write_cruft_pack(const struct write_pack_opts *opts, + const char *cruft_expiration, + unsigned long combine_cruft_below_size, + struct string_list *names, + struct existing_packs *existing) +{ + struct child_process cmd = CHILD_PROCESS_INIT; + struct string_list_item *item; + FILE *in; + int ret; + const char *pack_prefix = write_pack_opts_pack_prefix(opts); + + prepare_pack_objects(&cmd, opts->po_args, opts->destination); + + strvec_push(&cmd.args, "--cruft"); + if (cruft_expiration) + strvec_pushf(&cmd.args, "--cruft-expiration=%s", + cruft_expiration); + + strvec_push(&cmd.args, "--non-empty"); + + cmd.in = -1; + + ret = start_command(&cmd); + if (ret) + return ret; + + /* + * names has a confusing double use: it both provides the list + * of just-written new packs, and accepts the name of the cruft + * pack we are writing. + * + * By the time it is read here, it contains only the pack(s) + * that were just written, which is exactly the set of packs we + * want to consider kept. + * + * If `--expire-to` is given, the double-use served by `names` + * ensures that the pack written to `--expire-to` excludes any + * objects contained in the cruft pack. + */ + in = xfdopen(cmd.in, "w"); + for_each_string_list_item(item, names) + fprintf(in, "%s-%s.pack\n", pack_prefix, item->string); + if (combine_cruft_below_size && !cruft_expiration) { + combine_small_cruft_packs(in, combine_cruft_below_size, + existing); + } else { + for_each_string_list_item(item, &existing->non_kept_packs) + fprintf(in, "-%s.pack\n", item->string); + for_each_string_list_item(item, &existing->cruft_packs) + fprintf(in, "-%s.pack\n", item->string); + } + for_each_string_list_item(item, &existing->kept_packs) + fprintf(in, "%s.pack\n", item->string); + fclose(in); + + return finish_pack_objects_cmd(existing->repo->hash_algo, opts, &cmd, + names); +} diff --git a/repack-filtered.c b/repack-filtered.c new file mode 100644 index 0000000000..edcf7667c5 --- /dev/null +++ b/repack-filtered.c @@ -0,0 +1,51 @@ +#include "git-compat-util.h" +#include "repack.h" +#include "repository.h" +#include "run-command.h" +#include "string-list.h" + +int write_filtered_pack(const struct write_pack_opts *opts, + struct existing_packs *existing, + struct string_list *names) +{ + struct child_process cmd = CHILD_PROCESS_INIT; + struct string_list_item *item; + FILE *in; + int ret; + const char *caret; + const char *pack_prefix = write_pack_opts_pack_prefix(opts); + + prepare_pack_objects(&cmd, opts->po_args, opts->destination); + + strvec_push(&cmd.args, "--stdin-packs"); + + for_each_string_list_item(item, &existing->kept_packs) + strvec_pushf(&cmd.args, "--keep-pack=%s", item->string); + + cmd.in = -1; + + ret = start_command(&cmd); + if (ret) + return ret; + + /* + * Here 'names' contains only the pack(s) that were just + * written, which is exactly the packs we want to keep. Also + * 'existing_kept_packs' already contains the packs in + * 'keep_pack_list'. + */ + in = xfdopen(cmd.in, "w"); + for_each_string_list_item(item, names) + fprintf(in, "^%s-%s.pack\n", pack_prefix, item->string); + for_each_string_list_item(item, &existing->non_kept_packs) + fprintf(in, "%s.pack\n", item->string); + for_each_string_list_item(item, &existing->cruft_packs) + fprintf(in, "%s.pack\n", item->string); + caret = opts->po_args->pack_kept_objects ? "" : "^"; + for_each_string_list_item(item, &existing->kept_packs) + fprintf(in, "%s%s.pack\n", caret, item->string); + fclose(in); + + return finish_pack_objects_cmd(existing->repo->hash_algo, opts, &cmd, + names); +} diff --git a/repack-geometry.c b/repack-geometry.c new file mode 100644 index 0000000000..b3e32cd07e --- /dev/null +++ b/repack-geometry.c @@ -0,0 +1,232 @@ +#define DISABLE_SIGN_COMPARE_WARNINGS + +#include "git-compat-util.h" +#include "repack.h" +#include "repository.h" +#include "hex.h" +#include "packfile.h" + +static uint32_t pack_geometry_weight(struct packed_git *p) +{ + if (open_pack_index(p)) + die(_("cannot open index for %s"), p->pack_name); + return p->num_objects; +} + +static int pack_geometry_cmp(const void *va, const void *vb) +{ + uint32_t aw = pack_geometry_weight(*(struct packed_git **)va), + bw = pack_geometry_weight(*(struct packed_git **)vb); + + if (aw < bw) + return -1; + if (aw > bw) + return 1; + return 0; +} + +void pack_geometry_init(struct pack_geometry *geometry, + struct existing_packs *existing, + const struct pack_objects_args *args) +{ + struct packed_git *p; + struct strbuf buf = STRBUF_INIT; + + repo_for_each_pack(existing->repo, p) { + if (args->local && !p->pack_local) + /* + * When asked to only repack local packfiles we skip + * over any packfiles that are borrowed from alternate + * object directories. + */ + continue; + + if (!args->pack_kept_objects) { + /* + * Any pack that has its pack_keep bit set will + * appear in existing->kept_packs below, but + * this saves us from doing a more expensive + * check. + */ + if (p->pack_keep) + continue; + + /* + * The pack may be kept via the --keep-pack + * option; check 'existing->kept_packs' to + * determine whether to ignore it. + */ + strbuf_reset(&buf); + strbuf_addstr(&buf, pack_basename(p)); + strbuf_strip_suffix(&buf, ".pack"); + + if (string_list_has_string(&existing->kept_packs, buf.buf)) + continue; + } + if (p->is_cruft) + continue; + + ALLOC_GROW(geometry->pack, + geometry->pack_nr + 1, + geometry->pack_alloc); + + geometry->pack[geometry->pack_nr] = p; + geometry->pack_nr++; + } + + QSORT(geometry->pack, geometry->pack_nr, pack_geometry_cmp); + strbuf_release(&buf); +} + +void pack_geometry_split(struct pack_geometry *geometry) +{ + uint32_t i; + uint32_t split; + off_t total_size = 0; + + if (!geometry->pack_nr) { + geometry->split = geometry->pack_nr; + return; + } + + /* + * First, count the number of packs (in descending order of size) which + * already form a geometric progression. + */ + for (i = geometry->pack_nr - 1; i > 0; i--) { + struct packed_git *ours = geometry->pack[i]; + struct packed_git *prev = geometry->pack[i - 1]; + + if (unsigned_mult_overflows(geometry->split_factor, + pack_geometry_weight(prev))) + die(_("pack %s too large to consider in geometric " + "progression"), + prev->pack_name); + + if (pack_geometry_weight(ours) < + geometry->split_factor * pack_geometry_weight(prev)) + break; + } + + split = i; + + if (split) { + /* + * Move the split one to the right, since the top element in the + * last-compared pair can't be in the progression. Only do this + * when we split in the middle of the array (otherwise if we got + * to the end, then the split is in the right place). + */ + split++; + } + + /* + * Then, anything to the left of 'split' must be in a new pack. But, + * creating that new pack may cause packs in the heavy half to no longer + * form a geometric progression. + * + * Compute an expected size of the new pack, and then determine how many + * packs in the heavy half need to be joined into it (if any) to restore + * the geometric progression. + */ + for (i = 0; i < split; i++) { + struct packed_git *p = geometry->pack[i]; + + if (unsigned_add_overflows(total_size, pack_geometry_weight(p))) + die(_("pack %s too large to roll up"), p->pack_name); + total_size += pack_geometry_weight(p); + } + for (i = split; i < geometry->pack_nr; i++) { + struct packed_git *ours = geometry->pack[i]; + + if (unsigned_mult_overflows(geometry->split_factor, + total_size)) + die(_("pack %s too large to roll up"), ours->pack_name); + + if (pack_geometry_weight(ours) < + geometry->split_factor * total_size) { + if (unsigned_add_overflows(total_size, + pack_geometry_weight(ours))) + die(_("pack %s too large to roll up"), + ours->pack_name); + + split++; + total_size += pack_geometry_weight(ours); + } else + break; + } + + geometry->split = split; +} + +struct packed_git *pack_geometry_preferred_pack(struct pack_geometry *geometry) +{ + uint32_t i; + + if (!geometry) { + /* + * No geometry means either an all-into-one repack (in which + * case there is only one pack left and it is the largest) or an + * incremental one. + * + * If repacking incrementally, then we could check the size of + * all packs to determine which should be preferred, but leave + * this for later. + */ + return NULL; + } + if (geometry->split == geometry->pack_nr) + return NULL; + + /* + * The preferred pack is the largest pack above the split line. In + * other words, it is the largest pack that does not get rolled up in + * the geometric repack. + */ + for (i = geometry->pack_nr; i > geometry->split; i--) + /* + * A pack that is not local would never be included in a + * multi-pack index. We thus skip over any non-local packs. + */ + if (geometry->pack[i - 1]->pack_local) + return geometry->pack[i - 1]; + + return NULL; +} + +void pack_geometry_remove_redundant(struct pack_geometry *geometry, + struct string_list *names, + struct existing_packs *existing, + const char *packdir) +{ + const struct git_hash_algo *algop = existing->repo->hash_algo; + struct strbuf buf = STRBUF_INIT; + uint32_t i; + + for (i = 0; i < geometry->split; i++) { + struct packed_git *p = geometry->pack[i]; + if (string_list_has_string(names, hash_to_hex_algop(p->hash, + algop))) + continue; + + strbuf_reset(&buf); + strbuf_addstr(&buf, pack_basename(p)); + strbuf_strip_suffix(&buf, ".pack"); + + if ((p->pack_keep) || + (string_list_has_string(&existing->kept_packs, buf.buf))) + continue; + + repack_remove_redundant_pack(existing->repo, packdir, buf.buf); + } + + strbuf_release(&buf); +} + +void pack_geometry_release(struct pack_geometry *geometry) +{ + if (!geometry) + return; + + free(geometry->pack); +} diff --git a/repack-midx.c b/repack-midx.c new file mode 100644 index 0000000000..6f6202c5bc --- /dev/null +++ b/repack-midx.c @@ -0,0 +1,372 @@ +#include "git-compat-util.h" +#include "repack.h" +#include "hash.h" +#include "hex.h" +#include "odb.h" +#include "oidset.h" +#include "pack-bitmap.h" +#include "refs.h" +#include "run-command.h" +#include "tempfile.h" + +struct midx_snapshot_ref_data { + struct repository *repo; + struct tempfile *f; + struct oidset seen; + int preferred; +}; + +static int midx_snapshot_ref_one(const char *refname UNUSED, + const char *referent UNUSED, + const struct object_id *oid, + int flag UNUSED, void *_data) +{ + struct midx_snapshot_ref_data *data = _data; + struct object_id peeled; + + if (!peel_iterated_oid(data->repo, oid, &peeled)) + oid = &peeled; + + if (oidset_insert(&data->seen, oid)) + return 0; /* already seen */ + + if (odb_read_object_info(data->repo->objects, oid, NULL) != OBJ_COMMIT) + return 0; + + fprintf(data->f->fp, "%s%s\n", data->preferred ? "+" : "", + oid_to_hex(oid)); + + return 0; +} + +void midx_snapshot_refs(struct repository *repo, struct tempfile *f) +{ + struct midx_snapshot_ref_data data; + const struct string_list *preferred = bitmap_preferred_tips(repo); + + data.repo = repo; + data.f = f; + data.preferred = 0; + oidset_init(&data.seen, 0); + + if (!fdopen_tempfile(f, "w")) + die(_("could not open tempfile %s for writing"), + get_tempfile_path(f)); + + if (preferred) { + struct string_list_item *item; + + data.preferred = 1; + for_each_string_list_item(item, preferred) + refs_for_each_ref_in(get_main_ref_store(repo), + item->string, + midx_snapshot_ref_one, &data); + data.preferred = 0; + } + + refs_for_each_ref(get_main_ref_store(repo), + midx_snapshot_ref_one, &data); + + if (close_tempfile_gently(f)) { + int save_errno = errno; + delete_tempfile(&f); + errno = save_errno; + die_errno(_("could not close refs snapshot tempfile")); + } + + oidset_clear(&data.seen); +} + +static int midx_has_unknown_packs(struct string_list *include, + struct pack_geometry *geometry, + struct existing_packs *existing) +{ + struct string_list_item *item; + + string_list_sort(include); + + for_each_string_list_item(item, &existing->midx_packs) { + const char *pack_name = item->string; + + /* + * Determine whether or not each MIDX'd pack from the existing + * MIDX (if any) is represented in the new MIDX. For each pack + * in the MIDX, it must either be: + * + * - In the "include" list of packs to be included in the new + * MIDX. Note this function is called before the include + * list is populated with any cruft pack(s). + * + * - Below the geometric split line (if using pack geometry), + * indicating that the pack won't be included in the new + * MIDX, but its contents were rolled up as part of the + * geometric repack. + * + * - In the existing non-kept packs list (if not using pack + * geometry), and marked as non-deleted. + */ + if (string_list_has_string(include, pack_name)) { + continue; + } else if (geometry) { + struct strbuf buf = STRBUF_INIT; + uint32_t j; + + for (j = 0; j < geometry->split; j++) { + strbuf_reset(&buf); + strbuf_addstr(&buf, pack_basename(geometry->pack[j])); + strbuf_strip_suffix(&buf, ".pack"); + strbuf_addstr(&buf, ".idx"); + + if (!strcmp(pack_name, buf.buf)) { + strbuf_release(&buf); + break; + } + } + + strbuf_release(&buf); + + if (j < geometry->split) + continue; + } else { + struct string_list_item *item; + + item = string_list_lookup(&existing->non_kept_packs, + pack_name); + if (item && !existing_pack_is_marked_for_deletion(item)) + continue; + } + + /* + * If we got to this point, the MIDX includes some pack that we + * don't know about. + */ + return 1; + } + + return 0; +} + +static void midx_included_packs(struct string_list *include, + struct repack_write_midx_opts *opts) +{ + struct existing_packs *existing = opts->existing; + struct pack_geometry *geometry = opts->geometry; + struct string_list *names = opts->names; + struct string_list_item *item; + struct strbuf buf = STRBUF_INIT; + + for_each_string_list_item(item, &existing->kept_packs) { + strbuf_reset(&buf); + strbuf_addf(&buf, "%s.idx", item->string); + string_list_insert(include, buf.buf); + } + + for_each_string_list_item(item, names) { + strbuf_reset(&buf); + strbuf_addf(&buf, "pack-%s.idx", item->string); + string_list_insert(include, buf.buf); + } + + if (geometry->split_factor) { + uint32_t i; + + for (i = geometry->split; i < geometry->pack_nr; i++) { + struct packed_git *p = geometry->pack[i]; + + /* + * The multi-pack index never refers to packfiles part + * of an alternate object database, so we skip these. + * While git-multi-pack-index(1) would silently ignore + * them anyway, this allows us to skip executing the + * command completely when we have only non-local + * packfiles. + */ + if (!p->pack_local) + continue; + + strbuf_reset(&buf); + strbuf_addstr(&buf, pack_basename(p)); + strbuf_strip_suffix(&buf, ".pack"); + strbuf_addstr(&buf, ".idx"); + + string_list_insert(include, buf.buf); + } + } else { + for_each_string_list_item(item, &existing->non_kept_packs) { + if (existing_pack_is_marked_for_deletion(item)) + continue; + + strbuf_reset(&buf); + strbuf_addf(&buf, "%s.idx", item->string); + string_list_insert(include, buf.buf); + } + } + + if (opts->midx_must_contain_cruft || + midx_has_unknown_packs(include, geometry, existing)) { + /* + * If there are one or more unknown pack(s) present (see + * midx_has_unknown_packs() for what makes a pack + * "unknown") in the MIDX before the repack, keep them + * as they may be required to form a reachability + * closure if the MIDX is bitmapped. + * + * For example, a cruft pack can be required to form a + * reachability closure if the MIDX is bitmapped and one + * or more of the bitmap's selected commits reaches a + * once-cruft object that was later made reachable. + */ + for_each_string_list_item(item, &existing->cruft_packs) { + /* + * When doing a --geometric repack, there is no + * need to check for deleted packs, since we're + * by definition not doing an ALL_INTO_ONE + * repack (hence no packs will be deleted). + * Otherwise we must check for and exclude any + * packs which are enqueued for deletion. + * + * So we could omit the conditional below in the + * --geometric case, but doing so is unnecessary + * since no packs are marked as pending + * deletion (since we only call + * `existing_packs_mark_for_deletion()` when + * doing an all-into-one repack). + */ + if (existing_pack_is_marked_for_deletion(item)) + continue; + + strbuf_reset(&buf); + strbuf_addf(&buf, "%s.idx", item->string); + string_list_insert(include, buf.buf); + } + } else { + /* + * Modern versions of Git (with the appropriate + * configuration setting) will write new copies of + * once-cruft objects when doing a --geometric repack. + * + * If the MIDX has no cruft pack, new packs written + * during a --geometric repack will not rely on the + * cruft pack to form a reachability closure, so we can + * avoid including them in the MIDX in that case. + */ + ; + } + + strbuf_release(&buf); +} + +static void remove_redundant_bitmaps(struct string_list *include, + const char *packdir) +{ + struct strbuf path = STRBUF_INIT; + struct string_list_item *item; + size_t packdir_len; + + strbuf_addstr(&path, packdir); + strbuf_addch(&path, '/'); + packdir_len = path.len; + + /* + * Remove any pack bitmaps corresponding to packs which are now + * included in the MIDX. + */ + for_each_string_list_item(item, include) { + strbuf_addstr(&path, item->string); + strbuf_strip_suffix(&path, ".idx"); + strbuf_addstr(&path, ".bitmap"); + + if (unlink(path.buf) && errno != ENOENT) + warning_errno(_("could not remove stale bitmap: %s"), + path.buf); + + strbuf_setlen(&path, packdir_len); + } + strbuf_release(&path); +} + +int write_midx_included_packs(struct repack_write_midx_opts *opts) +{ + struct child_process cmd = CHILD_PROCESS_INIT; + struct string_list include = STRING_LIST_INIT_DUP; + struct string_list_item *item; + struct packed_git *preferred = pack_geometry_preferred_pack(opts->geometry); + FILE *in; + int ret = 0; + + midx_included_packs(&include, opts); + if (!include.nr) + goto done; + + cmd.in = -1; + cmd.git_cmd = 1; + + strvec_push(&cmd.args, "multi-pack-index"); + strvec_pushl(&cmd.args, "write", "--stdin-packs", NULL); + + if (opts->show_progress) + strvec_push(&cmd.args, "--progress"); + else + strvec_push(&cmd.args, "--no-progress"); + + if (opts->write_bitmaps) + strvec_push(&cmd.args, "--bitmap"); + + if (preferred) + strvec_pushf(&cmd.args, "--preferred-pack=%s", + pack_basename(preferred)); + else if (opts->names->nr) { + /* The largest pack was repacked, meaning that either + * one or two packs exist depending on whether the + * repository has a cruft pack or not. + * + * Select the non-cruft one as preferred to encourage + * pack-reuse among packs containing reachable objects + * over unreachable ones. + * + * (Note we could write multiple packs here if + * `--max-pack-size` was given, but any one of them + * will suffice, so pick the first one.) + */ + for_each_string_list_item(item, opts->names) { + struct generated_pack *pack = item->util; + if (generated_pack_has_ext(pack, ".mtimes")) + continue; + + strvec_pushf(&cmd.args, "--preferred-pack=pack-%s.pack", + item->string); + break; + } + } else { + /* + * No packs were kept, and no packs were written. The + * only thing remaining are .keep packs (unless + * --pack-kept-objects was given). + * + * Set the `--preferred-pack` arbitrarily here. + */ + ; + } + + if (opts->refs_snapshot) + strvec_pushf(&cmd.args, "--refs-snapshot=%s", + opts->refs_snapshot); + + ret = start_command(&cmd); + if (ret) + goto done; + + in = xfdopen(cmd.in, "w"); + for_each_string_list_item(item, &include) + fprintf(in, "%s\n", item->string); + fclose(in); + + ret = finish_command(&cmd); +done: + if (!ret && opts->write_bitmaps) + remove_redundant_bitmaps(&include, opts->packdir); + + string_list_clear(&include, 0); + + return ret; +} diff --git a/repack-promisor.c b/repack-promisor.c new file mode 100644 index 0000000000..ee6e0669f6 --- /dev/null +++ b/repack-promisor.c @@ -0,0 +1,102 @@ +#include "git-compat-util.h" +#include "repack.h" +#include "hex.h" +#include "pack.h" +#include "packfile.h" +#include "path.h" +#include "repository.h" +#include "run-command.h" + +struct write_oid_context { + struct child_process *cmd; + const struct git_hash_algo *algop; +}; + +/* + * Write oid to the given struct child_process's stdin, starting it first if + * necessary. + */ +static int write_oid(const struct object_id *oid, + struct packed_git *pack UNUSED, + uint32_t pos UNUSED, void *data) +{ + struct write_oid_context *ctx = data; + struct child_process *cmd = ctx->cmd; + + if (cmd->in == -1) { + if (start_command(cmd)) + die(_("could not start pack-objects to repack promisor objects")); + } + + if (write_in_full(cmd->in, oid_to_hex(oid), ctx->algop->hexsz) < 0 || + write_in_full(cmd->in, "\n", 1) < 0) + die(_("failed to feed promisor objects to pack-objects")); + return 0; +} + +void repack_promisor_objects(struct repository *repo, + const struct pack_objects_args *args, + struct string_list *names, const char *packtmp) +{ + struct write_oid_context ctx; + struct child_process cmd = CHILD_PROCESS_INIT; + FILE *out; + struct strbuf line = STRBUF_INIT; + + prepare_pack_objects(&cmd, args, packtmp); + cmd.in = -1; + + /* + * NEEDSWORK: Giving pack-objects only the OIDs without any ordering + * hints may result in suboptimal deltas in the resulting pack. See if + * the OIDs can be sent with fake paths such that pack-objects can use a + * {type -> existing pack order} ordering when computing deltas instead + * of a {type -> size} ordering, which may produce better deltas. + */ + ctx.cmd = &cmd; + ctx.algop = repo->hash_algo; + for_each_packed_object(repo, write_oid, &ctx, + FOR_EACH_OBJECT_PROMISOR_ONLY); + + if (cmd.in == -1) { + /* No packed objects; cmd was never started */ + child_process_clear(&cmd); + return; + } + + close(cmd.in); + + out = xfdopen(cmd.out, "r"); + while (strbuf_getline_lf(&line, out) != EOF) { + struct string_list_item *item; + char *promisor_name; + + if (line.len != repo->hash_algo->hexsz) + die(_("repack: Expecting full hex object ID lines only from pack-objects.")); + item = string_list_append(names, line.buf); + + /* + * pack-objects creates the .pack and .idx files, but not the + * .promisor file. Create the .promisor file, which is empty. + * + * NEEDSWORK: fetch-pack sometimes generates non-empty + * .promisor files containing the ref names and associated + * hashes at the point of generation of the corresponding + * packfile, but this would not preserve their contents. Maybe + * concatenate the contents of all .promisor files instead of + * just creating a new empty file. + */ + promisor_name = mkpathdup("%s-%s.promisor", packtmp, + line.buf); + write_promisor_file(promisor_name, NULL, 0); + + item->util = generated_pack_populate(item->string, packtmp); + + free(promisor_name); + } + + fclose(out); + if (finish_command(&cmd)) + die(_("could not finish pack-objects to repack promisor objects")); + strbuf_release(&line); +} diff --git a/repack.c b/repack.c new file mode 100644 index 0000000000..596841027a --- /dev/null +++ b/repack.c @@ -0,0 +1,359 @@ +#include "git-compat-util.h" +#include "dir.h" +#include "midx.h" +#include "odb.h" +#include "packfile.h" +#include "path.h" +#include "repack.h" +#include "repository.h" +#include "run-command.h" +#include "tempfile.h" + +void prepare_pack_objects(struct child_process *cmd, + const struct pack_objects_args *args, + const char *out) +{ + strvec_push(&cmd->args, "pack-objects"); + if (args->window) + strvec_pushf(&cmd->args, "--window=%s", args->window); + if (args->window_memory) + strvec_pushf(&cmd->args, "--window-memory=%s", args->window_memory); + if (args->depth) + strvec_pushf(&cmd->args, "--depth=%s", args->depth); + if (args->threads) + strvec_pushf(&cmd->args, "--threads=%s", args->threads); + if (args->max_pack_size) + strvec_pushf(&cmd->args, "--max-pack-size=%lu", args->max_pack_size); + if (args->no_reuse_delta) + strvec_pushf(&cmd->args, "--no-reuse-delta"); + if (args->no_reuse_object) + strvec_pushf(&cmd->args, "--no-reuse-object"); + if (args->name_hash_version) + strvec_pushf(&cmd->args, "--name-hash-version=%d", args->name_hash_version); + if (args->path_walk) + strvec_pushf(&cmd->args, "--path-walk"); + if (args->local) + strvec_push(&cmd->args, "--local"); + if (args->quiet) + strvec_push(&cmd->args, "--quiet"); + if (args->delta_base_offset) + strvec_push(&cmd->args, "--delta-base-offset"); + if (!args->pack_kept_objects) + strvec_push(&cmd->args, "--honor-pack-keep"); + strvec_push(&cmd->args, out); + cmd->git_cmd = 1; + cmd->out = -1; +} + +void pack_objects_args_release(struct pack_objects_args *args) +{ + free(args->window); + free(args->window_memory); + free(args->depth); + free(args->threads); + list_objects_filter_release(&args->filter_options); +} + +void repack_remove_redundant_pack(struct repository *repo, const char *dir_name, + const char *base_name) +{ + struct strbuf buf = STRBUF_INIT; + struct odb_source *source = repo->objects->sources; + struct multi_pack_index *m = get_multi_pack_index(source); + strbuf_addf(&buf, "%s.pack", base_name); + if (m && source->local && midx_contains_pack(m, buf.buf)) + clear_midx_file(repo); + strbuf_insertf(&buf, 0, "%s/", dir_name); + unlink_pack_path(buf.buf, 1); + strbuf_release(&buf); +} + +const char *write_pack_opts_pack_prefix(const struct write_pack_opts *opts) +{ + const char *pack_prefix; + if (!skip_prefix(opts->packtmp, opts->packdir, &pack_prefix)) + die(_("pack prefix %s does not begin with objdir %s"), + opts->packtmp, opts->packdir); + if (*pack_prefix == '/') + pack_prefix++; + return pack_prefix; +} + +bool write_pack_opts_is_local(const struct write_pack_opts *opts) +{ + return starts_with(opts->destination, opts->packdir); +} + +int finish_pack_objects_cmd(const struct git_hash_algo *algop, + const struct write_pack_opts *opts, + struct child_process *cmd, + struct string_list *names) +{ + FILE *out; + bool local = write_pack_opts_is_local(opts); + struct strbuf line = STRBUF_INIT; + + out = xfdopen(cmd->out, "r"); + while (strbuf_getline_lf(&line, out) != EOF) { + struct string_list_item *item; + + if (line.len != algop->hexsz) + die(_("repack: Expecting full hex object ID lines only " + "from pack-objects.")); + /* + * Avoid putting packs written outside of the repository in the + * list of names. + */ + if (local) { + item = string_list_append(names, line.buf); + item->util = generated_pack_populate(line.buf, + opts->packtmp); + } + } + fclose(out); + + strbuf_release(&line); + + return finish_command(cmd); +} + +#define DELETE_PACK 1 +#define RETAIN_PACK 2 + +void existing_packs_collect(struct existing_packs *existing, + const struct string_list *extra_keep) +{ + struct packed_git *p; + struct strbuf buf = STRBUF_INIT; + + repo_for_each_pack(existing->repo, p) { + size_t i; + const char *base; + + if (p->multi_pack_index) + string_list_append(&existing->midx_packs, + pack_basename(p)); + if (!p->pack_local) + continue; + + base = pack_basename(p); + + for (i = 0; i < extra_keep->nr; i++) + if (!fspathcmp(base, extra_keep->items[i].string)) + break; + + strbuf_reset(&buf); + strbuf_addstr(&buf, base); + strbuf_strip_suffix(&buf, ".pack"); + + if ((extra_keep->nr > 0 && i < extra_keep->nr) || p->pack_keep) + string_list_append(&existing->kept_packs, buf.buf); + else if (p->is_cruft) + string_list_append(&existing->cruft_packs, buf.buf); + else + string_list_append(&existing->non_kept_packs, buf.buf); + } + + string_list_sort(&existing->kept_packs); + string_list_sort(&existing->non_kept_packs); + string_list_sort(&existing->cruft_packs); + string_list_sort(&existing->midx_packs); + strbuf_release(&buf); +} + +int existing_packs_has_non_kept(const struct existing_packs *existing) +{ + return existing->non_kept_packs.nr || existing->cruft_packs.nr; +} + +static void existing_pack_mark_for_deletion(struct string_list_item *item) +{ + item->util = (void*)((uintptr_t)item->util | DELETE_PACK); +} + +static void existing_pack_unmark_for_deletion(struct string_list_item *item) +{ + item->util = (void*)((uintptr_t)item->util & ~DELETE_PACK); +} + +int existing_pack_is_marked_for_deletion(struct string_list_item *item) +{ + return (uintptr_t)item->util & DELETE_PACK; +} + +static void existing_packs_mark_retained(struct string_list_item *item) +{ + item->util = (void*)((uintptr_t)item->util | RETAIN_PACK); +} + +static int existing_pack_is_retained(struct string_list_item *item) +{ + return (uintptr_t)item->util & RETAIN_PACK; +} + +static void existing_packs_mark_for_deletion_1(const struct git_hash_algo *algop, + struct string_list *names, + struct string_list *list) +{ + struct string_list_item *item; + const size_t hexsz = algop->hexsz; + + for_each_string_list_item(item, list) { + char *sha1; + size_t len = strlen(item->string); + if (len < hexsz) + continue; + sha1 = item->string + len - hexsz; + + if (existing_pack_is_retained(item)) { + existing_pack_unmark_for_deletion(item); + } else if (!string_list_has_string(names, sha1)) { + /* + * Mark this pack for deletion, which ensures + * that this pack won't be included in a MIDX + * (if `--write-midx` was given) and that we + * will actually delete this pack (if `-d` was + * given). + */ + existing_pack_mark_for_deletion(item); + } + } +} + +void existing_packs_retain_cruft(struct existing_packs *existing, + struct packed_git *cruft) +{ + struct strbuf buf = STRBUF_INIT; + struct string_list_item *item; + + strbuf_addstr(&buf, pack_basename(cruft)); + strbuf_strip_suffix(&buf, ".pack"); + + item = string_list_lookup(&existing->cruft_packs, buf.buf); + if (!item) + BUG("could not find cruft pack '%s'", pack_basename(cruft)); + + existing_packs_mark_retained(item); + strbuf_release(&buf); +} + +void existing_packs_mark_for_deletion(struct existing_packs *existing, + struct string_list *names) + +{ + const struct git_hash_algo *algop = existing->repo->hash_algo; + existing_packs_mark_for_deletion_1(algop, names, + &existing->non_kept_packs); + existing_packs_mark_for_deletion_1(algop, names, + &existing->cruft_packs); +} + +static void remove_redundant_packs_1(struct repository *repo, + struct string_list *packs, + const char *packdir) +{ + struct string_list_item *item; + for_each_string_list_item(item, packs) { + if (!existing_pack_is_marked_for_deletion(item)) + continue; + repack_remove_redundant_pack(repo, packdir, item->string); + } +} + +void existing_packs_remove_redundant(struct existing_packs *existing, + const char *packdir) +{ + remove_redundant_packs_1(existing->repo, &existing->non_kept_packs, + packdir); + remove_redundant_packs_1(existing->repo, &existing->cruft_packs, + packdir); +} + +void existing_packs_release(struct existing_packs *existing) +{ + string_list_clear(&existing->kept_packs, 0); + string_list_clear(&existing->non_kept_packs, 0); + string_list_clear(&existing->cruft_packs, 0); + string_list_clear(&existing->midx_packs, 0); +} + +static struct { + const char *name; + unsigned optional:1; +} exts[] = { + {".pack"}, + {".rev", 1}, + {".mtimes", 1}, + {".bitmap", 1}, + {".promisor", 1}, + {".idx"}, +}; + +struct generated_pack { + struct tempfile *tempfiles[ARRAY_SIZE(exts)]; +}; + +struct generated_pack *generated_pack_populate(const char *name, + const char *packtmp) +{ + struct stat statbuf; + struct strbuf path = STRBUF_INIT; + struct generated_pack *pack = xcalloc(1, sizeof(*pack)); + size_t i; + + for (i = 0; i < ARRAY_SIZE(exts); i++) { + strbuf_reset(&path); + strbuf_addf(&path, "%s-%s%s", packtmp, name, exts[i].name); + + if (stat(path.buf, &statbuf)) + continue; + + pack->tempfiles[i] = register_tempfile(path.buf); + } + + strbuf_release(&path); + return pack; +} + +int generated_pack_has_ext(const struct generated_pack *pack, const char *ext) +{ + size_t i; + for (i = 0; i < ARRAY_SIZE(exts); i++) { + if (strcmp(exts[i].name, ext)) + continue; + return !!pack->tempfiles[i]; + } + BUG("unknown pack extension: '%s'", ext); +} + +void generated_pack_install(struct generated_pack *pack, const char *name, + const char *packdir, const char *packtmp) +{ + size_t ext; + for (ext = 0; ext < ARRAY_SIZE(exts); ext++) { + char *fname; + + fname = mkpathdup("%s/pack-%s%s", packdir, name, + exts[ext].name); + + if (pack->tempfiles[ext]) { + const char *fname_old = get_tempfile_path(pack->tempfiles[ext]); + struct stat statbuffer; + + if (!stat(fname_old, &statbuffer)) { + statbuffer.st_mode &= ~(S_IWUSR | S_IWGRP | S_IWOTH); + chmod(fname_old, statbuffer.st_mode); + } + + if (rename_tempfile(&pack->tempfiles[ext], fname)) + die_errno(_("renaming pack to '%s' failed"), + fname); + } else if (!exts[ext].optional) + die(_("pack-objects did not write a '%s' file for pack %s-%s"), + exts[ext].name, packtmp, name); + else if (unlink(fname) < 0 && errno != ENOENT) + die_errno(_("could not unlink: %s"), fname); + + free(fname); + } +} diff --git a/repack.h b/repack.h new file mode 100644 index 0000000000..3a688a12ee --- /dev/null +++ b/repack.h @@ -0,0 +1,146 @@ +#ifndef REPACK_H +#define REPACK_H + +#include "list-objects-filter-options.h" +#include "string-list.h" + +struct pack_objects_args { + char *window; + char *window_memory; + char *depth; + char *threads; + unsigned long max_pack_size; + int no_reuse_delta; + int no_reuse_object; + int quiet; + int local; + int name_hash_version; + int path_walk; + int delta_base_offset; + int pack_kept_objects; + struct list_objects_filter_options filter_options; +}; + +#define PACK_OBJECTS_ARGS_INIT { \ + .delta_base_offset = 1, \ + .pack_kept_objects = -1, \ +} + +struct child_process; + +void prepare_pack_objects(struct child_process *cmd, + const struct pack_objects_args *args, + const char *out); +void pack_objects_args_release(struct pack_objects_args *args); + +void repack_remove_redundant_pack(struct repository *repo, const char *dir_name, + const char *base_name); + +struct write_pack_opts { + struct pack_objects_args *po_args; + const char *destination; + const char *packdir; + const char *packtmp; +}; + +const char *write_pack_opts_pack_prefix(const struct write_pack_opts *opts); +bool write_pack_opts_is_local(const struct write_pack_opts *opts); + +int finish_pack_objects_cmd(const struct git_hash_algo *algop, + const struct write_pack_opts *opts, + struct child_process *cmd, + struct string_list *names); + +struct repository; +struct packed_git; + +struct existing_packs { + struct repository *repo; + struct string_list kept_packs; + struct string_list non_kept_packs; + struct string_list cruft_packs; + struct string_list midx_packs; +}; + +#define EXISTING_PACKS_INIT { \ + .kept_packs = STRING_LIST_INIT_DUP, \ + .non_kept_packs = STRING_LIST_INIT_DUP, \ + .cruft_packs = STRING_LIST_INIT_DUP, \ +} + +/* + * Adds all packs hex strings (pack-$HASH) to either packs->non_kept + * or packs->kept based on whether each pack has a corresponding + * .keep file or not. Packs without a .keep file are not to be kept + * if we are going to pack everything into one file. + */ +void existing_packs_collect(struct existing_packs *existing, + const struct string_list *extra_keep); +int existing_packs_has_non_kept(const struct existing_packs *existing); +int existing_pack_is_marked_for_deletion(struct string_list_item *item); +void existing_packs_retain_cruft(struct existing_packs *existing, + struct packed_git *cruft); +void existing_packs_mark_for_deletion(struct existing_packs *existing, + struct string_list *names); +void existing_packs_remove_redundant(struct existing_packs *existing, + const char *packdir); +void existing_packs_release(struct existing_packs *existing); + +struct generated_pack; + +struct generated_pack *generated_pack_populate(const char *name, + const char *packtmp); +int generated_pack_has_ext(const struct generated_pack *pack, const char *ext); +void generated_pack_install(struct generated_pack *pack, const char *name, + const char *packdir, const char *packtmp); + +void repack_promisor_objects(struct repository *repo, + const struct pack_objects_args *args, + struct string_list *names, const char *packtmp); + +struct pack_geometry { + struct packed_git **pack; + uint32_t pack_nr, pack_alloc; + uint32_t split; + + int split_factor; +}; + +void pack_geometry_init(struct pack_geometry *geometry, + struct existing_packs *existing, + const struct pack_objects_args *args); +void pack_geometry_split(struct pack_geometry *geometry); +struct packed_git *pack_geometry_preferred_pack(struct pack_geometry *geometry); +void pack_geometry_remove_redundant(struct pack_geometry *geometry, + struct string_list *names, + struct existing_packs *existing, + const char *packdir); +void pack_geometry_release(struct pack_geometry *geometry); + +struct tempfile; + +struct repack_write_midx_opts { + struct existing_packs *existing; + struct pack_geometry *geometry; + struct string_list *names; + const char *refs_snapshot; + const char *packdir; + int show_progress; + int write_bitmaps; + int midx_must_contain_cruft; +}; + +void midx_snapshot_refs(struct repository *repo, struct tempfile *f); +int write_midx_included_packs(struct repack_write_midx_opts *opts); + +int write_filtered_pack(const struct write_pack_opts *opts, + struct existing_packs *existing, + struct string_list *names); + +int write_cruft_pack(const struct write_pack_opts *opts, + const char *cruft_expiration, + unsigned long combine_cruft_below_size, + struct string_list *names, + struct existing_packs *existing); + +#endif /* REPACK_H */ @@ -166,6 +166,7 @@ static int set_recommended_config(int reconfigure) #endif /* Optional */ { "status.aheadBehind", "false" }, + { "commitGraph.changedPaths", "true" }, { "commitGraph.generationVersion", "1" }, { "core.autoCRLF", "false" }, { "core.safeCRLF", "false" }, diff --git a/server-info.c b/server-info.c index 1d33de821e..b9a710544a 100644 --- a/server-info.c +++ b/server-info.c @@ -287,13 +287,12 @@ static int compare_info(const void *a_, const void *b_) static void init_pack_info(struct repository *r, const char *infofile, int force) { - struct packfile_store *packs = r->objects->packfiles; struct packed_git *p; int stale; int i; size_t alloc = 0; - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(r, p) { /* we ignore things on alternate path since they are * not available to the pullers in general. */ diff --git a/sparse-index.c b/sparse-index.c index 5634abafaa..76f90da5f5 100644 --- a/sparse-index.c +++ b/sparse-index.c @@ -32,7 +32,9 @@ int give_advice_on_expansion = 1; "Your working directory likely has contents that are outside of\n" \ "your sparse-checkout patterns. Use 'git sparse-checkout list' to\n" \ "see your sparse-checkout definition and compare it to your working\n" \ - "directory contents. Running 'git clean' may assist in this cleanup." + "directory contents. Cleaning up any merge conflicts or staged\n" \ + "changes before running 'git sparse-checkout clean' or 'git\n" \ + "sparse-checkout reapply' may assist in this cleanup." struct modify_index_context { struct index_state *write; diff --git a/src/cargo-meson.sh b/src/cargo-meson.sh index 99400986d9..3998db0435 100755 --- a/src/cargo-meson.sh +++ b/src/cargo-meson.sh @@ -26,7 +26,14 @@ then exit $RET fi -if ! cmp "$BUILD_DIR/$BUILD_TYPE/libgitcore.a" "$BUILD_DIR/libgitcore.a" >/dev/null 2>&1 +case "$(cargo -vV | sed -s 's/^host: \(.*\)$/\1/')" in + *-windows-*) + LIBNAME=gitcore.lib;; + *) + LIBNAME=libgitcore.a;; +esac + +if ! cmp "$BUILD_DIR/$BUILD_TYPE/$LIBNAME" "$BUILD_DIR/libgitcore.a" >/dev/null 2>&1 then - cp "$BUILD_DIR/$BUILD_TYPE/libgitcore.a" "$BUILD_DIR/libgitcore.a" + cp "$BUILD_DIR/$BUILD_TYPE/$LIBNAME" "$BUILD_DIR/libgitcore.a" fi diff --git a/src/varint.rs b/src/varint.rs index 6e610bdd8e..06492dfc5e 100644 --- a/src/varint.rs +++ b/src/varint.rs @@ -1,3 +1,10 @@ +/// Decode the variable-length integer stored in `bufp` and return the decoded value. +/// +/// Returns 0 in case the decoded integer would overflow u64::MAX. +/// +/// # Safety +/// +/// The buffer must be NUL-terminated to ensure safety. #[no_mangle] pub unsafe extern "C" fn decode_varint(bufp: *mut *const u8) -> u64 { let mut buf = *bufp; @@ -22,6 +29,14 @@ pub unsafe extern "C" fn decode_varint(bufp: *mut *const u8) -> u64 { val } +/// Encode `value` into `buf` as a variable-length integer unless `buf` is null. +/// +/// Returns the number of bytes written, or, if `buf` is null, the number of bytes that would be +/// written to encode the integer. +/// +/// # Safety +/// +/// `buf` must either be null or point to at least 16 bytes of memory. #[no_mangle] pub unsafe extern "C" fn encode_varint(value: u64, buf: *mut u8) -> u8 { let mut varint: [u8; 16] = [0; 16]; diff --git a/t/helper/test-find-pack.c b/t/helper/test-find-pack.c index e001dc3066..fc4b8a77b3 100644 --- a/t/helper/test-find-pack.c +++ b/t/helper/test-find-pack.c @@ -39,11 +39,12 @@ int cmd__find_pack(int argc, const char **argv) if (repo_get_oid(the_repository, argv[0], &oid)) die("cannot parse %s as an object name", argv[0]); - for (p = packfile_store_get_all_packs(the_repository->objects->packfiles); p; p = p->next) + repo_for_each_pack(the_repository, p) { if (find_pack_entry_one(&oid, p)) { printf("%s\n", p->pack_name); actual_count++; } + } if (count > -1 && count != actual_count) die("bad packfile count %d instead of %d", actual_count, count); diff --git a/t/helper/test-pack-mtimes.c b/t/helper/test-pack-mtimes.c index 7c428c1601..7a8ee1de24 100644 --- a/t/helper/test-pack-mtimes.c +++ b/t/helper/test-pack-mtimes.c @@ -37,7 +37,7 @@ int cmd__pack_mtimes(int argc, const char **argv) if (argc != 2) usage(pack_mtimes_usage); - for (p = packfile_store_get_all_packs(the_repository->objects->packfiles); p; p = p->next) { + repo_for_each_pack(the_repository, p) { strbuf_addstr(&buf, basename(p->pack_name)); strbuf_strip_suffix(&buf, ".pack"); strbuf_addstr(&buf, ".mtimes"); diff --git a/t/lib-gpg.sh b/t/lib-gpg.sh index 937b876bd0..b99ae39a06 100644 --- a/t/lib-gpg.sh +++ b/t/lib-gpg.sh @@ -9,6 +9,16 @@ GNUPGHOME="$(pwd)/gpghome" export GNUPGHOME +# All the "test_lazy_prereq GPG*" below should use +# `prepare_gnupghome()` either directly or through a call to +# `test_have_prereq GPG*`. That's because `gpg` and `gpgsm` +# only create the directory specified using "$GNUPGHOME" or +# `--homedir` if it's the default (usually "~/.gnupg"). +prepare_gnupghome() { + mkdir -p "$GNUPGHOME" && + chmod 0700 "$GNUPGHOME" +} + test_lazy_prereq GPG ' gpg_version=$(gpg --version 2>&1) test $? != 127 || exit 1 @@ -38,8 +48,7 @@ test_lazy_prereq GPG ' # To export ownertrust: # gpg --homedir /tmp/gpghome --export-ownertrust \ # > lib-gpg/ownertrust - mkdir "$GNUPGHOME" && - chmod 0700 "$GNUPGHOME" && + prepare_gnupghome && (gpgconf --kill all || : ) && gpg --homedir "${GNUPGHOME}" --import \ "$TEST_DIRECTORY"/lib-gpg/keyring.gpg && @@ -63,6 +72,14 @@ test_lazy_prereq GPG2 ' ;; *) (gpgconf --kill all || : ) && + + # NEEDSWORK: prepare_gnupghome() should definitely be + # called here, but it looks like it exposes a + # pre-existing, hidden bug by allowing some tests in + # t1016-compatObjectFormat.sh to run instead of being + # skipped. See: + # https://lore.kernel.org/git/ZoV8b2RvYxLOotSJ@teonanacatl.net/ + gpg --homedir "${GNUPGHOME}" --import \ "$TEST_DIRECTORY"/lib-gpg/keyring.gpg && gpg --homedir "${GNUPGHOME}" --import-ownertrust \ @@ -132,8 +149,7 @@ test_lazy_prereq GPGSSH ' test $? = 0 || exit 1; # Setup some keys and an allowed signers file - mkdir -p "${GNUPGHOME}" && - chmod 0700 "${GNUPGHOME}" && + prepare_gnupghome && (setfacl -k "${GNUPGHOME}" 2>/dev/null || true) && ssh-keygen -t ed25519 -N "" -C "git ed25519 key" -f "${GPGSSH_KEY_PRIMARY}" >/dev/null && ssh-keygen -t rsa -b 2048 -N "" -C "git rsa2048 key" -f "${GPGSSH_KEY_SECONDARY}" >/dev/null && diff --git a/t/meson.build b/t/meson.build index 401b24e50e..c9ddd89889 100644 --- a/t/meson.build +++ b/t/meson.build @@ -1037,6 +1037,7 @@ integration_tests = [ 't9303-fast-import-compression.sh', 't9304-fast-import-marks.sh', 't9305-fast-import-signatures.sh', + 't9306-fast-import-signed-tags.sh', 't9350-fast-export.sh', 't9351-fast-export-anonymize.sh', 't9400-git-cvsserver-server.sh', diff --git a/t/t0600-reffiles-backend.sh b/t/t0600-reffiles-backend.sh index 1e62c791d9..b11126ed47 100755 --- a/t/t0600-reffiles-backend.sh +++ b/t/t0600-reffiles-backend.sh @@ -477,9 +477,29 @@ test_expect_success SYMLINKS 'symref transaction supports symlinks' ' prepare commit EOF - git update-ref --no-deref --stdin <stdin && - test_path_is_symlink .git/TEST_SYMREF_HEAD && - test "$(test_readlink .git/TEST_SYMREF_HEAD)" = refs/heads/new + git update-ref --no-deref --stdin <stdin 2>err && + if test_have_prereq WITH_BREAKING_CHANGES + then + test_path_is_file .git/TEST_SYMREF_HEAD && + echo "ref: refs/heads/new" >expect && + test_cmp expect .git/TEST_SYMREF_HEAD && + test_must_be_empty err + else + test_path_is_symlink .git/TEST_SYMREF_HEAD && + test "$(test_readlink .git/TEST_SYMREF_HEAD)" = refs/heads/new && + cat >expect <<-EOF && + warning: ${SQ}core.preferSymlinkRefs=true${SQ} is nominated for removal. + hint: The use of symbolic links for symbolic refs is deprecated + hint: and will be removed in Git 3.0. The configuration that + hint: tells Git to use them is thus going away. You can unset + hint: it with: + hint: + hint: git config unset core.preferSymlinkRefs + hint: + hint: Git will then use the textual symref format instead. + EOF + test_cmp expect err + fi ' test_expect_success 'symref transaction supports false symlink config' ' diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index ab3a105fff..b2da4feaef 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -1050,5 +1050,180 @@ test_expect_success 'check-rules null termination' ' test_cmp expect actual ' +test_expect_success 'clean' ' + git -C repo sparse-checkout set --cone deep/deeper1 && + git -C repo sparse-checkout reapply && + mkdir -p repo/deep/deeper2 repo/folder1/extra/inside && + + # Add untracked files + touch repo/deep/deeper2/file && + touch repo/folder1/extra/inside/file && + + test_must_fail git -C repo sparse-checkout clean 2>err && + grep "refusing to clean" err && + + git -C repo config clean.requireForce true && + test_must_fail git -C repo sparse-checkout clean 2>err && + grep "refusing to clean" err && + + cat >expect <<-\EOF && + Would remove deep/deeper2/ + Would remove folder1/ + EOF + + git -C repo sparse-checkout clean --dry-run >out && + test_cmp expect out && + test_path_exists repo/deep/deeper2 && + test_path_exists repo/folder1/extra/inside/file && + + cat >expect <<-\EOF && + Would remove deep/deeper2/file + Would remove folder1/extra/inside/file + EOF + + git -C repo sparse-checkout clean --dry-run --verbose >out && + test_cmp expect out && + + cat >expect <<-\EOF && + Removing deep/deeper2/ + Removing folder1/ + EOF + + git -C repo sparse-checkout clean -f >out && + test_cmp expect out && + + test_path_is_missing repo/deep/deeper2 && + test_path_is_missing repo/folder1 +' + +test_expect_success 'clean with sparse file states' ' + test_when_finished git reset --hard && + git -C repo sparse-checkout set --cone deep/deeper1 && + mkdir repo/folder2 && + + # The previous test case checked the -f option, so + # test the config option in this one. + git -C repo config clean.requireForce false && + + # create an untracked file and a modified file + touch repo/folder2/file && + echo dirty >repo/folder2/a && + + # First clean/reapply pass will do nothing. + git -C repo sparse-checkout clean >out && + test_must_be_empty out && + test_path_exists repo/folder2/a && + test_path_exists repo/folder2/file && + + git -C repo sparse-checkout reapply 2>err && + test_grep folder2 err && + test_path_exists repo/folder2/a && + test_path_exists repo/folder2/file && + + # Now, stage the change to the tracked file. + git -C repo add --sparse folder2/a && + + # Clean will continue not doing anything. + git -C repo sparse-checkout clean >out && + test_line_count = 0 out && + test_path_exists repo/folder2/a && + test_path_exists repo/folder2/file && + + # But we can reapply to remove the staged change. + git -C repo sparse-checkout reapply 2>err && + test_grep folder2 err && + test_path_is_missing repo/folder2/a && + test_path_exists repo/folder2/file && + + # We can clean now. + cat >expect <<-\EOF && + Removing folder2/ + EOF + git -C repo sparse-checkout clean >out && + test_cmp expect out && + test_path_is_missing repo/folder2 && + + # At the moment, the file is staged. + cat >expect <<-\EOF && + M folder2/a + EOF + + git -C repo status -s >out && + test_cmp expect out && + + # Reapply persists the modified state. + git -C repo sparse-checkout reapply && + cat >expect <<-\EOF && + M folder2/a + EOF + git -C repo status -s >out && + test_cmp expect out && + + # Committing the change leads to resolved status. + git -C repo commit -m "modified" && + git -C repo status -s >out && + test_must_be_empty out && + + # Repeat, but this time commit before reapplying. + mkdir repo/folder2/ && + echo dirtier >repo/folder2/a && + git -C repo add --sparse folder2/a && + git -C repo sparse-checkout clean >out && + test_must_be_empty out && + test_path_exists repo/folder2/a && + + # Committing without reapplying makes it look like a deletion + # due to no skip-worktree bit. + git -C repo commit -m "dirtier" && + git -C repo status -s >out && + test_must_be_empty out && + + git -C repo sparse-checkout reapply && + git -C repo status -s >out && + test_must_be_empty out +' + +test_expect_success 'sparse-checkout operations with merge conflicts' ' + git clone repo merge && + + ( + cd merge && + mkdir -p folder1/even/more/dirs && + echo base >folder1/even/more/dirs/file && + git add folder1 && + git commit -m "base" && + + git checkout -b right&& + echo right >folder1/even/more/dirs/file && + git commit -a -m "right" && + + git checkout -b left HEAD~1 && + echo left >folder1/even/more/dirs/file && + git commit -a -m "left" && + + git checkout -b merge && + git sparse-checkout set deep/deeper1 && + + test_must_fail git merge -m "will-conflict" right && + + test_must_fail git sparse-checkout clean -f 2>err && + grep "failed to convert index to a sparse index" err && + + echo merged >folder1/even/more/dirs/file && + git add --sparse folder1 && + git merge --continue && + + test_path_exists folder1/even/more/dirs/file && + + # clean does not remove the file, because the + # SKIP_WORKTREE bit was not cleared by the merge command. + git sparse-checkout clean -f >out && + test_line_count = 0 out && + test_path_exists folder1/even/more/dirs/file && + + git sparse-checkout reapply && + test_path_is_missing folder1 + ) +' test_done diff --git a/t/t4013-diff-various.sh b/t/t4013-diff-various.sh index 55a06eadb3..d35695f5b0 100755 --- a/t/t4013-diff-various.sh +++ b/t/t4013-diff-various.sh @@ -661,6 +661,43 @@ test_expect_success 'diff -I<regex>: ignore matching file' ' test_grep ! "file1" actual ' +test_expect_success 'diff -I<regex>: ignore all content changes' ' + test_when_finished "git rm -f file1 file2 file3" && + : >file1 && + git add file1 && + : >file2 && + git add file2 && + : >file3 && + git add file3 && + + rm -f file1 file2 && + mkdir file2 && + echo "A" >file3 && + A_hash=$(git hash-object -w file3) && + echo "B" >file3 && + B_hash=$(git hash-object -w file3) && + cat <<-EOF | git update-index --index-info && + 100644 $A_hash 1 file3 + 100644 $B_hash 2 file3 + EOF + + test_diff_no_content_changes () { + git diff $1 --ignore-blank-lines -I".*" >actual && + test_line_count = 3 actual && + test_grep "file1" actual && + test_grep "file2" actual && + test_grep "file3" actual && + test_grep ! "diff --git" actual + } && + test_diff_no_content_changes "--raw" && + test_diff_no_content_changes "--name-only" && + test_diff_no_content_changes "--name-status" && + + : >actual && + test_must_fail git diff --quiet -I".*" >actual && + test_must_be_empty actual +' + # check_prefix <patch> <src> <dst> # check only lines with paths to avoid dependency on exact oid/contents check_prefix () { diff --git a/t/t5318-commit-graph.sh b/t/t5318-commit-graph.sh index 0b3404f58f..98c6910963 100755 --- a/t/t5318-commit-graph.sh +++ b/t/t5318-commit-graph.sh @@ -946,4 +946,48 @@ test_expect_success 'stale commit cannot be parsed when traversing graph' ' ) ' +test_expect_success 'config commitGraph.changedPaths acts like --changed-paths' ' + git init config-changed-paths && + ( + cd config-changed-paths && + + # commitGraph.changedPaths is not set and it should not write Bloom filters + test_commit first && + GIT_PROGRESS_DELAY=0 git commit-graph write --reachable --progress 2>error && + test_grep ! "Bloom filters" error && + + # Set commitGraph.changedPaths to true and it should write Bloom filters + test_commit second && + git config commitGraph.changedPaths true && + GIT_PROGRESS_DELAY=0 git commit-graph write --reachable --progress 2>error && + test_grep "Bloom filters" error && + + # Add one more config commitGraph.changedPaths as false to disable the previous true config value + # It should still write Bloom filters due to existing filters + test_commit third && + git config --add commitGraph.changedPaths false && + GIT_PROGRESS_DELAY=0 git commit-graph write --reachable --progress 2>error && + test_grep "Bloom filters" error && + + # commitGraph.changedPaths is still false and command line options should take precedence + test_commit fourth && + GIT_PROGRESS_DELAY=0 git commit-graph write --no-changed-paths --reachable --progress 2>error && + test_grep ! "Bloom filters" error && + GIT_PROGRESS_DELAY=0 git commit-graph write --reachable --progress 2>error && + test_grep ! "Bloom filters" error && + + # commitGraph.changedPaths is all cleared and then set to false again, command line options should take precedence + test_commit fifth && + git config --unset-all commitGraph.changedPaths && + git config commitGraph.changedPaths false && + GIT_PROGRESS_DELAY=0 git commit-graph write --changed-paths --reachable --progress 2>error && + test_grep "Bloom filters" error && + + # commitGraph.changedPaths is still false and it should write Bloom filters due to existing filters + test_commit sixth && + GIT_PROGRESS_DELAY=0 git commit-graph write --reachable --progress 2>error && + test_grep "Bloom filters" error + ) +' + test_done diff --git a/t/t9306-fast-import-signed-tags.sh b/t/t9306-fast-import-signed-tags.sh new file mode 100755 index 0000000000..363619e7d1 --- /dev/null +++ b/t/t9306-fast-import-signed-tags.sh @@ -0,0 +1,80 @@ +#!/bin/sh + +test_description='git fast-import --signed-tags=<mode>' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main + +. ./test-lib.sh +. "$TEST_DIRECTORY/lib-gpg.sh" + +test_expect_success 'set up unsigned initial commit and import repo' ' + test_commit first && + git init new +' + +test_expect_success 'import no signed tag with --signed-tags=abort' ' + git fast-export --signed-tags=verbatim >output && + git -C new fast-import --quiet --signed-tags=abort <output +' + +test_expect_success GPG 'set up OpenPGP signed tag' ' + git tag -s -m "OpenPGP signed tag" openpgp-signed first && + OPENPGP_SIGNED=$(git rev-parse --verify refs/tags/openpgp-signed) && + git fast-export --signed-tags=verbatim openpgp-signed >output +' + +test_expect_success GPG 'import OpenPGP signed tag with --signed-tags=abort' ' + test_must_fail git -C new fast-import --quiet --signed-tags=abort <output +' + +test_expect_success GPG 'import OpenPGP signed tag with --signed-tags=verbatim' ' + git -C new fast-import --quiet --signed-tags=verbatim <output >log 2>&1 && + IMPORTED=$(git -C new rev-parse --verify refs/tags/openpgp-signed) && + test $OPENPGP_SIGNED = $IMPORTED && + test_must_be_empty log +' + +test_expect_success GPGSM 'setup X.509 signed tag' ' + test_config gpg.format x509 && + test_config user.signingkey $GIT_COMMITTER_EMAIL && + + git tag -s -m "X.509 signed tag" x509-signed first && + X509_SIGNED=$(git rev-parse --verify refs/tags/x509-signed) && + git fast-export --signed-tags=verbatim x509-signed >output +' + +test_expect_success GPGSM 'import X.509 signed tag with --signed-tags=warn-strip' ' + git -C new fast-import --quiet --signed-tags=warn-strip <output >log 2>&1 && + test_grep "stripping a tag signature for tag '\''x509-signed'\''" log && + IMPORTED=$(git -C new rev-parse --verify refs/tags/x509-signed) && + test $X509_SIGNED != $IMPORTED && + git -C new cat-file -p x509-signed >out && + test_grep ! "SIGNED MESSAGE" out +' + +test_expect_success GPGSSH 'setup SSH signed tag' ' + test_config gpg.format ssh && + test_config user.signingkey "${GPGSSH_KEY_PRIMARY}" && + + git tag -s -m "SSH signed tag" ssh-signed first && + SSH_SIGNED=$(git rev-parse --verify refs/tags/ssh-signed) && + git fast-export --signed-tags=verbatim ssh-signed >output +' + +test_expect_success GPGSSH 'import SSH signed tag with --signed-tags=warn-verbatim' ' + git -C new fast-import --quiet --signed-tags=warn-verbatim <output >log 2>&1 && + test_grep "importing a tag signature verbatim for tag '\''ssh-signed'\''" log && + IMPORTED=$(git -C new rev-parse --verify refs/tags/ssh-signed) && + test $SSH_SIGNED = $IMPORTED +' + +test_expect_success GPGSSH 'import SSH signed tag with --signed-tags=strip' ' + git -C new fast-import --quiet --signed-tags=strip <output >log 2>&1 && + test_must_be_empty log && + IMPORTED=$(git -C new rev-parse --verify refs/tags/ssh-signed) && + test $SSH_SIGNED != $IMPORTED && + git -C new cat-file -p ssh-signed >out && + test_grep ! "SSH SIGNATURE" out +' + +test_done diff --git a/t/t9350-fast-export.sh b/t/t9350-fast-export.sh index 8f85c69d62..3d153a4805 100755 --- a/t/t9350-fast-export.sh +++ b/t/t9350-fast-export.sh @@ -35,6 +35,7 @@ test_expect_success 'setup' ' git commit -m sitzt file2 && test_tick && git tag -a -m valentin muss && + ANNOTATED_TAG_COUNT=1 && git merge -s ours main ' @@ -229,7 +230,8 @@ EOF test_expect_success 'set up faked signed tag' ' - git fast-import <signed-tag-import + git fast-import <signed-tag-import && + ANNOTATED_TAG_COUNT=$((ANNOTATED_TAG_COUNT + 1)) ' @@ -277,6 +279,42 @@ test_expect_success 'signed-tags=warn-strip' ' test -s err ' +test_expect_success GPGSM 'setup X.509 signed tag' ' + test_config gpg.format x509 && + test_config user.signingkey $GIT_COMMITTER_EMAIL && + + git tag -s -m "X.509 signed tag" x509-signed $(git rev-parse HEAD) && + ANNOTATED_TAG_COUNT=$((ANNOTATED_TAG_COUNT + 1)) +' + +test_expect_success GPGSM 'signed-tags=verbatim with X.509' ' + git fast-export --signed-tags=verbatim x509-signed > output && + test_grep "SIGNED MESSAGE" output +' + +test_expect_success GPGSM 'signed-tags=strip with X.509' ' + git fast-export --signed-tags=strip x509-signed > output && + test_grep ! "SIGNED MESSAGE" output +' + +test_expect_success GPGSSH 'setup SSH signed tag' ' + test_config gpg.format ssh && + test_config user.signingkey "${GPGSSH_KEY_PRIMARY}" && + + git tag -s -m "SSH signed tag" ssh-signed $(git rev-parse HEAD) && + ANNOTATED_TAG_COUNT=$((ANNOTATED_TAG_COUNT + 1)) +' + +test_expect_success GPGSSH 'signed-tags=verbatim with SSH' ' + git fast-export --signed-tags=verbatim ssh-signed > output && + test_grep "SSH SIGNATURE" output +' + +test_expect_success GPGSSH 'signed-tags=strip with SSH' ' + git fast-export --signed-tags=strip ssh-signed > output && + test_grep ! "SSH SIGNATURE" output +' + test_expect_success GPG 'set up signed commit' ' # Generate a commit with both "gpgsig" and "encoding" set, so @@ -491,8 +529,9 @@ test_expect_success 'fast-export -C -C | fast-import' ' test_expect_success 'fast-export | fast-import when main is tagged' ' git tag -m msg last && + ANNOTATED_TAG_COUNT=$((ANNOTATED_TAG_COUNT + 1)) && git fast-export -C -C --signed-tags=strip --all > output && - test $(grep -c "^tag " output) = 3 + test $(grep -c "^tag " output) = $ANNOTATED_TAG_COUNT ' @@ -506,12 +545,13 @@ test_expect_success 'cope with tagger-less tags' ' TAG=$(git hash-object --literally -t tag -w tag-content) && git update-ref refs/tags/sonnenschein $TAG && + ANNOTATED_TAG_COUNT=$((ANNOTATED_TAG_COUNT + 1)) && git fast-export -C -C --signed-tags=strip --all > output && - test $(grep -c "^tag " output) = 4 && + test $(grep -c "^tag " output) = $ANNOTATED_TAG_COUNT && ! grep "Unspecified Tagger" output && git fast-export -C -C --signed-tags=strip --all \ --fake-missing-tagger > output && - test $(grep -c "^tag " output) = 4 && + test $(grep -c "^tag " output) = $ANNOTATED_TAG_COUNT && grep "Unspecified Tagger" output ' |